class TestUnitProtocolParameterDict(TestUnitStringsDict):
"""
Test cases for instrument driver class. Functions in this class provide
instrument driver unit tests and provide a tutorial on use of
the driver interface.
"""
__test__ = True
@staticmethod
def pick_byte2(input_val):
""" Get the 2nd byte as an example of something tricky and
arbitrary"""
val = int(input_val) >> 8
val &= 255
return val
def setUp(self):
self.param_dict = ProtocolParameterDict()
self.param_dict.add("foo", r'.*foo=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
direct_access=True,
startup_param=True,
default_value=10,
visibility=ParameterDictVisibility.READ_WRITE)
self.param_dict.add("bar", r'.*bar=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
direct_access=False,
startup_param=True,
default_value=15,
visibility=ParameterDictVisibility.READ_WRITE)
self.param_dict.add("baz", r'.*baz=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
direct_access=True,
default_value=20,
visibility=ParameterDictVisibility.DIRECT_ACCESS,
get_timeout=30,
set_timeout=40,
display_name="Baz",
description="The baz parameter",
type=ParameterDictType.INT,
units="nano-bazers",
value_description="Should be an integer between 2 and 2000")
self.param_dict.add("bat", r'.*bat=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
startup_param=False,
default_value=20,
visibility=ParameterDictVisibility.READ_ONLY,
get_timeout=10,
set_timeout=20,
display_name="Bat",
description="The bat parameter",
type=ParameterDictType.INT,
units="nano-batbit",
value_description="Should be an integer between 1 and 1000")
self.param_dict.add("qux", r'.*qux=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
startup_param=False,
visibility=ParameterDictVisibility.READ_ONLY)
self.param_dict.add("pho", r'.*qux=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
startup_param=False,
visibility=ParameterDictVisibility.IMMUTABLE)
self.param_dict.add("dil", r'.*qux=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
startup_param=False,
visibility=ParameterDictVisibility.IMMUTABLE)
self.param_dict.add("qut", r'.*qut=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
direct_access=True,
default_value=[10, 100],
visibility=ParameterDictVisibility.DIRECT_ACCESS,
expiration=1,
get_timeout=10,
set_timeout=20,
display_name="Qut",
description="The qut list parameter",
type=ParameterDictType.LIST,
units="nano-qutters",
value_description="Should be a 2-10 element list of integers between 2 and 2000")
self.target_schema = {
"bar": {
"direct_access": False,
"get_timeout": 10,
"set_timeout": 10,
"startup": True,
"value": {
"default": 15
},
"visibility": "READ_WRITE",
"range": None,
},
"bat": {
"description": "The bat parameter",
"direct_access": False,
"display_name": "Bat",
"get_timeout": 10,
"set_timeout": 20,
"startup": False,
"value": {
"default": 20,
"description": "Should be an integer between 1 and 1000",
"type": "int",
"units": "nano-batbit"
},
"visibility": "READ_ONLY",
"range": None,
},
"baz": {
"description": "The baz parameter",
"direct_access": True,
"display_name": "Baz",
"get_timeout": 30,
"set_timeout": 40,
"startup": False,
"value": {
"default": 20,
"description": "Should be an integer between 2 and 2000",
"type": "int",
"units": "nano-bazers"
},
"visibility": "DIRECT_ACCESS",
"range": None,
},
"dil": {
"direct_access": False,
"get_timeout": 10,
"set_timeout": 10,
"startup": False,
"value": {},
"visibility": "IMMUTABLE",
"range": None,
},
"foo": {
"direct_access": True,
"get_timeout": 10,
"set_timeout": 10,
"startup": True,
"value": {
"default": 10
},
"visibility": "READ_WRITE",
"range": None,
},
"pho": {
"direct_access": False,
"get_timeout": 10,
"set_timeout": 10,
"startup": False,
"value": {},
"visibility": "IMMUTABLE",
"range": None,
},
"qut": {
"description": "The qut list parameter",
"direct_access": True,
"display_name": "Qut",
"get_timeout": 10,
"set_timeout": 20,
"startup": False,
"value": {
"default": [
10,
100
],
"description": "Should be a 2-10 element list of integers between 2 and 2000",
"type": "list",
"units": "nano-qutters"
},
"visibility": "DIRECT_ACCESS",
"range": None,
},
"qux": {
"direct_access": False,
"get_timeout": 10,
"set_timeout": 10,
"startup": False,
"value": {},
"visibility": "READ_ONLY",
"range": None,
}
}
self.test_yaml = '''
parameters: {
qut: {
description: "QutFileDesc",
units: "QutFileUnits",
value_description: "QutFileValueDesc",
type: "QutFileType",
display_name: "QutDisplay"
},
extra_param: {
description: "ExtraFileDesc",
units: "ExtraFileUnits",
value_description: "ExtraFileValueDesc",
type: "ExtraFileType"
}
}
commands: {
dummy: stuff
}
'''
def test_get_direct_access_list(self):
"""
Test to see we can get a list of direct access parameters
"""
result = self.param_dict.get_direct_access_list()
self.assertTrue(isinstance(result, list))
self.assertEquals(len(result), 3)
self.assert_("foo" in result)
self.assert_("baz" in result)
self.assert_("qut" in result)
def test_get_startup_list(self):
"""
Test to see we can get a list of direct access parameters
"""
result = self.param_dict.get_startup_list()
self.assertTrue(isinstance(result, list))
self.assertEquals(len(result), 2)
self.assert_("foo" in result)
self.assert_("bar" in result)
def test_set_default(self):
"""
Test setting a default value
"""
result = self.param_dict.get_config()
self.assertEquals(result["foo"], None)
self.param_dict.set_default("foo")
self.assertEquals(self.param_dict.get("foo"), 10)
self.param_dict.update("foo=1000")
self.assertEquals(self.param_dict.get("foo"), 1000)
self.param_dict.set_default("foo")
self.assertEquals(self.param_dict.get("foo"), 10)
self.assertRaises(ValueError, self.param_dict.set_default, "qux")
def test_update_many(self):
"""
Test updating of multiple variables from the same input
"""
sample_input = """
foo=100
bar=200, baz=300
"""
self.assertNotEquals(self.param_dict.get("foo"), 100)
self.assertNotEquals(self.param_dict.get("bar"), 200)
self.assertNotEquals(self.param_dict.get("baz"), 300)
result = self.param_dict.update_many(sample_input)
log.debug("result: %s", result)
self.assertEquals(result["foo"], True)
self.assertEquals(result["bar"], True)
self.assertEquals(result["baz"], True)
self.assertEquals(self.param_dict.get("foo"), 100)
self.assertEquals(self.param_dict.get("bar"), 200)
self.assertEquals(self.param_dict.get("baz"), 300)
def test_update_specific_values(self):
"""
test to verify we can limit update to a specific
set of parameters
"""
sample_input = "foo=100, bar=200"
# First verify we can set both
self.assertNotEquals(self.param_dict.get("foo"), 100)
self.assertNotEquals(self.param_dict.get("bar"), 200)
self.assertTrue(self.param_dict.update(sample_input))
self.assertEquals(self.param_dict.get("foo"), 100)
self.assertEquals(self.param_dict.get("bar"), 200)
# Now let's only have it update 1 parameter with a name
sample_input = "foo=200, bar=300"
self.assertTrue(self.param_dict.update(sample_input, target_params="foo"))
self.assertEquals(self.param_dict.get("foo"), 200)
self.assertEquals(self.param_dict.get("bar"), 200)
# Now let's only have it update 1 parameter using a list
sample_input = "foo=300, bar=400"
self.assertTrue(self.param_dict.update(sample_input, target_params=["foo"]))
self.assertEquals(self.param_dict.get("foo"), 300)
self.assertEquals(self.param_dict.get("bar"), 200)
# Test our exceptions
with self.assertRaises(KeyError):
self.param_dict.update(sample_input, "key_does_not_exist")
with self.assertRaises(InstrumentParameterException):
self.param_dict.update(sample_input, {'bad': "key_does_not_exist"})
def test_visibility_list(self):
lst = self.param_dict.get_visibility_list(ParameterDictVisibility.READ_WRITE)
lst.sort()
self.assertEquals(lst, ["bar", "foo"])
lst = self.param_dict.get_visibility_list(ParameterDictVisibility.DIRECT_ACCESS)
lst.sort()
self.assertEquals(lst, ["baz", "qut"])
lst = self.param_dict.get_visibility_list(ParameterDictVisibility.READ_ONLY)
lst.sort()
self.assertEquals(lst, ["bat", "qux"])
lst = self.param_dict.get_visibility_list(ParameterDictVisibility.IMMUTABLE)
lst.sort()
self.assertEquals(lst, ["dil", "pho"])
def test_function_values(self):
"""
Make sure we can add and update values with functions instead of patterns
"""
self.param_dict.add_parameter(
FunctionParameter("fn_foo",
self.pick_byte2,
lambda x: str(x),
direct_access=True,
startup_param=True,
value=1,
visibility=ParameterDictVisibility.READ_WRITE)
)
self.param_dict.add_parameter(
FunctionParameter("fn_bar",
lambda x: bool(x & 2), # bit map example
lambda x: str(x),
direct_access=True,
startup_param=True,
value=False,
visibility=ParameterDictVisibility.READ_WRITE)
)
# check defaults just to be safe
val = self.param_dict.get("fn_foo")
self.assertEqual(val, 1)
val = self.param_dict.get("fn_bar")
self.assertEqual(val, False)
self.param_dict.update(1005) # just change first in list
val = self.param_dict.get("fn_foo")
self.assertEqual(val, 3)
val = self.param_dict.get("fn_bar")
self.assertEqual(val, False)
# fn_bar does not get updated here
result = self.param_dict.update_many(1205)
self.assertEqual(result['fn_foo'], True)
self.assertEqual(len(result), 1)
val = self.param_dict.get("fn_foo")
self.assertEqual(val, 4)
val = self.param_dict.get("fn_bar")
self.assertEqual(val, False)
# both are updated now
result = self.param_dict.update_many(6)
self.assertEqual(result['fn_foo'], True)
self.assertEqual(result['fn_bar'], True)
self.assertEqual(len(result), 2)
val = self.param_dict.get("fn_foo")
self.assertEqual(val, 0)
val = self.param_dict.get("fn_bar")
self.assertEqual(val, True)
def test_mixed_pdv_types(self):
""" Verify we can add different types of PDVs in one container """
self.param_dict.add_parameter(
FunctionParameter("fn_foo",
self.pick_byte2,
lambda x: str(x),
direct_access=True,
startup_param=True,
value=1,
visibility=ParameterDictVisibility.READ_WRITE)
)
self.param_dict.add_parameter(
RegexParameter("foo", r'.*foo=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
direct_access=True,
startup_param=True,
value=10,
visibility=ParameterDictVisibility.READ_WRITE)
)
self.param_dict.add("bar", r'.*bar=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
direct_access=False,
startup_param=True,
value=15,
visibility=ParameterDictVisibility.READ_WRITE)
self.assertEqual(self.param_dict.get("fn_foo"), 1)
self.assertEqual(self.param_dict.get("foo"), 10)
self.assertEqual(self.param_dict.get("bar"), 15)
def test_base_update(self):
pdv = Parameter("foo",
lambda x: str(x),
value=12)
self.assertEqual(pdv.get_value(), 12)
result = pdv.update(1)
self.assertEqual(result, True)
self.assertEqual(pdv.get_value(), 1)
# Its a base class...monkey see, monkey do
result = pdv.update("foo=1")
self.assertEqual(result, True)
self.assertEqual(pdv.get_value(), "foo=1")
def test_regex_val(self):
pdv = RegexParameter("foo",
r'.*foo=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
value=12)
self.assertEqual(pdv.get_value(), 12)
result = pdv.update(1)
self.assertEqual(result, False)
self.assertEqual(pdv.get_value(), 12)
result = pdv.update("foo=1")
self.assertEqual(result, True)
self.assertEqual(pdv.get_value(), 1)
def test_function_val(self):
pdv = FunctionParameter("foo",
self.pick_byte2,
lambda x: str(x),
value=12)
self.assertEqual(pdv.get_value(), 12)
self.assertRaises(TypeError, pdv.update(1))
result = pdv.update("1205")
self.assertEqual(pdv.get_value(), 4)
self.assertEqual(result, True)
def test_set_init_value(self):
result = self.param_dict.get("foo")
self.assertEqual(result, None)
self.param_dict.set_init_value("foo", 42)
result = self.param_dict.get_init_value("foo")
self.assertEqual(result, 42)
def test_schema_generation(self):
self.maxDiff = None
result = self.param_dict.generate_dict()
json_result = json.dumps(result, indent=4, sort_keys=True)
log.debug("Expected: %s", self.target_schema)
log.debug("Result: %s", json_result)
self.assertEqual(result, self.target_schema)
def test_empty_schema(self):
self.param_dict = ProtocolParameterDict()
result = self.param_dict.generate_dict()
self.assertEqual(result, {})
def test_bad_descriptions(self):
self.param_dict._param_dict["foo"].description = None
self.param_dict._param_dict["foo"].value = None
self.assertRaises(InstrumentParameterException,
self.param_dict.get_init_value, "foo")
self.assertRaises(InstrumentParameterException,
self.param_dict.get_default_value, "foo")
self.assertRaises(InstrumentParameterException,
self.param_dict.set_default, "foo")
self.assertRaises(InstrumentParameterException,
self.param_dict.get_init_value, "foo")
self.assertRaises(InstrumentParameterException,
self.param_dict.get_menu_path_read, "foo")
self.assertRaises(InstrumentParameterException,
self.param_dict.get_submenu_read, "foo")
self.assertRaises(InstrumentParameterException,
self.param_dict.get_menu_path_write, "foo")
self.assertRaises(InstrumentParameterException,
self.param_dict.get_submenu_write, "foo")
self.assertRaises(InstrumentParameterException,
self.param_dict.format, "foo", 1)
self.assertRaises(InstrumentParameterException,
self.param_dict.get_direct_access_list)
self.assertRaises(InstrumentParameterException,
self.param_dict.is_startup_param, "foo")
def test_set(self):
"""
Test a simple set of the parameter. Make sure the right values get
called and the correct exceptions are raised.
"""
new_param = FunctionParameter("foo",
self.pick_byte2,
lambda x: str(x),
direct_access=True,
startup_param=True,
value=1000,
visibility=ParameterDictVisibility.READ_WRITE)
self.assertEquals(new_param.get_value(), 1000)
self.assertEquals(self.param_dict.get("foo"), None)
# overwrites existing param
self.param_dict.add_parameter(new_param)
self.assertEquals(self.param_dict.get("foo"), 1000)
self.param_dict.set_value("foo", 2000)
self.assertEquals(self.param_dict.get("foo"), 2000)
def test_invalid_type(self):
self.assertRaises(InstrumentParameterException,
FunctionParameter,
"fn_bar",
lambda x: bool(x & 2), # bit map example
lambda x: str(x),
direct_access=True,
startup_param=True,
value=False,
type="bad_type",
visibility=ParameterDictVisibility.READ_WRITE)
def test_get(self):
"""
test getting values with expiration
"""
# from mi.core.exceptions import InstrumentParameterExpirationException
pd = ProtocolParameterDict()
# No expiration, should work just fine
pd.add('noexp', r'', None, None, expiration=None)
pd.add('zeroexp', r'', None, None, expiration=0)
pd.add('lateexp', r'', None, None, expiration=2)
###
# Set and get with no expire
###
pd.set_value('noexp', 1)
self.assertEqual(pd.get('noexp'), 1)
###
# Set and get with a 0 expire
###
basetime = pd.get_current_timestamp()
pd.set_value('zeroexp', 2)
# We should fail because we are calculating exp against current time
with self.assertRaises(InstrumentParameterExpirationException):
pd.get('zeroexp')
# Should succeed because exp is calculated using basetime
self.assertEqual(pd.get('zeroexp', basetime), 2)
###
# Set and get with a delayed expire
###
basetime = pd.get_current_timestamp()
futuretime = pd.get_current_timestamp(3)
self.assertGreater(futuretime - basetime, 3)
pd.set_value('lateexp', 2)
# Success because data is not expired
self.assertEqual(pd.get('lateexp', basetime), 2)
# Fail because data is expired (simulated three seconds from now)
with self.assertRaises(InstrumentParameterExpirationException):
pd.get('lateexp', futuretime)
def test_regex_flags(self):
pdv = RegexParameter("foo",
r'.+foo=(\d+).+',
lambda match: int(match.group(1)),
lambda x: str(x),
regex_flags=re.DOTALL,
value=12)
# Assert something good with dotall update()
self.assertTrue(pdv)
pdv.update("\n\nfoo=1212\n\n")
self.assertEqual(pdv.get_value(), 1212)
# negative test with no regex_flags
pdv = RegexParameter("foo",
r'.+foo=(\d+).+',
lambda match: int(match.group(1)),
lambda x: str(x),
value=12)
# Assert something good with dotall update()
self.assertTrue(pdv)
pdv.update("\n\nfoo=1212\n\n")
self.assertEqual(pdv.get_value(), 12)
self.assertRaises(TypeError,
RegexParameter,
"foo",
r'.*foo=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
regex_flags="bad flag",
value=12)
def test_format_current(self):
self.param_dict.add("test_format", r'.*foo=(\d+).*',
lambda match: int(match.group(1)),
lambda x: x + 5,
value=10)
self.assertEqual(self.param_dict.format("test_format", 20), 25)
self.assertEqual(self.param_dict.format("test_format"), 15)
self.assertRaises(KeyError,
self.param_dict.format, "bad_name")
def _assert_metadata_change(self):
new_dict = self.param_dict.generate_dict()
log.debug("Generated dictionary: %s", new_dict)
self.assertEqual(new_dict["qut"][ParameterDictKey.DESCRIPTION], "QutFileDesc")
self.assertEqual(new_dict["qut"][ParameterDictKey.DISPLAY_NAME], "QutDisplay")
self.assertEqual(new_dict["qut"][ParameterDictKey.VALUE][ParameterDictKey.UNITS], "QutFileUnits")
self.assertEqual(new_dict["qut"][ParameterDictKey.VALUE][ParameterDictKey.DESCRIPTION], "QutFileValueDesc")
self.assertEqual(new_dict["qut"][ParameterDictKey.VALUE][ParameterDictKey.TYPE], "QutFileType")
# Should come from hard code
# self.assertEqual(new_dict["qut"][ParameterDictKey.DISPLAY_NAME], "QutFileName")
# from base hard code
new_dict = self.param_dict.generate_dict()
self.assertEqual(new_dict["baz"][ParameterDictKey.DESCRIPTION],
"The baz parameter")
self.assertEqual(new_dict["baz"][ParameterDictKey.VALUE][ParameterDictKey.UNITS],
"nano-bazers")
self.assertEqual(new_dict["baz"][ParameterDictKey.VALUE][ParameterDictKey.DESCRIPTION],
"Should be an integer between 2 and 2000")
self.assertEqual(new_dict["baz"][ParameterDictKey.VALUE][ParameterDictKey.TYPE],
ParameterDictType.INT)
self.assertEqual(new_dict["baz"][ParameterDictKey.DISPLAY_NAME], "Baz")
self.assertTrue('extra_param' not in new_dict)
class TestUnitProtocolParameterDict(TestUnitStringsDict):
"""
Test cases for instrument driver class. Functions in this class provide
instrument driver unit tests and provide a tutorial on use of
the driver interface.
"""
__test__ = True
@staticmethod
def pick_byte2(input_val):
""" Get the 2nd byte as an example of something tricky and
arbitrary"""
val = int(input_val) >> 8
val &= 255
return val
def setUp(self):
self.param_dict = ProtocolParameterDict()
self.param_dict.add("foo",
r'.*foo=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
direct_access=True,
startup_param=True,
default_value=10,
visibility=ParameterDictVisibility.READ_WRITE)
self.param_dict.add("bar",
r'.*bar=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
direct_access=False,
startup_param=True,
default_value=15,
visibility=ParameterDictVisibility.READ_WRITE)
self.param_dict.add(
"baz",
r'.*baz=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
direct_access=True,
default_value=20,
visibility=ParameterDictVisibility.DIRECT_ACCESS,
get_timeout=30,
set_timeout=40,
display_name="Baz",
description="The baz parameter",
type=ParameterDictType.INT,
units="nano-bazers",
value_description="Should be an integer between 2 and 2000")
self.param_dict.add(
"bat",
r'.*bat=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
startup_param=False,
default_value=20,
visibility=ParameterDictVisibility.READ_ONLY,
get_timeout=10,
set_timeout=20,
display_name="Bat",
description="The bat parameter",
type=ParameterDictType.INT,
units="nano-batbit",
value_description="Should be an integer between 1 and 1000")
self.param_dict.add("qux",
r'.*qux=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
startup_param=False,
visibility=ParameterDictVisibility.READ_ONLY)
self.param_dict.add("pho",
r'.*qux=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
startup_param=False,
visibility=ParameterDictVisibility.IMMUTABLE)
self.param_dict.add("dil",
r'.*qux=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
startup_param=False,
visibility=ParameterDictVisibility.IMMUTABLE)
self.param_dict.add(
"qut",
r'.*qut=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
direct_access=True,
default_value=[10, 100],
visibility=ParameterDictVisibility.DIRECT_ACCESS,
expiration=1,
get_timeout=10,
set_timeout=20,
display_name="Qut",
description="The qut list parameter",
type=ParameterDictType.LIST,
units="nano-qutters",
value_description=
"Should be a 2-10 element list of integers between 2 and 2000")
self.target_schema = {
"bar": {
"direct_access": False,
"get_timeout": 10,
"set_timeout": 10,
"startup": True,
"value": {
"default": 15
},
"visibility": "READ_WRITE",
"range": None,
},
"bat": {
"description": "The bat parameter",
"direct_access": False,
"display_name": "Bat",
"get_timeout": 10,
"set_timeout": 20,
"startup": False,
"value": {
"default": 20,
"description": "Should be an integer between 1 and 1000",
"type": "int",
"units": "nano-batbit"
},
"visibility": "READ_ONLY",
"range": None,
},
"baz": {
"description": "The baz parameter",
"direct_access": True,
"display_name": "Baz",
"get_timeout": 30,
"set_timeout": 40,
"startup": False,
"value": {
"default": 20,
"description": "Should be an integer between 2 and 2000",
"type": "int",
"units": "nano-bazers"
},
"visibility": "DIRECT_ACCESS",
"range": None,
},
"dil": {
"direct_access": False,
"get_timeout": 10,
"set_timeout": 10,
"startup": False,
"value": {},
"visibility": "IMMUTABLE",
"range": None,
},
"foo": {
"direct_access": True,
"get_timeout": 10,
"set_timeout": 10,
"startup": True,
"value": {
"default": 10
},
"visibility": "READ_WRITE",
"range": None,
},
"pho": {
"direct_access": False,
"get_timeout": 10,
"set_timeout": 10,
"startup": False,
"value": {},
"visibility": "IMMUTABLE",
"range": None,
},
"qut": {
"description": "The qut list parameter",
"direct_access": True,
"display_name": "Qut",
"get_timeout": 10,
"set_timeout": 20,
"startup": False,
"value": {
"default": [10, 100],
"description":
"Should be a 2-10 element list of integers between 2 and 2000",
"type": "list",
"units": "nano-qutters"
},
"visibility": "DIRECT_ACCESS",
"range": None,
},
"qux": {
"direct_access": False,
"get_timeout": 10,
"set_timeout": 10,
"startup": False,
"value": {},
"visibility": "READ_ONLY",
"range": None,
}
}
self.test_yaml = '''
parameters: {
qut: {
description: "QutFileDesc",
units: "QutFileUnits",
value_description: "QutFileValueDesc",
type: "QutFileType",
display_name: "QutDisplay"
},
extra_param: {
description: "ExtraFileDesc",
units: "ExtraFileUnits",
value_description: "ExtraFileValueDesc",
type: "ExtraFileType"
}
}
commands: {
dummy: stuff
}
'''
def test_get_direct_access_list(self):
"""
Test to see we can get a list of direct access parameters
"""
result = self.param_dict.get_direct_access_list()
self.assertTrue(isinstance(result, list))
self.assertEquals(len(result), 3)
self.assert_("foo" in result)
self.assert_("baz" in result)
self.assert_("qut" in result)
def test_get_startup_list(self):
"""
Test to see we can get a list of direct access parameters
"""
result = self.param_dict.get_startup_list()
self.assertTrue(isinstance(result, list))
self.assertEquals(len(result), 2)
self.assert_("foo" in result)
self.assert_("bar" in result)
def test_set_default(self):
"""
Test setting a default value
"""
result = self.param_dict.get_config()
self.assertEquals(result["foo"], None)
self.param_dict.set_default("foo")
self.assertEquals(self.param_dict.get("foo"), 10)
self.param_dict.update("foo=1000")
self.assertEquals(self.param_dict.get("foo"), 1000)
self.param_dict.set_default("foo")
self.assertEquals(self.param_dict.get("foo"), 10)
self.assertRaises(ValueError, self.param_dict.set_default, "qux")
def test_update_many(self):
"""
Test updating of multiple variables from the same input
"""
sample_input = """
foo=100
bar=200, baz=300
"""
self.assertNotEquals(self.param_dict.get("foo"), 100)
self.assertNotEquals(self.param_dict.get("bar"), 200)
self.assertNotEquals(self.param_dict.get("baz"), 300)
result = self.param_dict.update_many(sample_input)
log.debug("result: %s", result)
self.assertEquals(result["foo"], True)
self.assertEquals(result["bar"], True)
self.assertEquals(result["baz"], True)
self.assertEquals(self.param_dict.get("foo"), 100)
self.assertEquals(self.param_dict.get("bar"), 200)
self.assertEquals(self.param_dict.get("baz"), 300)
def test_update_specific_values(self):
"""
test to verify we can limit update to a specific
set of parameters
"""
sample_input = "foo=100, bar=200"
# First verify we can set both
self.assertNotEquals(self.param_dict.get("foo"), 100)
self.assertNotEquals(self.param_dict.get("bar"), 200)
self.assertTrue(self.param_dict.update(sample_input))
self.assertEquals(self.param_dict.get("foo"), 100)
self.assertEquals(self.param_dict.get("bar"), 200)
# Now let's only have it update 1 parameter with a name
sample_input = "foo=200, bar=300"
self.assertTrue(
self.param_dict.update(sample_input, target_params="foo"))
self.assertEquals(self.param_dict.get("foo"), 200)
self.assertEquals(self.param_dict.get("bar"), 200)
# Now let's only have it update 1 parameter using a list
sample_input = "foo=300, bar=400"
self.assertTrue(
self.param_dict.update(sample_input, target_params=["foo"]))
self.assertEquals(self.param_dict.get("foo"), 300)
self.assertEquals(self.param_dict.get("bar"), 200)
# Test our exceptions
with self.assertRaises(KeyError):
self.param_dict.update(sample_input, "key_does_not_exist")
with self.assertRaises(InstrumentParameterException):
self.param_dict.update(sample_input, {'bad': "key_does_not_exist"})
def test_visibility_list(self):
lst = self.param_dict.get_visibility_list(
ParameterDictVisibility.READ_WRITE)
lst.sort()
self.assertEquals(lst, ["bar", "foo"])
lst = self.param_dict.get_visibility_list(
ParameterDictVisibility.DIRECT_ACCESS)
lst.sort()
self.assertEquals(lst, ["baz", "qut"])
lst = self.param_dict.get_visibility_list(
ParameterDictVisibility.READ_ONLY)
lst.sort()
self.assertEquals(lst, ["bat", "qux"])
lst = self.param_dict.get_visibility_list(
ParameterDictVisibility.IMMUTABLE)
lst.sort()
self.assertEquals(lst, ["dil", "pho"])
def test_function_values(self):
"""
Make sure we can add and update values with functions instead of patterns
"""
self.param_dict.add_parameter(
FunctionParameter("fn_foo",
self.pick_byte2,
lambda x: str(x),
direct_access=True,
startup_param=True,
value=1,
visibility=ParameterDictVisibility.READ_WRITE))
self.param_dict.add_parameter(
FunctionParameter(
"fn_bar",
lambda x: bool(x & 2), # bit map example
lambda x: str(x),
direct_access=True,
startup_param=True,
value=False,
visibility=ParameterDictVisibility.READ_WRITE))
# check defaults just to be safe
val = self.param_dict.get("fn_foo")
self.assertEqual(val, 1)
val = self.param_dict.get("fn_bar")
self.assertEqual(val, False)
self.param_dict.update(1005) # just change first in list
val = self.param_dict.get("fn_foo")
self.assertEqual(val, 3)
val = self.param_dict.get("fn_bar")
self.assertEqual(val, False)
# fn_bar does not get updated here
result = self.param_dict.update_many(1205)
self.assertEqual(result['fn_foo'], True)
self.assertEqual(len(result), 1)
val = self.param_dict.get("fn_foo")
self.assertEqual(val, 4)
val = self.param_dict.get("fn_bar")
self.assertEqual(val, False)
# both are updated now
result = self.param_dict.update_many(6)
self.assertEqual(result['fn_foo'], True)
self.assertEqual(result['fn_bar'], True)
self.assertEqual(len(result), 2)
val = self.param_dict.get("fn_foo")
self.assertEqual(val, 0)
val = self.param_dict.get("fn_bar")
self.assertEqual(val, True)
def test_mixed_pdv_types(self):
""" Verify we can add different types of PDVs in one container """
self.param_dict.add_parameter(
FunctionParameter("fn_foo",
self.pick_byte2,
lambda x: str(x),
direct_access=True,
startup_param=True,
value=1,
visibility=ParameterDictVisibility.READ_WRITE))
self.param_dict.add_parameter(
RegexParameter("foo",
r'.*foo=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
direct_access=True,
startup_param=True,
value=10,
visibility=ParameterDictVisibility.READ_WRITE))
self.param_dict.add("bar",
r'.*bar=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
direct_access=False,
startup_param=True,
value=15,
visibility=ParameterDictVisibility.READ_WRITE)
self.assertEqual(self.param_dict.get("fn_foo"), 1)
self.assertEqual(self.param_dict.get("foo"), 10)
self.assertEqual(self.param_dict.get("bar"), 15)
def test_base_update(self):
pdv = Parameter("foo", lambda x: str(x), value=12)
self.assertEqual(pdv.get_value(), 12)
result = pdv.update(1)
self.assertEqual(result, True)
self.assertEqual(pdv.get_value(), 1)
# Its a base class...monkey see, monkey do
result = pdv.update("foo=1")
self.assertEqual(result, True)
self.assertEqual(pdv.get_value(), "foo=1")
def test_regex_val(self):
pdv = RegexParameter("foo",
r'.*foo=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
value=12)
self.assertEqual(pdv.get_value(), 12)
result = pdv.update(1)
self.assertEqual(result, False)
self.assertEqual(pdv.get_value(), 12)
result = pdv.update("foo=1")
self.assertEqual(result, True)
self.assertEqual(pdv.get_value(), 1)
def test_function_val(self):
pdv = FunctionParameter("foo",
self.pick_byte2,
lambda x: str(x),
value=12)
self.assertEqual(pdv.get_value(), 12)
self.assertRaises(TypeError, pdv.update(1))
result = pdv.update("1205")
self.assertEqual(pdv.get_value(), 4)
self.assertEqual(result, True)
def test_set_init_value(self):
result = self.param_dict.get("foo")
self.assertEqual(result, None)
self.param_dict.set_init_value("foo", 42)
result = self.param_dict.get_init_value("foo")
self.assertEqual(result, 42)
def test_schema_generation(self):
self.maxDiff = None
result = self.param_dict.generate_dict()
json_result = json.dumps(result, indent=4, sort_keys=True)
log.debug("Expected: %s", self.target_schema)
log.debug("Result: %s", json_result)
self.assertEqual(result, self.target_schema)
def test_empty_schema(self):
self.param_dict = ProtocolParameterDict()
result = self.param_dict.generate_dict()
self.assertEqual(result, {})
def test_bad_descriptions(self):
self.param_dict._param_dict["foo"].description = None
self.param_dict._param_dict["foo"].value = None
self.assertRaises(InstrumentParameterException,
self.param_dict.get_init_value, "foo")
self.assertRaises(InstrumentParameterException,
self.param_dict.get_default_value, "foo")
self.assertRaises(InstrumentParameterException,
self.param_dict.set_default, "foo")
self.assertRaises(InstrumentParameterException,
self.param_dict.get_init_value, "foo")
self.assertRaises(InstrumentParameterException,
self.param_dict.get_menu_path_read, "foo")
self.assertRaises(InstrumentParameterException,
self.param_dict.get_submenu_read, "foo")
self.assertRaises(InstrumentParameterException,
self.param_dict.get_menu_path_write, "foo")
self.assertRaises(InstrumentParameterException,
self.param_dict.get_submenu_write, "foo")
self.assertRaises(InstrumentParameterException, self.param_dict.format,
"foo", 1)
self.assertRaises(InstrumentParameterException,
self.param_dict.get_direct_access_list)
self.assertRaises(InstrumentParameterException,
self.param_dict.is_startup_param, "foo")
def test_set(self):
"""
Test a simple set of the parameter. Make sure the right values get
called and the correct exceptions are raised.
"""
new_param = FunctionParameter(
"foo",
self.pick_byte2,
lambda x: str(x),
direct_access=True,
startup_param=True,
value=1000,
visibility=ParameterDictVisibility.READ_WRITE)
self.assertEquals(new_param.get_value(), 1000)
self.assertEquals(self.param_dict.get("foo"), None)
# overwrites existing param
self.param_dict.add_parameter(new_param)
self.assertEquals(self.param_dict.get("foo"), 1000)
self.param_dict.set_value("foo", 2000)
self.assertEquals(self.param_dict.get("foo"), 2000)
def test_invalid_type(self):
self.assertRaises(
InstrumentParameterException,
FunctionParameter,
"fn_bar",
lambda x: bool(x & 2), # bit map example
lambda x: str(x),
direct_access=True,
startup_param=True,
value=False,
type="bad_type",
visibility=ParameterDictVisibility.READ_WRITE)
def test_get(self):
"""
test getting values with expiration
"""
# from mi.core.exceptions import InstrumentParameterExpirationException
pd = ProtocolParameterDict()
# No expiration, should work just fine
pd.add('noexp', r'', None, None, expiration=None)
pd.add('zeroexp', r'', None, None, expiration=0)
pd.add('lateexp', r'', None, None, expiration=2)
###
# Set and get with no expire
###
pd.set_value('noexp', 1)
self.assertEqual(pd.get('noexp'), 1)
###
# Set and get with a 0 expire
###
basetime = pd.get_current_timestamp()
pd.set_value('zeroexp', 2)
# We should fail because we are calculating exp against current time
with self.assertRaises(InstrumentParameterExpirationException):
pd.get('zeroexp')
# Should succeed because exp is calculated using basetime
self.assertEqual(pd.get('zeroexp', basetime), 2)
###
# Set and get with a delayed expire
###
basetime = pd.get_current_timestamp()
futuretime = pd.get_current_timestamp(3)
self.assertGreater(futuretime - basetime, 3)
pd.set_value('lateexp', 2)
# Success because data is not expired
self.assertEqual(pd.get('lateexp', basetime), 2)
# Fail because data is expired (simulated three seconds from now)
with self.assertRaises(InstrumentParameterExpirationException):
pd.get('lateexp', futuretime)
def test_regex_flags(self):
pdv = RegexParameter("foo",
r'.+foo=(\d+).+',
lambda match: int(match.group(1)),
lambda x: str(x),
regex_flags=re.DOTALL,
value=12)
# Assert something good with dotall update()
self.assertTrue(pdv)
pdv.update("\n\nfoo=1212\n\n")
self.assertEqual(pdv.get_value(), 1212)
# negative test with no regex_flags
pdv = RegexParameter("foo",
r'.+foo=(\d+).+',
lambda match: int(match.group(1)),
lambda x: str(x),
value=12)
# Assert something good with dotall update()
self.assertTrue(pdv)
pdv.update("\n\nfoo=1212\n\n")
self.assertEqual(pdv.get_value(), 12)
self.assertRaises(TypeError,
RegexParameter,
"foo",
r'.*foo=(\d+).*',
lambda match: int(match.group(1)),
lambda x: str(x),
regex_flags="bad flag",
value=12)
def test_format_current(self):
self.param_dict.add("test_format",
r'.*foo=(\d+).*',
lambda match: int(match.group(1)),
lambda x: x + 5,
value=10)
self.assertEqual(self.param_dict.format("test_format", 20), 25)
self.assertEqual(self.param_dict.format("test_format"), 15)
self.assertRaises(KeyError, self.param_dict.format, "bad_name")
def _assert_metadata_change(self):
new_dict = self.param_dict.generate_dict()
log.debug("Generated dictionary: %s", new_dict)
self.assertEqual(new_dict["qut"][ParameterDictKey.DESCRIPTION],
"QutFileDesc")
self.assertEqual(new_dict["qut"][ParameterDictKey.DISPLAY_NAME],
"QutDisplay")
self.assertEqual(
new_dict["qut"][ParameterDictKey.VALUE][ParameterDictKey.UNITS],
"QutFileUnits")
self.assertEqual(
new_dict["qut"][ParameterDictKey.VALUE][
ParameterDictKey.DESCRIPTION], "QutFileValueDesc")
self.assertEqual(
new_dict["qut"][ParameterDictKey.VALUE][ParameterDictKey.TYPE],
"QutFileType")
# Should come from hard code
# self.assertEqual(new_dict["qut"][ParameterDictKey.DISPLAY_NAME], "QutFileName")
# from base hard code
new_dict = self.param_dict.generate_dict()
self.assertEqual(new_dict["baz"][ParameterDictKey.DESCRIPTION],
"The baz parameter")
self.assertEqual(
new_dict["baz"][ParameterDictKey.VALUE][ParameterDictKey.UNITS],
"nano-bazers")
self.assertEqual(
new_dict["baz"][ParameterDictKey.VALUE][
ParameterDictKey.DESCRIPTION],
"Should be an integer between 2 and 2000")
self.assertEqual(
new_dict["baz"][ParameterDictKey.VALUE][ParameterDictKey.TYPE],
ParameterDictType.INT)
self.assertEqual(new_dict["baz"][ParameterDictKey.DISPLAY_NAME], "Baz")
self.assertTrue('extra_param' not in new_dict)
class DataSetDriver(object):
"""
Base class for data set drivers. Provides:
- an interface via callback to publish data
- an interface via callback to persist driver state
- an interface via callback to handle exceptions
- an start and stop sampling
- a client interface for execute resource
Subclasses need to include harvesters and parsers and
be specialized to handle the interaction between the two.
Configurations should contain keys from the DataSetDriverConfigKey class
and should look something like this example (more full documentation in the
"Dataset Agent Architecture" page on the OOI wiki):
{
'harvester':
{
'directory': '/tmp/dsatest',
'pattern': '*.txt',
'frequency': 1,
},
'parser': {}
'driver': {
'records_per_second'
'harvester_polling_interval'
'batched_particle_count'
}
}
"""
def __init__(self, config, memento, data_callback, state_callback, exception_callback):
self._config = config
self._data_callback = data_callback
self._state_callback = state_callback
self._exception_callback = exception_callback
self._memento = memento
self._publisher_thread = None
self._verify_config()
# Updated my set_resource, defaults defined in build_param_dict
self._polling_interval = None
self._generate_particle_count = None
self._particle_count_per_second = None
self._param_dict = ProtocolParameterDict()
self._cmd_dict = ProtocolCommandDict()
self._driver_dict = DriverDict()
self._build_command_dict()
self._build_driver_dict()
self._build_param_dict()
def shutdown(self):
self.stop_sampling()
def start_sampling(self):
"""
Start a new thread to monitor for data
"""
self._start_sampling()
self._start_publisher_thread()
def stop_sampling(self):
"""
Stop the sampling thread
"""
log.debug("Stopping driver now")
self._stop_sampling()
self._stop_publisher_thread()
def _start_sampling(self):
raise NotImplementedException('virtual method needs to be specialized')
def _stop_sampling(self):
raise NotImplementedException('virtual method needs to be specialized')
def _is_sampling(self):
"""
Currently the drivers only have two states, command and streaming and
all resource commands are common, either start or stop autosample.
Therefore we didn't implement an enitre state machine to manage states
and commands. If it does get more complex than this we should take the
time to implement a state machine to add some flexibility
"""
raise NotImplementedException('virtual method needs to be specialized')
def cmd_dvr(self, cmd, *args, **kwargs):
log.warn("DRIVER: cmd_dvr %s", cmd)
if cmd == 'execute_resource':
resource_cmd = args[0]
if resource_cmd == DriverEvent.START_AUTOSAMPLE:
return (ResourceAgentState.STREAMING, None)
elif resource_cmd == DriverEvent.STOP_AUTOSAMPLE:
self.stop_sampling()
return (ResourceAgentState.COMMAND, None)
else:
log.error("Unhandled resource command: %s", resource_cmd)
raise
elif cmd == 'get_resource_capabilities':
return self.get_resource_capabilities()
elif cmd == 'set_resource':
return self.set_resource(*args, **kwargs)
elif cmd == 'get_resource':
return self.get_resource(*args, **kwargs)
elif cmd == 'get_config_metadata':
return self.get_config_metadata(*args, **kwargs)
elif cmd == 'disconnect':
pass
elif cmd == 'initialize':
pass
else:
log.error("Unhandled command: %s", cmd)
raise InstrumentStateException("Unhandled command: %s" % cmd)
def get_resource_capabilities(self, current_state=True, *args, **kwargs):
"""
Return driver commands and parameters.
@param current_state True to retrieve commands available in current
state, otherwise reutrn all commands.
@retval list of AgentCapability objects representing the drivers
capabilities.
@raises NotImplementedException if not implemented by subclass.
"""
res_params = self._param_dict.get_keys()
res_cmds = [DriverEvent.STOP_AUTOSAMPLE, DriverEvent.START_AUTOSAMPLE]
if current_state and self._is_sampling():
res_cmds = [DriverEvent.STOP_AUTOSAMPLE]
elif current_state and not self._is_sampling():
res_cmds = [DriverEvent.START_AUTOSAMPLE]
return [res_cmds, res_params]
def set_resource(self, *args, **kwargs):
"""
Set the driver parameter
"""
log.trace("start set_resource")
try:
params = args[0]
except IndexError:
raise InstrumentParameterException('Set command requires a parameter dict.')
log.trace("set_resource: iterate through params: %s", params)
for (key, val) in params.iteritems():
if key in [DriverParameter.BATCHED_PARTICLE_COUNT, DriverParameter.RECORDS_PER_SECOND]:
if not isinstance(val, int): raise InstrumentParameterException("%s must be an integer" % key)
if key in [DriverParameter.PUBLISHER_POLLING_INTERVAL]:
if not isinstance(val, (int, float)): raise InstrumentParameterException("%s must be an float" % key)
if val <= 0:
raise InstrumentParameterException("%s must be > 0" % key)
self._param_dict.set_value(key, val)
# Set the driver parameters
self._generate_particle_count = self._param_dict.get(DriverParameter.BATCHED_PARTICLE_COUNT)
self._particle_count_per_second = self._param_dict.get(DriverParameter.RECORDS_PER_SECOND)
self._polling_interval = self._param_dict.get(DriverParameter.PUBLISHER_POLLING_INTERVAL)
log.trace("Driver Parameters: %s, %s, %s", self._polling_interval, self._particle_count_per_second, self._generate_particle_count)
def get_resource(self, *args, **kwargs):
"""
Get driver parameter
"""
result = {}
try:
params = args[0]
except IndexError:
raise InstrumentParameterException('Set command requires a parameter list.')
# If all params requested, retrieve config.
if params == DriverParameter.ALL:
result = self._param_dict.get_config()
# If not all params, confirm a list or tuple of params to retrieve.
# Raise if not a list or tuple.
# Retrieve each key in the list, raise if any are invalid.
else:
if not isinstance(params, (list, tuple)):
raise InstrumentParameterException('Get argument not a list or tuple.')
result = {}
for key in params:
try:
val = self._param_dict.get(key)
result[key] = val
except KeyError:
raise InstrumentParameterException(('%s is not a valid parameter.' % key))
return result
def get_config_metadata(self):
"""
Return the configuration metadata object in JSON format
@retval The description of the parameters, commands, and driver info
in a JSON string
@see https://confluence.oceanobservatories.org/display/syseng/CIAD+MI+SV+Instrument+Driver-Agent+parameter+and+command+metadata+exchange
"""
log.debug("Getting metadata from driver...")
log.debug("Getting metadata dict from protocol...")
return_dict = {}
return_dict[ConfigMetadataKey.DRIVER] = self._driver_dict.generate_dict()
return_dict[ConfigMetadataKey.COMMANDS] = self._cmd_dict.generate_dict()
return_dict[ConfigMetadataKey.PARAMETERS] = self._param_dict.generate_dict()
return return_dict
def _verify_config(self):
"""
virtual method to verify the supplied driver configuration is value. Must
be overloaded in sub classes.
raises an ConfigurationException when a configuration error is detected.
"""
raise NotImplementedException('virtual methond needs to be specialized')
def _build_driver_dict(self):
"""
Populate the driver dictionary with options
"""
pass
def _build_command_dict(self):
"""
Populate the command dictionary with command.
"""
self._cmd_dict.add(DriverEvent.START_AUTOSAMPLE, display_name="start autosample")
self._cmd_dict.add(DriverEvent.STOP_AUTOSAMPLE, display_name="stop autosample")
def _build_param_dict(self):
"""
Setup three common driver parameters
"""
self._param_dict.add_parameter(
Parameter(
DriverParameter.RECORDS_PER_SECOND,
int,
value=60,
type=ParameterDictType.INT,
visibility=ParameterDictVisibility.IMMUTABLE,
display_name="Records Per Second",
description="Number of records to process per second")
)
self._param_dict.add_parameter(
Parameter(
DriverParameter.PUBLISHER_POLLING_INTERVAL,
float,
value=1,
type=ParameterDictType.FLOAT,
visibility=ParameterDictVisibility.IMMUTABLE,
display_name="Harvester Polling Interval",
description="Duration in minutes to wait before checking for new files.")
)
self._param_dict.add_parameter(
Parameter(
DriverParameter.BATCHED_PARTICLE_COUNT,
int,
value=1,
type=ParameterDictType.INT,
visibility=ParameterDictVisibility.IMMUTABLE,
display_name="Batched Particle Count",
description="Number of particles to batch before sending to the agent")
)
config = self._config.get(DataSourceConfigKey.DRIVER, {})
log.debug("set_resource on startup with: %s", config)
self.set_resource(config)
def _start_publisher_thread(self):
self._publisher_thread = gevent.spawn(self._publisher_loop)
self._publisher_shutdown = False
def _stop_publisher_thread(self):
log.debug("Signal shutdown")
self._publisher_shutdown = True
if self._publisher_thread:
self._publisher_thread.kill(block=False)
log.debug("shutdown complete")
def _publisher_loop(self):
"""
Main loop to listen for new files to parse. Parse them and move on.
"""
log.info("Starting main publishing loop")
try:
while(not self._publisher_shutdown):
self._poll()
gevent.sleep(self._polling_interval)
except Exception as e:
log.error("Exception in publisher thread: %s", e)
self._exception_callback(e)
log.debug("publisher thread detected shutdown request")
def _poll(self):
raise NotImplementedException('virtual methond needs to be specialized')
def _new_file_exception(self):
raise NotImplementedException('virtual methond needs to be specialized')
class DataSetDriver(object):
"""
Base class for data set drivers. Provides:
- an interface via callback to publish data
- an interface via callback to persist driver state
- an interface via callback to handle exceptions
- an start and stop sampling
- a client interface for execute resource
Subclasses need to include harvesters and parsers and
be specialized to handle the interaction between the two.
Configurations should contain keys from the DataSetDriverConfigKey class
and should look something like this example (more full documentation in the
"Dataset Agent Architecture" page on the OOI wiki):
{
'harvester':
{
'directory': '/tmp/dsatest',
'pattern': '*.txt',
'frequency': 1,
},
'parser': {}
'driver': {
'records_per_second'
'harvester_polling_interval'
'batched_particle_count'
}
}
"""
def __init__(self, config, memento, data_callback, state_callback,
exception_callback):
self._config = config
self._data_callback = data_callback
self._state_callback = state_callback
self._exception_callback = exception_callback
self._memento = memento
self._publisher_thread = None
self._verify_config()
self._param_dict = ProtocolParameterDict()
# Updated my set_resource, defaults defined in build_param_dict
self._polling_interval = None
self._generate_particle_count = None
self._particle_count_per_second = None
self._build_param_dict()
def shutdown(self):
self.stop_sampling()
def start_sampling(self):
"""
Start a new thread to monitor for data
"""
self._start_sampling()
self._start_publisher_thread()
def stop_sampling(self):
"""
Stop the sampling thread
"""
log.debug("Stopping driver now")
self._stop_sampling()
self._stop_publisher_thread()
def _start_sampling(self):
raise NotImplementedException(
'virtual methond needs to be specialized')
def _stop_sampling(self):
raise NotImplementedException(
'virtual methond needs to be specialized')
def cmd_dvr(self, cmd, *args, **kwargs):
log.warn("DRIVER: cmd_dvr %s", cmd)
if not cmd in [
'execute_resource', 'get_resource_capabilities',
'set_resource', 'get_resource'
]:
log.error("Unhandled command: %s", cmd)
raise InstrumentStateException("Unhandled command: %s" % cmd)
resource_cmd = args[0]
if cmd == 'execute_resource':
if resource_cmd == DriverEvent.START_AUTOSAMPLE:
return (ResourceAgentState.STREAMING, None)
elif resource_cmd == DriverEvent.STOP_AUTOSAMPLE:
self.stop_sampling()
return (ResourceAgentState.COMMAND, None)
else:
log.error("Unhandled resource command: %s", resource_cmd)
raise
elif cmd == 'get_resource_capabilities':
return self.get_resource_capabilities()
elif cmd == 'set_resource':
return self.set_resource(*args, **kwargs)
elif cmd == 'get_resource':
return self.get_resource(*args, **kwargs)
def get_resource_capabilities(self, current_state=True, *args, **kwargs):
"""
Return driver commands and parameters.
@param current_state True to retrieve commands available in current
state, otherwise reutrn all commands.
@retval list of AgentCapability objects representing the drivers
capabilities.
@raises NotImplementedException if not implemented by subclass.
"""
res_params = self._param_dict.get_keys()
return [[], res_params]
def set_resource(self, *args, **kwargs):
"""
Set the driver parameter
"""
log.trace("start set_resource")
try:
params = args[0]
except IndexError:
raise InstrumentParameterException(
'Set command requires a parameter dict.')
log.trace("set_resource: iterate through params: %s", params)
for (key, val) in params.iteritems():
if key in [
DriverParameter.BATCHED_PARTICLE_COUNT,
DriverParameter.RECORDS_PER_SECOND
]:
if not isinstance(val, int):
raise InstrumentParameterException(
"%s must be an integer" % key)
if key in [DriverParameter.PUBLISHER_POLLING_INTERVAL]:
if not isinstance(val, (int, float)):
raise InstrumentParameterException("%s must be an float" %
key)
if val <= 0:
raise InstrumentParameterException("%s must be > 0" % key)
self._param_dict.set_value(key, val)
# Set the driver parameters
self._generate_particle_count = self._param_dict.get(
DriverParameter.BATCHED_PARTICLE_COUNT)
self._particle_count_per_second = self._param_dict.get(
DriverParameter.RECORDS_PER_SECOND)
self._polling_interval = self._param_dict.get(
DriverParameter.PUBLISHER_POLLING_INTERVAL)
log.trace("Driver Parameters: %s, %s, %s", self._polling_interval,
self._particle_count_per_second,
self._generate_particle_count)
def get_resource(self, *args, **kwargs):
"""
Get driver parameter
"""
result = {}
try:
params = args[0]
except IndexError:
raise InstrumentParameterException(
'Set command requires a parameter list.')
# If all params requested, retrieve config.
if params == DriverParameter.ALL:
result = self._param_dict.get_config()
# If not all params, confirm a list or tuple of params to retrieve.
# Raise if not a list or tuple.
# Retrieve each key in the list, raise if any are invalid.
else:
if not isinstance(params, (list, tuple)):
raise InstrumentParameterException(
'Get argument not a list or tuple.')
result = {}
for key in params:
try:
val = self._param_dict.get(key)
result[key] = val
except KeyError:
raise InstrumentParameterException(
('%s is not a valid parameter.' % key))
return result
def _verify_config(self):
"""
virtual method to verify the supplied driver configuration is value. Must
be overloaded in sub classes.
raises an ConfigurationException when a configuration error is detected.
"""
raise NotImplementedException(
'virtual methond needs to be specialized')
def _build_param_dict(self):
"""
Setup three common driver parameters
"""
self._param_dict.add_parameter(
Parameter(DriverParameter.RECORDS_PER_SECOND,
int,
value=60,
type=ParameterDictType.INT,
display_name="Records Per Second",
description="Number of records to process per second"))
self._param_dict.add_parameter(
Parameter(
DriverParameter.PUBLISHER_POLLING_INTERVAL,
float,
value=1,
type=ParameterDictType.FLOAT,
display_name="Harvester Polling Interval",
description=
"Duration in minutes to wait before checking for new files."))
self._param_dict.add_parameter(
Parameter(
DriverParameter.BATCHED_PARTICLE_COUNT,
int,
value=1,
type=ParameterDictType.INT,
display_name="Batched Particle Count",
description=
"Number of particles to batch before sending to the agent"))
config = self._config.get(DataSourceConfigKey.DRIVER, {})
log.debug("set_resource on startup with: %s", config)
self.set_resource(config)
def _start_publisher_thread(self):
self._publisher_thread = gevent.spawn(self._publisher_loop)
self._publisher_shutdown = False
def _stop_publisher_thread(self):
log.debug("Signal shutdown")
self._publisher_shutdown = True
if self._publisher_thread:
self._publisher_thread.kill(block=False)
log.debug("shutdown complete")
def _publisher_loop(self):
"""
Main loop to listen for new files to parse. Parse them and move on.
"""
log.info("Starting main publishing loop")
try:
while (not self._publisher_shutdown):
self._poll()
gevent.sleep(self._polling_interval)
except Exception as e:
log.error("Exception in publisher thread: %s", e)
self._exception_callback(e)
log.debug("publisher thread detected shutdown request")
def _poll(self):
raise NotImplementedException(
'virtual methond needs to be specialized')
def _new_file_exception(self):
raise NotImplementedException(
'virtual methond needs to be specialized')
class InstrumentProtocol(object):
"""
Base instrument protocol class.
"""
def __init__(self, driver_event):
"""
Base constructor.
@param driver_event The callback for asynchronous driver events.
"""
# Event callback to send asynchronous events to the agent.
self._driver_event = driver_event
# The connection used to talk to the device.
self._connection = None
# The protocol state machine.
self._protocol_fsm = None
# The parameter dictionary.
self._param_dict = ProtocolParameterDict()
# The spot to stash a configuration before going into direct access
# mode
self._pre_direct_access_config = None
# Driver configuration passed from the user
self._startup_config = {}
# scheduler config is a bit redundant now, but if we ever want to
# re-initialize a scheduler we will need it.
self._scheduler = None
self._scheduler_callback = {}
self._scheduler_config = {}
########################################################################
# Helper methods
########################################################################
def got_data(self, port_agent_packet):
"""
Called by the instrument connection when data is available.
Defined in subclasses.
"""
log.error("base got_data. Who called me?")
pass
def _extract_sample(self, particle_class, regex, line, timestamp, publish=True):
"""
Extract sample from a response line if present and publish
parsed particle
@param particle_class The class to instantiate for this specific
data particle. Parameterizing this allows for simple, standard
behavior from this routine
@param regex The regular expression that matches a data sample
@param line string to match for sample.
@param timestamp port agent timestamp to include with the particle
@param publish boolean to publish samples (default True). If True,
two different events are published: one to notify raw data and
the other to notify parsed data.
@retval dict of dicts {'parsed': parsed_sample, 'raw': raw_sample} if
the line can be parsed for a sample. Otherwise, None.
@todo Figure out how the agent wants the results for a single poll
and return them that way from here
"""
sample = None
if regex.match(line):
particle = particle_class(line, port_timestamp=timestamp)
parsed_sample = particle.generate()
if publish and self._driver_event:
self._driver_event(DriverAsyncEvent.SAMPLE, parsed_sample)
sample = json.loads(parsed_sample)
return sample
return sample
def get_current_state(self):
"""
Return current state of the protocol FSM.
"""
return self._protocol_fsm.get_current_state()
def get_resource_capabilities(self, current_state=True):
"""
"""
res_cmds = self._protocol_fsm.get_events(current_state)
res_cmds = self._filter_capabilities(res_cmds)
res_params = self._param_dict.get_keys()
return [res_cmds, res_params]
def _filter_capabilities(self, events):
"""
"""
return events
########################################################################
# Scheduler interface.
########################################################################
def _add_scheduler(self, name, callback):
"""
Stage a scheduler in a driver. The job will actually be configured
and started by initialize_scheduler
@param name the name of the job
@param callback the handler when the job is triggered
@raise KeyError if we try to add a job twice
"""
if(self._scheduler_callback.get(name)):
raise KeyError("duplicate scheduler exists for '%s'" % name)
log.debug("Add scheduler callback: %s" % name)
self._scheduler_callback[name] = callback
self._add_scheduler_job(name)
def _add_scheduler_event(self, name, event):
"""
Create a scheduler, but instead of passing a callback we pass in
an event to raise. A callback function is dynamically created
to do this.
@param name the name of the job
@param event: event to raise when the scheduler is triggered
@raise KeyError if we try to add a job twice
"""
# Create a callback for the scheduler to raise an event
def event_callback(self, event):
log.info("driver job triggered, raise event: %s" % event)
self._protocol_fsm.on_event(event)
# Dynamically create the method and add it
method = partial(event_callback, self, event)
self._add_scheduler(name, method)
def _add_scheduler_job(self, name):
"""
Map the driver configuration to a scheduler configuration. If
the scheduler has been started then also add the job.
@param name the name of the job
@raise KeyError if job name does not exists in the callback config
@raise KeyError if job is already configured
"""
# Do nothing if the scheduler isn't initialized
if(not self._scheduler):
return
callback = self._scheduler_callback.get(name)
if(not callback):
raise KeyError("callback not defined in driver for '%s'" % name)
if(self._scheduler_config.get(name)):
raise KeyError("scheduler job already configured '%s'" % name)
scheduler_config = self._get_scheduler_config()
log.debug("Scheduler config: %s" % scheduler_config)
# No config? Nothing to do then.
if(scheduler_config == None):
return
job_config = scheduler_config.get(name)
if(job_config):
# Store the scheduler configuration
self._scheduler_config[name] = {
DriverSchedulerConfigKey.TRIGGER: job_config.get(DriverSchedulerConfigKey.TRIGGER),
DriverSchedulerConfigKey.CALLBACK: callback
}
config = {name: self._scheduler_config[name]}
log.debug("Scheduler job with config: %s" % config)
# start the job. Note, this lazily starts the scheduler too :)
self._scheduler.add_config(config)
def _get_scheduler_config(self):
"""
Get the configuration dictionary to use for initializing jobs
Returned dictionary structure:
{
'job_name': {
DriverSchedulerConfigKey.TRIGGER: {}
}
}
@return: scheduler configuration dictionary
"""
# Currently the startup config is in the child class.
# @TODO should the config code be promoted?
config = self._startup_config
return config.get(DriverConfigKey.SCHEDULER)
def initialize_scheduler(self):
"""
Activate all configured schedulers added using _add_scheduler.
Timers start when the job is activated.
"""
log.debug("Scheduler config: %s" % self._get_scheduler_config())
log.debug("Scheduler callbacks: %s" % self._scheduler_callback)
self._scheduler = DriverScheduler()
for name in self._scheduler_callback.keys():
log.debug("Add job for callback: %s" % name)
self._add_scheduler_job(name)
#############################################################
# Configuration logic
#############################################################
def apply_startup_params(self):
"""
Apply the startup values previously stored in the protocol to
the running config of the live instrument. The startup values are the
values that are (1) marked as startup parameters and are (2) the "best"
value to use at startup. Preference is given to the previously-set init
value, then the default value, then the currently used value.
This default method assumes a dict of parameter name and value for
the configuration.
This is the base stub for applying startup parameters at the protocol layer.
@raise InstrumentParameterException If the config cannot be applied
@raise NotImplementedException In the base class it isnt implemented
"""
raise NotImplementedException("Base class does not implement apply_startup_params()")
def set_init_params(self, config):
"""
Set the initialization parameters to the given values in the protocol
parameter dictionary.
@param config The parameter_name/value to set in the initialization
fields of the parameter dictionary
@raise InstrumentParameterException If the config cannot be set
"""
if not isinstance(config, dict):
raise InstrumentParameterException("Invalid init config format")
self._startup_config = config
param_config = config.get(DriverConfigKey.PARAMETERS)
if(param_config):
for name in param_config.keys():
log.debug("Setting init value for %s to %s", name, param_config[name])
self._param_dict.set_init_value(name, param_config[name])
def get_startup_config(self):
"""
Gets the startup configuration for the instrument. The parameters
returned are marked as startup, and the values are the best as chosen
from the initialization, default, and current parameters.
@retval The dict of parameter_name/values (override this method if it
is more involved for a specific instrument) that should be set at
a higher level.
@raise InstrumentProtocolException if a startup parameter doesn't
have a init or default value
"""
return_dict = {}
start_list = self._param_dict.get_keys()
log.trace("Startup list: %s", start_list)
assert isinstance(start_list, list)
for param in start_list:
result = self._param_dict.get_config_value(param)
if(result != None):
return_dict[param] = result
elif(self._param_dict.is_startup_param(param)):
raise InstrumentProtocolException("Required startup value not specified: %s" % param)
return return_dict
def get_direct_access_params(self):
"""
Get the list of direct access parameters, useful for restoring direct
access configurations up in the driver.
@retval a list of direct access parameter names
"""
return self._param_dict.get_direct_access_list()
def get_cached_config(self):
"""
Return the configuration object that shows the instrument's
configuration as cached in the parameter dictionary...usually in
sync with the instrument, but accessible when offline...
@retval The cached configuration in the instruments config format. By
default, it is a dictionary of parameter names and values.
"""
assert self._param_dict != None
return self._param_dict.get_config()
########################################################################
# Command build and response parse handlers.
########################################################################
def _add_response_handler(self, cmd, func, state=None):
"""
Insert a handler class responsible for handling the response to a
command sent to the instrument, optionally available only in a
specific state.
@param cmd The high level key of the command to respond to.
@param func The function that handles the response
@param state The state to pair with the command for which the function
should be used
"""
if state == None:
self._response_handlers[cmd] = func
else:
self._response_handlers[(state, cmd)] = func
def _add_build_handler(self, cmd, func):
"""
Add a command building function.
@param cmd The device command to build.
@param func The function that constructs the command.
"""
self._build_handlers[cmd] = func
########################################################################
# Helpers to build commands.
########################################################################
def _build_simple_command(self, cmd, *args):
"""
Builder for simple commands
@param cmd The command to build
@param args Unused arguments
@retval Returns string ready for sending to instrument
"""
return "%s%s" % (cmd, self._newline)
def _build_keypress_command(self, cmd, *args):
"""
Builder for simple, non-EOLN-terminated commands
@param cmd The command to build
@param args Unused arguments
@retval Returns string ready for sending to instrument
"""
return "%s" % (cmd)
def _build_multi_keypress_command(self, cmd, *args):
"""
Builder for simple, non-EOLN-terminated commands
@param cmd The command to build
@param args Unused arguments
@retval Returns string ready for sending to instrument
"""
return "%s%s%s%s%s%s" % (cmd, cmd, cmd, cmd, cmd, cmd)
########################################################################
# Static helpers to format set commands.
########################################################################
def _true_false_to_string(v):
"""
Write a boolean value to string formatted for "generic" set operations.
Subclasses should overload this as needed for instrument-specific
formatting.
@param v a boolean value.
@retval A yes/no string formatted as a Python boolean for set operations.
@throws InstrumentParameterException if value not a bool.
"""
if not isinstance(v,bool):
raise InstrumentParameterException('Value %s is not a bool.' % str(v))
return str(v)
@staticmethod
def _int_to_string(v):
"""
Write an int value to string formatted for "generic" set operations.
Subclasses should overload this as needed for instrument-specific
formatting.
@param v An int val.
@retval an int string formatted for generic set operations.
@throws InstrumentParameterException if value not an int.
"""
if not isinstance(v,int):
raise InstrumentParameterException('Value %s is not an int.' % str(v))
else:
return '%i' % v
@staticmethod
def _float_to_string(v):
"""
Write a float value to string formatted for "generic" set operations.
Subclasses should overload this as needed for instrument-specific
formatting.
@param v A float val.
@retval a float string formatted for "generic" set operations.
@throws InstrumentParameterException if value is not a float.
"""
if not isinstance(v,float):
raise InstrumentParameterException('Value %s is not a float.' % v)
else:
return '%e' % v
class DataSetDriver(object):
"""
Base class for data set drivers. Provides:
- an interface via callback to publish data
- an interface via callback to persist driver state
- an interface via callback to handle exceptions
- an start and stop sampling
- a client interface for execute resource
Subclasses need to include harvesters and parsers and
be specialized to handle the interaction between the two.
Configurations should contain keys from the DataSetDriverConfigKey class
and should look something like this example (more full documentation in the
"Dataset Agent Architecture" page on the OOI wiki):
{
'harvester':
{
'directory': '/tmp/dsatest',
'storage_directory': '/tmp/stored_dsatest',
'pattern': '*.txt',
'frequency': 1,
'file_mod_wait_time': 30,
},
'parser': {}
'driver': {
'records_per_second'
'harvester_polling_interval'
'batched_particle_count'
}
}
"""
def __init__(self, config, memento, data_callback, state_callback, event_callback, exception_callback):
self._config = copy.deepcopy(config)
self._data_callback = data_callback
self._state_callback = state_callback
self._event_callback = event_callback
self._exception_callback = exception_callback
self._memento = memento
self._publisher_thread = None
self._verify_config()
# Updated my set_resource, defaults defined in build_param_dict
self._polling_interval = None
self._generate_particle_count = None
self._particle_count_per_second = None
self._resource_id = None
self._param_dict = ProtocolParameterDict()
self._cmd_dict = ProtocolCommandDict()
self._driver_dict = DriverDict()
self._build_command_dict()
self._build_driver_dict()
self._build_param_dict()
def shutdown(self):
self.stop_sampling()
def start_sampling(self):
"""
Start a new thread to monitor for data
"""
self._start_sampling()
self._start_publisher_thread()
def stop_sampling(self):
"""
Stop the sampling thread
"""
log.debug("Stopping sampling and publisher now")
self._stop_sampling()
self._stop_publisher_thread()
def _start_sampling(self):
raise NotImplementedException('virtual method needs to be specialized')
def _stop_sampling(self):
raise NotImplementedException('virtual method needs to be specialized')
def _is_sampling(self):
"""
Currently the drivers only have two states, command and streaming and
all resource commands are common, either start or stop autosample.
Therefore we didn't implement an enitre state machine to manage states
and commands. If it does get more complex than this we should take the
time to implement a state machine to add some flexibility
"""
raise NotImplementedException('virtual method needs to be specialized')
def cmd_dvr(self, cmd, *args, **kwargs):
log.warn("DRIVER: cmd_dvr %s", cmd)
if cmd == 'execute_resource':
resource_cmd = args[0]
if resource_cmd == DriverEvent.START_AUTOSAMPLE:
return (ResourceAgentState.STREAMING, None)
elif resource_cmd == DriverEvent.STOP_AUTOSAMPLE:
self.stop_sampling()
return (ResourceAgentState.COMMAND, None)
else:
log.error("Unhandled resource command: %s", resource_cmd)
raise
elif cmd == 'get_resource_capabilities':
return self.get_resource_capabilities()
elif cmd == 'set_resource':
return self.set_resource(*args, **kwargs)
elif cmd == 'get_resource':
return self.get_resource(*args, **kwargs)
elif cmd == 'get_config_metadata':
return self.get_config_metadata(*args, **kwargs)
elif cmd == 'disconnect':
pass
elif cmd == 'initialize':
pass
else:
log.error("Unhandled command: %s", cmd)
raise InstrumentStateException("Unhandled command: %s" % cmd)
def get_resource_capabilities(self, current_state=True, *args, **kwargs):
"""
Return driver commands and parameters.
@param current_state True to retrieve commands available in current
state, otherwise reutrn all commands.
@retval list of AgentCapability objects representing the drivers
capabilities.
@raises NotImplementedException if not implemented by subclass.
"""
res_params = self._param_dict.get_keys()
res_cmds = [DriverEvent.STOP_AUTOSAMPLE, DriverEvent.START_AUTOSAMPLE]
if current_state and self._is_sampling():
res_cmds = [DriverEvent.STOP_AUTOSAMPLE]
elif current_state and not self._is_sampling():
res_cmds = [DriverEvent.START_AUTOSAMPLE]
return [res_cmds, res_params]
def set_resource(self, *args, **kwargs):
"""
Set the driver parameter
"""
log.trace("start set_resource")
try:
params = args[0]
except IndexError:
raise InstrumentParameterException('Set command requires a parameter dict.')
log.trace("set_resource: iterate through params: %s", params)
for (key, val) in params.iteritems():
if key in [DriverParameter.BATCHED_PARTICLE_COUNT, DriverParameter.RECORDS_PER_SECOND]:
if not isinstance(val, int): raise InstrumentParameterException("%s must be an integer" % key)
if key in [DriverParameter.PUBLISHER_POLLING_INTERVAL]:
if not isinstance(val, (int, float)): raise InstrumentParameterException("%s must be an float" % key)
if val <= 0:
raise InstrumentParameterException("%s must be > 0" % key)
self._param_dict.set_value(key, val)
# Set the driver parameters
self._generate_particle_count = self._param_dict.get(DriverParameter.BATCHED_PARTICLE_COUNT)
self._particle_count_per_second = self._param_dict.get(DriverParameter.RECORDS_PER_SECOND)
self._polling_interval = self._param_dict.get(DriverParameter.PUBLISHER_POLLING_INTERVAL)
log.trace("Driver Parameters: %s, %s, %s", self._polling_interval, self._particle_count_per_second,
self._generate_particle_count)
def get_resource(self, *args, **kwargs):
"""
Get driver parameter
"""
result = {}
try:
params = args[0]
except IndexError:
raise InstrumentParameterException('Set command requires a parameter list.')
# If all params requested, retrieve config.
if params == DriverParameter.ALL:
result = self._param_dict.get_config()
# If not all params, confirm a list or tuple of params to retrieve.
# Raise if not a list or tuple.
# Retrieve each key in the list, raise if any are invalid.
else:
if not isinstance(params, (list, tuple)):
raise InstrumentParameterException('Get argument not a list or tuple.')
result = {}
for key in params:
try:
val = self._param_dict.get(key)
result[key] = val
except KeyError:
raise InstrumentParameterException(('%s is not a valid parameter.' % key))
return result
def get_config_metadata(self):
"""
Return the configuration metadata object in JSON format
@retval The description of the parameters, commands, and driver info
in a JSON string
@see https://confluence.oceanobservatories.org/display/syseng/CIAD+MI+SV+Instrument+Driver-Agent+parameter+and+command+metadata+exchange
"""
log.debug("Getting metadata from driver...")
log.debug("Getting metadata dict from protocol...")
return_dict = {}
return_dict[ConfigMetadataKey.DRIVER] = self._driver_dict.generate_dict()
return_dict[ConfigMetadataKey.COMMANDS] = self._cmd_dict.generate_dict()
return_dict[ConfigMetadataKey.PARAMETERS] = self._param_dict.generate_dict()
return return_dict
def _verify_config(self):
"""
virtual method to verify the supplied driver configuration is value. Must
be overloaded in sub classes.
raises an ConfigurationException when a configuration error is detected.
"""
raise NotImplementedException('virtual methond needs to be specialized')
def _build_driver_dict(self):
"""
Populate the driver dictionary with options
"""
pass
def _build_command_dict(self):
"""
Populate the command dictionary with command.
"""
self._cmd_dict.add(DriverEvent.START_AUTOSAMPLE, display_name="start autosample")
self._cmd_dict.add(DriverEvent.STOP_AUTOSAMPLE, display_name="stop autosample")
def _build_param_dict(self):
"""
Setup three common driver parameters
"""
self._param_dict.add_parameter(
Parameter(
DriverParameter.RECORDS_PER_SECOND,
int,
value=60,
type=ParameterDictType.INT,
visibility=ParameterDictVisibility.IMMUTABLE,
display_name="Records Per Second",
description="Number of records to process per second")
)
self._param_dict.add_parameter(
Parameter(
DriverParameter.PUBLISHER_POLLING_INTERVAL,
float,
value=1,
type=ParameterDictType.FLOAT,
visibility=ParameterDictVisibility.IMMUTABLE,
display_name="Harvester Polling Interval",
description="Duration in minutes to wait before checking for new files.")
)
self._param_dict.add_parameter(
Parameter(
DriverParameter.BATCHED_PARTICLE_COUNT,
int,
value=1,
type=ParameterDictType.INT,
visibility=ParameterDictVisibility.IMMUTABLE,
display_name="Batched Particle Count",
description="Number of particles to batch before sending to the agent")
)
config = self._config.get(DataSourceConfigKey.DRIVER, {})
log.debug("set_resource on startup with: %s", config)
self.set_resource(config)
def _start_publisher_thread(self):
self._publisher_thread = gevent.spawn(self._publisher_loop)
self._publisher_shutdown = False
def _stop_publisher_thread(self):
log.debug("Signal shutdown")
self._publisher_shutdown = True
if self._publisher_thread:
self._publisher_thread.kill(block=False)
log.debug("shutdown complete")
def _publisher_loop(self):
"""
Main loop to listen for new files to parse. Parse them and move on.
"""
log.info("Starting main publishing loop")
try:
while(not self._publisher_shutdown):
self._poll()
gevent.sleep(self._polling_interval)
except Exception as e:
log.error("Exception in publisher thread (resource id: %s): %s", self._resource_id, traceback.format_exc(e))
self._exception_callback(e)
log.debug("publisher thread detected shutdown request")
def _poll(self):
raise NotImplementedException('virtual methond needs to be specialized')
def _new_file_exception(self):
raise NotImplementedException('virtual methond needs to be specialized')
def _sample_exception_callback(self, exception):
"""
Publish an event when a sample exception is detected
"""
self._event_callback(event_type="ResourceAgentErrorEvent", error_msg = "%s" % exception)
def _raise_new_file_event(self, name):
"""
Raise a ResourceAgentIOEvent when a new file is detected. Add file stats
to the payload of the event.
"""
s = os.stat(name)
checksum = ""
with open(name, 'rb') as filehandle:
checksum = hashlib.md5(filehandle.read()).hexdigest()
stats = {
'name': name,
'size': s.st_size,
'mod': s.st_mtime,
'md5_checksum': checksum
}
self._event_callback(event_type="ResourceAgentIOEvent", source_type="new file", stats=stats)
class TestUnitProtocolParameterDict(MiUnitTestCase):
@staticmethod
def pick_byte2(input):
""" Get the 2nd byte as an example of something tricky and
arbitrary"""
val = int(input) >> 8
val = val & 255
return val
"""
Test cases for instrument driver class. Functions in this class provide
instrument driver unit tests and provide a tutorial on use of
the driver interface.
"""
def setUp(self):
self.param_dict = ProtocolParameterDict()
self.param_dict.add("foo", r'.*foo=(\d+).*',
lambda match : int(match.group(1)),
lambda x : str(x),
direct_access=True,
startup_param=True,
default_value=10,
visibility=ParameterDictVisibility.READ_WRITE)
self.param_dict.add("bar", r'.*bar=(\d+).*',
lambda match : int(match.group(1)),
lambda x : str(x),
direct_access=False,
startup_param=True,
default_value=15,
visibility=ParameterDictVisibility.READ_WRITE)
self.param_dict.add("baz", r'.*baz=(\d+).*',
lambda match : int(match.group(1)),
lambda x : str(x),
direct_access=True,
default_value=20,
visibility=ParameterDictVisibility.DIRECT_ACCESS)
self.param_dict.add("bat", r'.*bat=(\d+).*',
lambda match : int(match.group(1)),
lambda x : str(x),
startup_param=False,
default_value=20,
visibility=ParameterDictVisibility.READ_ONLY)
self.param_dict.add("qux", r'.*qux=(\d+).*',
lambda match : int(match.group(1)),
lambda x : str(x),
startup_param=False,
visibility=ParameterDictVisibility.READ_ONLY)
def test_get_direct_access_list(self):
"""
Test to see we can get a list of direct access parameters
"""
result = self.param_dict.get_direct_access_list()
self.assertTrue(isinstance(result, list))
self.assertEquals(len(result), 2)
self.assert_("foo" in result)
self.assert_("baz" in result)
def test_get_startup_list(self):
"""
Test to see we can get a list of direct access parameters
"""
result = self.param_dict.get_startup_list()
self.assertTrue(isinstance(result, list))
self.assertEquals(len(result), 2)
self.assert_("foo" in result)
self.assert_("bar" in result)
def test_set_default(self):
"""
Test setting a default value
"""
result = self.param_dict.get_config()
self.assertEquals(result["foo"], None)
self.param_dict.set_default("foo")
self.assertEquals(self.param_dict.get("foo"), 10)
self.param_dict.update("foo=1000")
self.assertEquals(self.param_dict.get("foo"), 1000)
self.param_dict.set_default("foo")
self.assertEquals(self.param_dict.get("foo"), 10)
self.assertRaises(ValueError, self.param_dict.set_default, "qux")
def test_update_many(self):
"""
Test updating of multiple variables from the same input
"""
sample_input = """
foo=100
bar=200, baz=300
"""
self.assertNotEquals(self.param_dict.get("foo"), 100)
self.assertNotEquals(self.param_dict.get("bar"), 200)
self.assertNotEquals(self.param_dict.get("baz"), 300)
result = self.param_dict.update_many(sample_input)
log.debug("result: %s", result)
self.assertEquals(result["foo"], True)
self.assertEquals(result["bar"], True)
self.assertEquals(result["baz"], True)
self.assertEquals(self.param_dict.get("foo"), 100)
self.assertEquals(self.param_dict.get("bar"), 200)
self.assertEquals(self.param_dict.get("baz"), 300)
def test_visibility_list(self):
lst = self.param_dict.get_visibility_list(ParameterDictVisibility.READ_WRITE)
self.assertEquals(lst, ["foo", "bar"])
lst = self.param_dict.get_visibility_list(ParameterDictVisibility.DIRECT_ACCESS)
self.assertEquals(lst, ["baz"])
lst = self.param_dict.get_visibility_list(ParameterDictVisibility.READ_ONLY)
self.assertEquals(lst, ["bat", "qux"])
def test_function_values(self):
"""
Make sure we can add and update values with functions instead of patterns
"""
self.param_dict.add_paramdictval(
FunctionParamDictVal(
"fn_foo",
self.pick_byte2,
lambda x : str(x),
direct_access=True,
startup_param=True,
value=1,
visibility=ParameterDictVisibility.READ_WRITE)
)
self.param_dict.add_paramdictval(
FunctionParamDictVal(
"fn_bar",
lambda x : bool(x&2), # bit map example
lambda x : str(x),
direct_access=True,
startup_param=True,
value=False,
visibility=ParameterDictVisibility.READ_WRITE)
)
# check defaults just to be safe
val = self.param_dict.get("fn_foo")
self.assertEqual(val, 1)
val = self.param_dict.get("fn_bar")
self.assertEqual(val, False)
result = self.param_dict.update(1005) # just change first in list
val = self.param_dict.get("fn_foo")
self.assertEqual(val, 3)
val = self.param_dict.get("fn_bar")
self.assertEqual(val, False)
# fn_bar does not get updated here
result = self.param_dict.update_many(1205)
self.assertEqual(result['fn_foo'], True)
self.assertEqual(len(result), 1)
val = self.param_dict.get("fn_foo")
self.assertEqual(val, 4)
val = self.param_dict.get("fn_bar")
self.assertEqual(val, False)
# both are updated now
result = self.param_dict.update_many(6)
self.assertEqual(result['fn_foo'], True)
self.assertEqual(result['fn_bar'], True)
self.assertEqual(len(result), 2)
val = self.param_dict.get("fn_foo")
self.assertEqual(val, 0)
val = self.param_dict.get("fn_bar")
self.assertEqual(val, True)
def test_mixed_pdv_types(self):
""" Verify we can add different types of PDVs in one container """
self.param_dict.add_paramdictval(
FunctionParamDictVal(
"fn_foo",
self.pick_byte2,
lambda x : str(x),
direct_access=True,
startup_param=True,
value=1,
visibility=ParameterDictVisibility.READ_WRITE)
)
self.param_dict.add_paramdictval(
RegexParamDictVal("foo", r'.*foo=(\d+).*',
lambda match : int(match.group(1)),
lambda x : str(x),
direct_access=True,
startup_param=True,
value=10,
visibility=ParameterDictVisibility.READ_WRITE)
)
self.param_dict.add("bar", r'.*bar=(\d+).*',
lambda match : int(match.group(1)),
lambda x : str(x),
direct_access=False,
startup_param=True,
value=15,
visibility=ParameterDictVisibility.READ_WRITE)
self.assertEqual(self.param_dict.get("fn_foo"), 1)
self.assertEqual(self.param_dict.get("foo"), 10)
self.assertEqual(self.param_dict.get("bar"), 15)
def test_base_update(self):
pdv = ParameterDictVal("foo",
lambda x : str(x),
value=12)
self.assertEqual(pdv.value, 12)
result = pdv.update(1)
self.assertEqual(result, True)
self.assertEqual(pdv.value, 1)
# Its a base class...monkey see, monkey do
result = pdv.update("foo=1")
self.assertEqual(result, True)
self.assertEqual(pdv.value, "foo=1")
def test_regex_val(self):
pdv = RegexParamDictVal("foo",
r'.*foo=(\d+).*',
lambda match : int(match.group(1)),
lambda x : str(x),
value=12)
self.assertEqual(pdv.value, 12)
result = pdv.update(1)
self.assertEqual(result, False)
self.assertEqual(pdv.value, 12)
result = pdv.update("foo=1")
self.assertEqual(result, True)
self.assertEqual(pdv.value, 1)
def test_function_val(self):
pdv = FunctionParamDictVal("foo",
self.pick_byte2,
lambda x : str(x),
value=12)
self.assertEqual(pdv.value, 12)
self.assertRaises(TypeError, pdv.update(1))
result = pdv.update("1205")
self.assertEqual(pdv.value, 4)
self.assertEqual(result, True)
def test_set_init_value(self):
result = self.param_dict.get("foo")
self.assertEqual(result, None)
self.param_dict.set_init_value("foo", 42)
result = self.param_dict.get_init_value("foo")
self.assertEqual(result, 42)
class TestUnitProtocolParameterDict(MiUnitTestCase):
"""
Test cases for instrument driver class. Functions in this class provide
instrument driver unit tests and provide a tutorial on use of
the driver interface.
"""
def setUp(self):
self.param_dict = ProtocolParameterDict()
self.param_dict.add("foo", r'.*foo=(\d*).*',
lambda match : int(match.group(1)),
lambda x : str(x),
direct_access=True,
startup_param=True,
default_value=10,
visibility=ParameterDictVisibility.READ_WRITE)
self.param_dict.add("bar", r'.*bar=(\d*).*',
lambda match : int(match.group(1)),
lambda x : str(x),
direct_access=False,
startup_param=True,
default_value=15,
visibility=ParameterDictVisibility.READ_WRITE)
self.param_dict.add("baz", r'.*baz=(\d*).*',
lambda match : int(match.group(1)),
lambda x : str(x),
direct_access=True,
default_value=20,
visibility=ParameterDictVisibility.DIRECT_ACCESS)
self.param_dict.add("bat", r'.*bat=(\d*).*',
lambda match : int(match.group(1)),
lambda x : str(x),
startup_param=False,
default_value=20,
visibility=ParameterDictVisibility.READ_ONLY)
self.param_dict.add("qux", r'.*qux=(\d*).*',
lambda match : int(match.group(1)),
lambda x : str(x),
startup_param=False,
visibility=ParameterDictVisibility.READ_ONLY)
def test_get_direct_access_list(self):
"""
Test to see we can get a list of direct access parameters
"""
result = self.param_dict.get_direct_access_list()
self.assertTrue(isinstance(result, list))
self.assertEquals(len(result), 2)
self.assert_("foo" in result)
self.assert_("baz" in result)
def test_get_startup_list(self):
"""
Test to see we can get a list of direct access parameters
"""
result = self.param_dict.get_startup_list()
self.assertTrue(isinstance(result, list))
self.assertEquals(len(result), 2)
self.assert_("foo" in result)
self.assert_("bar" in result)
def test_set_default(self):
"""
Test setting a default value
"""
result = self.param_dict.get_config()
self.assertEquals(result["foo"], None)
self.param_dict.set_default("foo")
self.assertEquals(self.param_dict.get("foo"), 10)
self.param_dict.update("foo=1000")
self.assertEquals(self.param_dict.get("foo"), 1000)
self.param_dict.set_default("foo")
self.assertEquals(self.param_dict.get("foo"), 10)
self.assertRaises(ValueError, self.param_dict.set_default, "qux")
def test_update_many(self):
"""
Test updating of multiple variables from the same input
"""
sample_input = """
foo=100
bar=200, baz=300
"""
self.assertNotEquals(self.param_dict.get("foo"), 100)
self.assertNotEquals(self.param_dict.get("bar"), 200)
self.assertNotEquals(self.param_dict.get("baz"), 300)
result = self.param_dict.update_many(sample_input)
log.debug("result: %s", result)
self.assertEquals(result["foo"], True)
self.assertEquals(result["bar"], True)
self.assertEquals(result["baz"], True)
self.assertEquals(self.param_dict.get("foo"), 100)
self.assertEquals(self.param_dict.get("bar"), 200)
self.assertEquals(self.param_dict.get("baz"), 300)
def test_visibility_list(self):
lst = self.param_dict.get_visibility_list(ParameterDictVisibility.READ_WRITE)
self.assertEquals(lst, ["foo", "bar"])
lst = self.param_dict.get_visibility_list(ParameterDictVisibility.DIRECT_ACCESS)
self.assertEquals(lst, ["baz"])
lst = self.param_dict.get_visibility_list(ParameterDictVisibility.READ_ONLY)
self.assertEquals(lst, ["bat", "qux"])
class DataSetDriver(object):
"""
Base class for data set drivers. Provides:
- an interface via callback to publish data
- an interface via callback to persist driver state
- an interface via callback to handle exceptions
- an start and stop sampling
- a client interface for execute resource
Subclasses need to include harvesters and parsers and
be specialized to handle the interaction between the two.
Configurations should contain keys from the DataSetDriverConfigKey class
and should look something like this example (more full documentation in the
"Dataset Agent Architecture" page on the OOI wiki):
{
'harvester':
{
'directory': '/tmp/dsatest',
'pattern': '*.txt',
'frequency': 1,
},
'parser': {}
'driver': {
'records_per_second'
'harvester_polling_interval'
'batched_particle_count'
}
}
"""
def __init__(self, config, memento, data_callback, state_callback, exception_callback):
self._config = config
self._data_callback = data_callback
self._state_callback = state_callback
self._exception_callback = exception_callback
self._memento = memento
self._verify_config()
self._param_dict = ProtocolParameterDict()
# Updated my set_resource, defaults defined in build_param_dict
self._polling_interval = None
self._generate_particle_count = None
self._particle_count_per_second = None
self._build_param_dict()
def start_sampling(self):
"""
Start a new thread to monitor for data
"""
self._start_sampling()
self._start_publisher_thread()
def stop_sampling(self):
"""
Stop the sampling thread
"""
self._stop_publisher_thread()
def _start_sampling(self):
raise NotImplementedException('virtual methond needs to be specialized')
def _stop_sampling(self):
raise NotImplementedException('virtual methond needs to be specialized')
def cmd_dvr(self, cmd, *args, **kwargs):
log.warn("DRIVER: cmd_dvr %s", cmd)
if not cmd in ['execute_resource', 'get_resource_capabilities', 'set_resource', 'get_resource']:
log.error("Unhandled command: %s", cmd)
raise InstrumentStateException("Unhandled command: %s" % cmd)
resource_cmd = args[0]
if cmd == 'execute_resource':
if resource_cmd == DriverEvent.START_AUTOSAMPLE:
try:
log.debug("start autosample")
self.start_sampling()
except:
log.error("Failed to start sampling", exc_info=True)
raise
return (ResourceAgentState.STREAMING, None)
elif resource_cmd == DriverEvent.STOP_AUTOSAMPLE:
log.debug("stop autosample")
self.stop_sampling()
return (ResourceAgentState.COMMAND, None)
else:
log.error("Unhandled resource command: %s", resource_cmd)
raise
elif cmd == 'get_resource_capabilities':
return self.get_resource_capabilities()
elif cmd == 'set_resource':
return self.set_resource(*args, **kwargs)
elif cmd == 'get_resource':
return self.get_resource(*args, **kwargs)
def get_resource_capabilities(self, current_state=True, *args, **kwargs):
"""
Return driver commands and parameters.
@param current_state True to retrieve commands available in current
state, otherwise reutrn all commands.
@retval list of AgentCapability objects representing the drivers
capabilities.
@raises NotImplementedException if not implemented by subclass.
"""
res_params = self._param_dict.get_keys()
return [[], res_params]
def set_resource(self, *args, **kwargs):
"""
Set the driver parameter
"""
log.trace("start set_resource")
try:
params = args[0]
except IndexError:
raise InstrumentParameterException('Set command requires a parameter dict.')
log.trace("set_resource: iterate through params: %s", params)
for (key, val) in params.iteritems():
if key in [DriverParameter.BATCHED_PARTICLE_COUNT, DriverParameter.RECORDS_PER_SECOND]:
if not isinstance(val, int): raise InstrumentParameterException("%s must be an integer" % key)
if key in [DriverParameter.HARVESTER_POLLING_INTERVAL]:
if not isinstance(val, (int, float)): raise InstrumentParameterException("%s must be an float" % key)
if val <= 0:
raise InstrumentParameterException("%s must be > 0" % key)
self._param_dict.set_value(key, val)
# Set the driver parameters
self._generate_particle_count = self._param_dict.get(DriverParameter.BATCHED_PARTICLE_COUNT)
self._particle_count_per_second = self._param_dict.get(DriverParameter.RECORDS_PER_SECOND)
self._polling_interval = self._param_dict.get(DriverParameter.HARVESTER_POLLING_INTERVAL)
log.trace("Driver Parameters: %s, %s, %s", self._polling_interval, self._particle_count_per_second, self._generate_particle_count)
def get_resource(self, *args, **kwargs):
"""
Get driver parameter
"""
result = {}
try:
params = args[0]
except IndexError:
raise InstrumentParameterException('Set command requires a parameter list.')
# If all params requested, retrieve config.
if params == DriverParameter.ALL:
result = self._param_dict.get_config()
# If not all params, confirm a list or tuple of params to retrieve.
# Raise if not a list or tuple.
# Retrieve each key in the list, raise if any are invalid.
else:
if not isinstance(params, (list, tuple)):
raise InstrumentParameterException('Get argument not a list or tuple.')
result = {}
for key in params:
try:
val = self._param_dict.get(key)
result[key] = val
except KeyError:
raise InstrumentParameterException(('%s is not a valid parameter.' % key))
return result
def _verify_config(self):
"""
virtual method to verify the supplied driver configuration is value. Must
be overloaded in sub classes.
raises an ConfigurationException when a configuration error is detected.
"""
raise NotImplementedException('virtual methond needs to be specialized')
def _build_param_dict(self):
"""
Setup three common driver parameters
"""
self._param_dict.add_parameter(
Parameter(
DriverParameter.RECORDS_PER_SECOND,
int,
value=60,
type=ParameterDictType.INT,
display_name="Records Per Second",
description="Number of records to process per second")
)
self._param_dict.add_parameter(
Parameter(
DriverParameter.HARVESTER_POLLING_INTERVAL,
float,
value=1,
type=ParameterDictType.FLOAT,
display_name="Harvester Polling Interval",
description="Duration in minutes to wait before checking for new files.")
)
self._param_dict.add_parameter(
Parameter(
DriverParameter.BATCHED_PARTICLE_COUNT,
int,
value=1,
type=ParameterDictType.INT,
display_name="Batched Particle Count",
description="Number of particles to batch before sending to the agent")
)
config = self._config.get(DataSourceConfigKey.DRIVER, {})
log.debug("set_resource on startup with: %s", config)
self.set_resource(config)
def _start_publisher_thread(self):
self._publisher_thread = gevent.spawn(self._poll)
def _stop_publisher_thread(self):
self._publisher_thread.kill()
def _poll(self):
raise NotImplementedException('virtual methond needs to be specialized')
def _new_file_exception(self):
raise NotImplementedException('virtual methond needs to be specialized')
