예제 #1
0
class TestUnitProtocolParameterDict(TestUnitStringsDict):
    """
    Test cases for instrument driver class. Functions in this class provide
    instrument driver unit tests and provide a tutorial on use of
    the driver interface.
    """

    __test__ = True

    @staticmethod
    def pick_byte2(input_val):
        """ Get the 2nd byte as an example of something tricky and
        arbitrary"""
        val = int(input_val) >> 8
        val &= 255
        return val

    def setUp(self):
        self.param_dict = ProtocolParameterDict()

        self.param_dict.add("foo",
                            r'.*foo=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: str(x),
                            direct_access=True,
                            startup_param=True,
                            default_value=10,
                            visibility=ParameterDictVisibility.READ_WRITE)
        self.param_dict.add("bar",
                            r'.*bar=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: str(x),
                            direct_access=False,
                            startup_param=True,
                            default_value=15,
                            visibility=ParameterDictVisibility.READ_WRITE)
        self.param_dict.add(
            "baz",
            r'.*baz=(\d+).*',
            lambda match: int(match.group(1)),
            lambda x: str(x),
            direct_access=True,
            default_value=20,
            visibility=ParameterDictVisibility.DIRECT_ACCESS,
            get_timeout=30,
            set_timeout=40,
            display_name="Baz",
            description="The baz parameter",
            type=ParameterDictType.INT,
            units="nano-bazers",
            value_description="Should be an integer between 2 and 2000")
        self.param_dict.add(
            "bat",
            r'.*bat=(\d+).*',
            lambda match: int(match.group(1)),
            lambda x: str(x),
            startup_param=False,
            default_value=20,
            visibility=ParameterDictVisibility.READ_ONLY,
            get_timeout=10,
            set_timeout=20,
            display_name="Bat",
            description="The bat parameter",
            type=ParameterDictType.INT,
            units="nano-batbit",
            value_description="Should be an integer between 1 and 1000")
        self.param_dict.add("qux",
                            r'.*qux=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: str(x),
                            startup_param=False,
                            visibility=ParameterDictVisibility.READ_ONLY)
        self.param_dict.add("pho",
                            r'.*qux=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: str(x),
                            startup_param=False,
                            visibility=ParameterDictVisibility.IMMUTABLE)
        self.param_dict.add("dil",
                            r'.*qux=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: str(x),
                            startup_param=False,
                            visibility=ParameterDictVisibility.IMMUTABLE)
        self.param_dict.add(
            "qut",
            r'.*qut=(\d+).*',
            lambda match: int(match.group(1)),
            lambda x: str(x),
            direct_access=True,
            default_value=[10, 100],
            visibility=ParameterDictVisibility.DIRECT_ACCESS,
            expiration=1,
            get_timeout=10,
            set_timeout=20,
            display_name="Qut",
            description="The qut list parameter",
            type=ParameterDictType.LIST,
            units="nano-qutters",
            value_description=
            "Should be a 2-10 element list of integers between 2 and 2000")

        self.target_schema = {
            "bar": {
                "direct_access": False,
                "get_timeout": 10,
                "set_timeout": 10,
                "startup": True,
                "value": {
                    "default": 15
                },
                "visibility": "READ_WRITE",
                "range": None,
            },
            "bat": {
                "description": "The bat parameter",
                "direct_access": False,
                "display_name": "Bat",
                "get_timeout": 10,
                "set_timeout": 20,
                "startup": False,
                "value": {
                    "default": 20,
                    "description": "Should be an integer between 1 and 1000",
                    "type": "int",
                    "units": "nano-batbit"
                },
                "visibility": "READ_ONLY",
                "range": None,
            },
            "baz": {
                "description": "The baz parameter",
                "direct_access": True,
                "display_name": "Baz",
                "get_timeout": 30,
                "set_timeout": 40,
                "startup": False,
                "value": {
                    "default": 20,
                    "description": "Should be an integer between 2 and 2000",
                    "type": "int",
                    "units": "nano-bazers"
                },
                "visibility": "DIRECT_ACCESS",
                "range": None,
            },
            "dil": {
                "direct_access": False,
                "get_timeout": 10,
                "set_timeout": 10,
                "startup": False,
                "value": {},
                "visibility": "IMMUTABLE",
                "range": None,
            },
            "foo": {
                "direct_access": True,
                "get_timeout": 10,
                "set_timeout": 10,
                "startup": True,
                "value": {
                    "default": 10
                },
                "visibility": "READ_WRITE",
                "range": None,
            },
            "pho": {
                "direct_access": False,
                "get_timeout": 10,
                "set_timeout": 10,
                "startup": False,
                "value": {},
                "visibility": "IMMUTABLE",
                "range": None,
            },
            "qut": {
                "description": "The qut list parameter",
                "direct_access": True,
                "display_name": "Qut",
                "get_timeout": 10,
                "set_timeout": 20,
                "startup": False,
                "value": {
                    "default": [10, 100],
                    "description":
                    "Should be a 2-10 element list of integers between 2 and 2000",
                    "type": "list",
                    "units": "nano-qutters"
                },
                "visibility": "DIRECT_ACCESS",
                "range": None,
            },
            "qux": {
                "direct_access": False,
                "get_timeout": 10,
                "set_timeout": 10,
                "startup": False,
                "value": {},
                "visibility": "READ_ONLY",
                "range": None,
            }
        }

        self.test_yaml = '''
        parameters: {
            qut: {
            description: "QutFileDesc",
            units: "QutFileUnits",
            value_description: "QutFileValueDesc",
            type: "QutFileType",
            display_name: "QutDisplay"
            },
            extra_param: {
            description: "ExtraFileDesc",
            units: "ExtraFileUnits",
            value_description: "ExtraFileValueDesc",
            type: "ExtraFileType"
            }
          }

        commands: {
          dummy: stuff
          }
        '''

    def test_get_direct_access_list(self):
        """
        Test to see we can get a list of direct access parameters
        """
        result = self.param_dict.get_direct_access_list()
        self.assertTrue(isinstance(result, list))
        self.assertEquals(len(result), 3)
        self.assert_("foo" in result)
        self.assert_("baz" in result)
        self.assert_("qut" in result)

    def test_get_startup_list(self):
        """
        Test to see we can get a list of direct access parameters
        """
        result = self.param_dict.get_startup_list()
        self.assertTrue(isinstance(result, list))
        self.assertEquals(len(result), 2)
        self.assert_("foo" in result)
        self.assert_("bar" in result)

    def test_set_default(self):
        """
        Test setting a default value
        """
        result = self.param_dict.get_config()
        self.assertEquals(result["foo"], None)
        self.param_dict.set_default("foo")
        self.assertEquals(self.param_dict.get("foo"), 10)
        self.param_dict.update("foo=1000")
        self.assertEquals(self.param_dict.get("foo"), 1000)
        self.param_dict.set_default("foo")
        self.assertEquals(self.param_dict.get("foo"), 10)

        self.assertRaises(ValueError, self.param_dict.set_default, "qux")

    def test_update_many(self):
        """
        Test updating of multiple variables from the same input
        """
        sample_input = """
foo=100
bar=200, baz=300
"""
        self.assertNotEquals(self.param_dict.get("foo"), 100)
        self.assertNotEquals(self.param_dict.get("bar"), 200)
        self.assertNotEquals(self.param_dict.get("baz"), 300)
        result = self.param_dict.update_many(sample_input)
        log.debug("result: %s", result)
        self.assertEquals(result["foo"], True)
        self.assertEquals(result["bar"], True)
        self.assertEquals(result["baz"], True)
        self.assertEquals(self.param_dict.get("foo"), 100)
        self.assertEquals(self.param_dict.get("bar"), 200)
        self.assertEquals(self.param_dict.get("baz"), 300)

    def test_update_specific_values(self):
        """
        test to verify we can limit update to a specific
        set of parameters
        """
        sample_input = "foo=100, bar=200"

        # First verify we can set both
        self.assertNotEquals(self.param_dict.get("foo"), 100)
        self.assertNotEquals(self.param_dict.get("bar"), 200)
        self.assertTrue(self.param_dict.update(sample_input))
        self.assertEquals(self.param_dict.get("foo"), 100)
        self.assertEquals(self.param_dict.get("bar"), 200)

        # Now let's only have it update 1 parameter with a name
        sample_input = "foo=200, bar=300"
        self.assertTrue(
            self.param_dict.update(sample_input, target_params="foo"))
        self.assertEquals(self.param_dict.get("foo"), 200)
        self.assertEquals(self.param_dict.get("bar"), 200)

        # Now let's only have it update 1 parameter using a list
        sample_input = "foo=300, bar=400"
        self.assertTrue(
            self.param_dict.update(sample_input, target_params=["foo"]))
        self.assertEquals(self.param_dict.get("foo"), 300)
        self.assertEquals(self.param_dict.get("bar"), 200)

        # Test our exceptions
        with self.assertRaises(KeyError):
            self.param_dict.update(sample_input, "key_does_not_exist")

        with self.assertRaises(InstrumentParameterException):
            self.param_dict.update(sample_input, {'bad': "key_does_not_exist"})

    def test_visibility_list(self):
        lst = self.param_dict.get_visibility_list(
            ParameterDictVisibility.READ_WRITE)
        lst.sort()
        self.assertEquals(lst, ["bar", "foo"])
        lst = self.param_dict.get_visibility_list(
            ParameterDictVisibility.DIRECT_ACCESS)
        lst.sort()
        self.assertEquals(lst, ["baz", "qut"])
        lst = self.param_dict.get_visibility_list(
            ParameterDictVisibility.READ_ONLY)
        lst.sort()
        self.assertEquals(lst, ["bat", "qux"])
        lst = self.param_dict.get_visibility_list(
            ParameterDictVisibility.IMMUTABLE)
        lst.sort()
        self.assertEquals(lst, ["dil", "pho"])

    def test_function_values(self):
        """
        Make sure we can add and update values with functions instead of patterns
        """

        self.param_dict.add_parameter(
            FunctionParameter("fn_foo",
                              self.pick_byte2,
                              lambda x: str(x),
                              direct_access=True,
                              startup_param=True,
                              value=1,
                              visibility=ParameterDictVisibility.READ_WRITE))
        self.param_dict.add_parameter(
            FunctionParameter(
                "fn_bar",
                lambda x: bool(x & 2),  # bit map example
                lambda x: str(x),
                direct_access=True,
                startup_param=True,
                value=False,
                visibility=ParameterDictVisibility.READ_WRITE))

        # check defaults just to be safe
        val = self.param_dict.get("fn_foo")
        self.assertEqual(val, 1)
        val = self.param_dict.get("fn_bar")
        self.assertEqual(val, False)

        self.param_dict.update(1005)  # just change first in list
        val = self.param_dict.get("fn_foo")
        self.assertEqual(val, 3)
        val = self.param_dict.get("fn_bar")
        self.assertEqual(val, False)

        # fn_bar does not get updated here
        result = self.param_dict.update_many(1205)
        self.assertEqual(result['fn_foo'], True)
        self.assertEqual(len(result), 1)
        val = self.param_dict.get("fn_foo")
        self.assertEqual(val, 4)
        val = self.param_dict.get("fn_bar")
        self.assertEqual(val, False)

        # both are updated now
        result = self.param_dict.update_many(6)
        self.assertEqual(result['fn_foo'], True)
        self.assertEqual(result['fn_bar'], True)
        self.assertEqual(len(result), 2)

        val = self.param_dict.get("fn_foo")
        self.assertEqual(val, 0)
        val = self.param_dict.get("fn_bar")
        self.assertEqual(val, True)

    def test_mixed_pdv_types(self):
        """ Verify we can add different types of PDVs in one container """
        self.param_dict.add_parameter(
            FunctionParameter("fn_foo",
                              self.pick_byte2,
                              lambda x: str(x),
                              direct_access=True,
                              startup_param=True,
                              value=1,
                              visibility=ParameterDictVisibility.READ_WRITE))
        self.param_dict.add_parameter(
            RegexParameter("foo",
                           r'.*foo=(\d+).*',
                           lambda match: int(match.group(1)),
                           lambda x: str(x),
                           direct_access=True,
                           startup_param=True,
                           value=10,
                           visibility=ParameterDictVisibility.READ_WRITE))
        self.param_dict.add("bar",
                            r'.*bar=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: str(x),
                            direct_access=False,
                            startup_param=True,
                            value=15,
                            visibility=ParameterDictVisibility.READ_WRITE)

        self.assertEqual(self.param_dict.get("fn_foo"), 1)
        self.assertEqual(self.param_dict.get("foo"), 10)
        self.assertEqual(self.param_dict.get("bar"), 15)

    def test_base_update(self):
        pdv = Parameter("foo", lambda x: str(x), value=12)
        self.assertEqual(pdv.get_value(), 12)
        result = pdv.update(1)
        self.assertEqual(result, True)
        self.assertEqual(pdv.get_value(), 1)

        # Its a base class...monkey see, monkey do
        result = pdv.update("foo=1")
        self.assertEqual(result, True)
        self.assertEqual(pdv.get_value(), "foo=1")

    def test_regex_val(self):
        pdv = RegexParameter("foo",
                             r'.*foo=(\d+).*',
                             lambda match: int(match.group(1)),
                             lambda x: str(x),
                             value=12)
        self.assertEqual(pdv.get_value(), 12)
        result = pdv.update(1)
        self.assertEqual(result, False)
        self.assertEqual(pdv.get_value(), 12)
        result = pdv.update("foo=1")
        self.assertEqual(result, True)
        self.assertEqual(pdv.get_value(), 1)

    def test_function_val(self):
        pdv = FunctionParameter("foo",
                                self.pick_byte2,
                                lambda x: str(x),
                                value=12)
        self.assertEqual(pdv.get_value(), 12)
        self.assertRaises(TypeError, pdv.update(1))
        result = pdv.update("1205")
        self.assertEqual(pdv.get_value(), 4)
        self.assertEqual(result, True)

    def test_set_init_value(self):
        result = self.param_dict.get("foo")
        self.assertEqual(result, None)
        self.param_dict.set_init_value("foo", 42)
        result = self.param_dict.get_init_value("foo")
        self.assertEqual(result, 42)

    def test_schema_generation(self):
        self.maxDiff = None
        result = self.param_dict.generate_dict()
        json_result = json.dumps(result, indent=4, sort_keys=True)
        log.debug("Expected: %s", self.target_schema)
        log.debug("Result: %s", json_result)
        self.assertEqual(result, self.target_schema)

    def test_empty_schema(self):
        self.param_dict = ProtocolParameterDict()
        result = self.param_dict.generate_dict()
        self.assertEqual(result, {})

    def test_bad_descriptions(self):
        self.param_dict._param_dict["foo"].description = None
        self.param_dict._param_dict["foo"].value = None
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.get_init_value, "foo")
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.get_default_value, "foo")
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.set_default, "foo")
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.get_init_value, "foo")
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.get_menu_path_read, "foo")
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.get_submenu_read, "foo")
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.get_menu_path_write, "foo")
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.get_submenu_write, "foo")
        self.assertRaises(InstrumentParameterException, self.param_dict.format,
                          "foo", 1)
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.get_direct_access_list)
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.is_startup_param, "foo")

    def test_set(self):
        """
        Test a simple set of the parameter. Make sure the right values get
        called and the correct exceptions are raised.
        """
        new_param = FunctionParameter(
            "foo",
            self.pick_byte2,
            lambda x: str(x),
            direct_access=True,
            startup_param=True,
            value=1000,
            visibility=ParameterDictVisibility.READ_WRITE)
        self.assertEquals(new_param.get_value(), 1000)
        self.assertEquals(self.param_dict.get("foo"), None)
        # overwrites existing param
        self.param_dict.add_parameter(new_param)
        self.assertEquals(self.param_dict.get("foo"), 1000)
        self.param_dict.set_value("foo", 2000)
        self.assertEquals(self.param_dict.get("foo"), 2000)

    def test_invalid_type(self):
        self.assertRaises(
            InstrumentParameterException,
            FunctionParameter,
            "fn_bar",
            lambda x: bool(x & 2),  # bit map example
            lambda x: str(x),
            direct_access=True,
            startup_param=True,
            value=False,
            type="bad_type",
            visibility=ParameterDictVisibility.READ_WRITE)

    def test_get(self):
        """
        test getting values with expiration
        """
        # from mi.core.exceptions import InstrumentParameterExpirationException
        pd = ProtocolParameterDict()

        # No expiration, should work just fine
        pd.add('noexp', r'', None, None, expiration=None)
        pd.add('zeroexp', r'', None, None, expiration=0)
        pd.add('lateexp', r'', None, None, expiration=2)

        ###
        # Set and get with no expire
        ###
        pd.set_value('noexp', 1)
        self.assertEqual(pd.get('noexp'), 1)

        ###
        # Set and get with a 0 expire
        ###
        basetime = pd.get_current_timestamp()
        pd.set_value('zeroexp', 2)

        # We should fail because we are calculating exp against current time
        with self.assertRaises(InstrumentParameterExpirationException):
            pd.get('zeroexp')

        # Should succeed because exp is calculated using basetime
        self.assertEqual(pd.get('zeroexp', basetime), 2)

        ###
        # Set and get with a delayed expire
        ###
        basetime = pd.get_current_timestamp()
        futuretime = pd.get_current_timestamp(3)
        self.assertGreater(futuretime - basetime, 3)

        pd.set_value('lateexp', 2)

        # Success because data is not expired
        self.assertEqual(pd.get('lateexp', basetime), 2)

        # Fail because data is expired (simulated three seconds from now)
        with self.assertRaises(InstrumentParameterExpirationException):
            pd.get('lateexp', futuretime)

    def test_regex_flags(self):
        pdv = RegexParameter("foo",
                             r'.+foo=(\d+).+',
                             lambda match: int(match.group(1)),
                             lambda x: str(x),
                             regex_flags=re.DOTALL,
                             value=12)
        # Assert something good with dotall update()
        self.assertTrue(pdv)
        pdv.update("\n\nfoo=1212\n\n")
        self.assertEqual(pdv.get_value(), 1212)

        # negative test with no regex_flags
        pdv = RegexParameter("foo",
                             r'.+foo=(\d+).+',
                             lambda match: int(match.group(1)),
                             lambda x: str(x),
                             value=12)
        # Assert something good with dotall update()
        self.assertTrue(pdv)
        pdv.update("\n\nfoo=1212\n\n")
        self.assertEqual(pdv.get_value(), 12)

        self.assertRaises(TypeError,
                          RegexParameter,
                          "foo",
                          r'.*foo=(\d+).*',
                          lambda match: int(match.group(1)),
                          lambda x: str(x),
                          regex_flags="bad flag",
                          value=12)

    def test_format_current(self):
        self.param_dict.add("test_format",
                            r'.*foo=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: x + 5,
                            value=10)
        self.assertEqual(self.param_dict.format("test_format", 20), 25)
        self.assertEqual(self.param_dict.format("test_format"), 15)
        self.assertRaises(KeyError, self.param_dict.format, "bad_name")

    def _assert_metadata_change(self):
        new_dict = self.param_dict.generate_dict()
        log.debug("Generated dictionary: %s", new_dict)
        self.assertEqual(new_dict["qut"][ParameterDictKey.DESCRIPTION],
                         "QutFileDesc")
        self.assertEqual(new_dict["qut"][ParameterDictKey.DISPLAY_NAME],
                         "QutDisplay")
        self.assertEqual(
            new_dict["qut"][ParameterDictKey.VALUE][ParameterDictKey.UNITS],
            "QutFileUnits")
        self.assertEqual(
            new_dict["qut"][ParameterDictKey.VALUE][
                ParameterDictKey.DESCRIPTION], "QutFileValueDesc")
        self.assertEqual(
            new_dict["qut"][ParameterDictKey.VALUE][ParameterDictKey.TYPE],
            "QutFileType")
        # Should come from hard code
        # self.assertEqual(new_dict["qut"][ParameterDictKey.DISPLAY_NAME], "QutFileName")

        # from base hard code
        new_dict = self.param_dict.generate_dict()
        self.assertEqual(new_dict["baz"][ParameterDictKey.DESCRIPTION],
                         "The baz parameter")
        self.assertEqual(
            new_dict["baz"][ParameterDictKey.VALUE][ParameterDictKey.UNITS],
            "nano-bazers")
        self.assertEqual(
            new_dict["baz"][ParameterDictKey.VALUE][
                ParameterDictKey.DESCRIPTION],
            "Should be an integer between 2 and 2000")
        self.assertEqual(
            new_dict["baz"][ParameterDictKey.VALUE][ParameterDictKey.TYPE],
            ParameterDictType.INT)
        self.assertEqual(new_dict["baz"][ParameterDictKey.DISPLAY_NAME], "Baz")

        self.assertTrue('extra_param' not in new_dict)
class TestUnitProtocolParameterDict(TestUnitStringsDict):
    """
    Test cases for instrument driver class. Functions in this class provide
    instrument driver unit tests and provide a tutorial on use of
    the driver interface.
    """

    __test__ = True

    @staticmethod
    def pick_byte2(input_val):
        """ Get the 2nd byte as an example of something tricky and
        arbitrary"""
        val = int(input_val) >> 8
        val &= 255
        return val

    def setUp(self):
        self.param_dict = ProtocolParameterDict()

        self.param_dict.add("foo", r'.*foo=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: str(x),
                            direct_access=True,
                            startup_param=True,
                            default_value=10,
                            visibility=ParameterDictVisibility.READ_WRITE)
        self.param_dict.add("bar", r'.*bar=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: str(x),
                            direct_access=False,
                            startup_param=True,
                            default_value=15,
                            visibility=ParameterDictVisibility.READ_WRITE)
        self.param_dict.add("baz", r'.*baz=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: str(x),
                            direct_access=True,
                            default_value=20,
                            visibility=ParameterDictVisibility.DIRECT_ACCESS,
                            get_timeout=30,
                            set_timeout=40,
                            display_name="Baz",
                            description="The baz parameter",
                            type=ParameterDictType.INT,
                            units="nano-bazers",
                            value_description="Should be an integer between 2 and 2000")
        self.param_dict.add("bat", r'.*bat=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: str(x),
                            startup_param=False,
                            default_value=20,
                            visibility=ParameterDictVisibility.READ_ONLY,
                            get_timeout=10,
                            set_timeout=20,
                            display_name="Bat",
                            description="The bat parameter",
                            type=ParameterDictType.INT,
                            units="nano-batbit",
                            value_description="Should be an integer between 1 and 1000")
        self.param_dict.add("qux", r'.*qux=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: str(x),
                            startup_param=False,
                            visibility=ParameterDictVisibility.READ_ONLY)
        self.param_dict.add("pho", r'.*qux=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: str(x),
                            startup_param=False,
                            visibility=ParameterDictVisibility.IMMUTABLE)
        self.param_dict.add("dil", r'.*qux=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: str(x),
                            startup_param=False,
                            visibility=ParameterDictVisibility.IMMUTABLE)
        self.param_dict.add("qut", r'.*qut=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: str(x),
                            direct_access=True,
                            default_value=[10, 100],
                            visibility=ParameterDictVisibility.DIRECT_ACCESS,
                            expiration=1,
                            get_timeout=10,
                            set_timeout=20,
                            display_name="Qut",
                            description="The qut list parameter",
                            type=ParameterDictType.LIST,
                            units="nano-qutters",
                            value_description="Should be a 2-10 element list of integers between 2 and 2000")

        self.target_schema = {
            "bar": {
                "direct_access": False,
                "get_timeout": 10,
                "set_timeout": 10,
                "startup": True,
                "value": {
                    "default": 15
                },
                "visibility": "READ_WRITE",
                "range": None,
            },
            "bat": {
                "description": "The bat parameter",
                "direct_access": False,
                "display_name": "Bat",
                "get_timeout": 10,
                "set_timeout": 20,
                "startup": False,
                "value": {
                    "default": 20,
                    "description": "Should be an integer between 1 and 1000",
                    "type": "int",
                    "units": "nano-batbit"
                },
                "visibility": "READ_ONLY",
                "range": None,
            },
            "baz": {
                "description": "The baz parameter",
                "direct_access": True,
                "display_name": "Baz",
                "get_timeout": 30,
                "set_timeout": 40,
                "startup": False,
                "value": {
                    "default": 20,
                    "description": "Should be an integer between 2 and 2000",
                    "type": "int",
                    "units": "nano-bazers"
                },
                "visibility": "DIRECT_ACCESS",
                "range": None,
            },
            "dil": {
                "direct_access": False,
                "get_timeout": 10,
                "set_timeout": 10,
                "startup": False,
                "value": {},
                "visibility": "IMMUTABLE",
                "range": None,
            },
            "foo": {
                "direct_access": True,
                "get_timeout": 10,
                "set_timeout": 10,
                "startup": True,
                "value": {
                    "default": 10
                },
                "visibility": "READ_WRITE",
                "range": None,
            },
            "pho": {
                "direct_access": False,
                "get_timeout": 10,
                "set_timeout": 10,
                "startup": False,
                "value": {},
                "visibility": "IMMUTABLE",
                "range": None,
            },
            "qut": {
                "description": "The qut list parameter",
                "direct_access": True,
                "display_name": "Qut",
                "get_timeout": 10,
                "set_timeout": 20,
                "startup": False,
                "value": {
                    "default": [
                        10,
                        100
                    ],
                    "description": "Should be a 2-10 element list of integers between 2 and 2000",
                    "type": "list",
                    "units": "nano-qutters"
                },
                "visibility": "DIRECT_ACCESS",
                "range": None,
            },
            "qux": {
                "direct_access": False,
                "get_timeout": 10,
                "set_timeout": 10,
                "startup": False,
                "value": {},
                "visibility": "READ_ONLY",
                "range": None,
            }
        }

        self.test_yaml = '''
        parameters: {
            qut: {
            description: "QutFileDesc",
            units: "QutFileUnits",
            value_description: "QutFileValueDesc",
            type: "QutFileType",
            display_name: "QutDisplay"
            },
            extra_param: {
            description: "ExtraFileDesc",
            units: "ExtraFileUnits",
            value_description: "ExtraFileValueDesc",
            type: "ExtraFileType"
            }
          }

        commands: {
          dummy: stuff
          }
        '''

    def test_get_direct_access_list(self):
        """
        Test to see we can get a list of direct access parameters
        """
        result = self.param_dict.get_direct_access_list()
        self.assertTrue(isinstance(result, list))
        self.assertEquals(len(result), 3)
        self.assert_("foo" in result)
        self.assert_("baz" in result)
        self.assert_("qut" in result)

    def test_get_startup_list(self):
        """
        Test to see we can get a list of direct access parameters
        """
        result = self.param_dict.get_startup_list()
        self.assertTrue(isinstance(result, list))
        self.assertEquals(len(result), 2)
        self.assert_("foo" in result)
        self.assert_("bar" in result)

    def test_set_default(self):
        """
        Test setting a default value
        """
        result = self.param_dict.get_config()
        self.assertEquals(result["foo"], None)
        self.param_dict.set_default("foo")
        self.assertEquals(self.param_dict.get("foo"), 10)
        self.param_dict.update("foo=1000")
        self.assertEquals(self.param_dict.get("foo"), 1000)
        self.param_dict.set_default("foo")
        self.assertEquals(self.param_dict.get("foo"), 10)

        self.assertRaises(ValueError, self.param_dict.set_default, "qux")

    def test_update_many(self):
        """
        Test updating of multiple variables from the same input
        """
        sample_input = """
foo=100
bar=200, baz=300
"""
        self.assertNotEquals(self.param_dict.get("foo"), 100)
        self.assertNotEquals(self.param_dict.get("bar"), 200)
        self.assertNotEquals(self.param_dict.get("baz"), 300)
        result = self.param_dict.update_many(sample_input)
        log.debug("result: %s", result)
        self.assertEquals(result["foo"], True)
        self.assertEquals(result["bar"], True)
        self.assertEquals(result["baz"], True)
        self.assertEquals(self.param_dict.get("foo"), 100)
        self.assertEquals(self.param_dict.get("bar"), 200)
        self.assertEquals(self.param_dict.get("baz"), 300)

    def test_update_specific_values(self):
        """
        test to verify we can limit update to a specific
        set of parameters
        """
        sample_input = "foo=100, bar=200"

        # First verify we can set both
        self.assertNotEquals(self.param_dict.get("foo"), 100)
        self.assertNotEquals(self.param_dict.get("bar"), 200)
        self.assertTrue(self.param_dict.update(sample_input))
        self.assertEquals(self.param_dict.get("foo"), 100)
        self.assertEquals(self.param_dict.get("bar"), 200)

        # Now let's only have it update 1 parameter with a name
        sample_input = "foo=200, bar=300"
        self.assertTrue(self.param_dict.update(sample_input, target_params="foo"))
        self.assertEquals(self.param_dict.get("foo"), 200)
        self.assertEquals(self.param_dict.get("bar"), 200)

        # Now let's only have it update 1 parameter using a list
        sample_input = "foo=300, bar=400"
        self.assertTrue(self.param_dict.update(sample_input, target_params=["foo"]))
        self.assertEquals(self.param_dict.get("foo"), 300)
        self.assertEquals(self.param_dict.get("bar"), 200)

        # Test our exceptions
        with self.assertRaises(KeyError):
            self.param_dict.update(sample_input, "key_does_not_exist")

        with self.assertRaises(InstrumentParameterException):
            self.param_dict.update(sample_input, {'bad': "key_does_not_exist"})

    def test_visibility_list(self):
        lst = self.param_dict.get_visibility_list(ParameterDictVisibility.READ_WRITE)
        lst.sort()
        self.assertEquals(lst, ["bar", "foo"])
        lst = self.param_dict.get_visibility_list(ParameterDictVisibility.DIRECT_ACCESS)
        lst.sort()
        self.assertEquals(lst, ["baz", "qut"])
        lst = self.param_dict.get_visibility_list(ParameterDictVisibility.READ_ONLY)
        lst.sort()
        self.assertEquals(lst, ["bat", "qux"])
        lst = self.param_dict.get_visibility_list(ParameterDictVisibility.IMMUTABLE)
        lst.sort()
        self.assertEquals(lst, ["dil", "pho"])

    def test_function_values(self):
        """
        Make sure we can add and update values with functions instead of patterns
        """

        self.param_dict.add_parameter(
                FunctionParameter("fn_foo",
                                  self.pick_byte2,
                                  lambda x: str(x),
                                  direct_access=True,
                                  startup_param=True,
                                  value=1,
                                  visibility=ParameterDictVisibility.READ_WRITE)
        )
        self.param_dict.add_parameter(
                FunctionParameter("fn_bar",
                                  lambda x: bool(x & 2),  # bit map example
                                  lambda x: str(x),
                                  direct_access=True,
                                  startup_param=True,
                                  value=False,
                                  visibility=ParameterDictVisibility.READ_WRITE)
        )

        # check defaults just to be safe
        val = self.param_dict.get("fn_foo")
        self.assertEqual(val, 1)
        val = self.param_dict.get("fn_bar")
        self.assertEqual(val, False)

        self.param_dict.update(1005)  # just change first in list
        val = self.param_dict.get("fn_foo")
        self.assertEqual(val, 3)
        val = self.param_dict.get("fn_bar")
        self.assertEqual(val, False)

        # fn_bar does not get updated here
        result = self.param_dict.update_many(1205)
        self.assertEqual(result['fn_foo'], True)
        self.assertEqual(len(result), 1)
        val = self.param_dict.get("fn_foo")
        self.assertEqual(val, 4)
        val = self.param_dict.get("fn_bar")
        self.assertEqual(val, False)

        # both are updated now
        result = self.param_dict.update_many(6)
        self.assertEqual(result['fn_foo'], True)
        self.assertEqual(result['fn_bar'], True)
        self.assertEqual(len(result), 2)

        val = self.param_dict.get("fn_foo")
        self.assertEqual(val, 0)
        val = self.param_dict.get("fn_bar")
        self.assertEqual(val, True)

    def test_mixed_pdv_types(self):
        """ Verify we can add different types of PDVs in one container """
        self.param_dict.add_parameter(
                FunctionParameter("fn_foo",
                                  self.pick_byte2,
                                  lambda x: str(x),
                                  direct_access=True,
                                  startup_param=True,
                                  value=1,
                                  visibility=ParameterDictVisibility.READ_WRITE)
        )
        self.param_dict.add_parameter(
                RegexParameter("foo", r'.*foo=(\d+).*',
                               lambda match: int(match.group(1)),
                               lambda x: str(x),
                               direct_access=True,
                               startup_param=True,
                               value=10,
                               visibility=ParameterDictVisibility.READ_WRITE)
        )
        self.param_dict.add("bar", r'.*bar=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: str(x),
                            direct_access=False,
                            startup_param=True,
                            value=15,
                            visibility=ParameterDictVisibility.READ_WRITE)

        self.assertEqual(self.param_dict.get("fn_foo"), 1)
        self.assertEqual(self.param_dict.get("foo"), 10)
        self.assertEqual(self.param_dict.get("bar"), 15)

    def test_base_update(self):
        pdv = Parameter("foo",
                        lambda x: str(x),
                        value=12)
        self.assertEqual(pdv.get_value(), 12)
        result = pdv.update(1)
        self.assertEqual(result, True)
        self.assertEqual(pdv.get_value(), 1)

        # Its a base class...monkey see, monkey do
        result = pdv.update("foo=1")
        self.assertEqual(result, True)
        self.assertEqual(pdv.get_value(), "foo=1")

    def test_regex_val(self):
        pdv = RegexParameter("foo",
                             r'.*foo=(\d+).*',
                             lambda match: int(match.group(1)),
                             lambda x: str(x),
                             value=12)
        self.assertEqual(pdv.get_value(), 12)
        result = pdv.update(1)
        self.assertEqual(result, False)
        self.assertEqual(pdv.get_value(), 12)
        result = pdv.update("foo=1")
        self.assertEqual(result, True)
        self.assertEqual(pdv.get_value(), 1)

    def test_function_val(self):
        pdv = FunctionParameter("foo",
                                self.pick_byte2,
                                lambda x: str(x),
                                value=12)
        self.assertEqual(pdv.get_value(), 12)
        self.assertRaises(TypeError, pdv.update(1))
        result = pdv.update("1205")
        self.assertEqual(pdv.get_value(), 4)
        self.assertEqual(result, True)

    def test_set_init_value(self):
        result = self.param_dict.get("foo")
        self.assertEqual(result, None)
        self.param_dict.set_init_value("foo", 42)
        result = self.param_dict.get_init_value("foo")
        self.assertEqual(result, 42)

    def test_schema_generation(self):
        self.maxDiff = None
        result = self.param_dict.generate_dict()
        json_result = json.dumps(result, indent=4, sort_keys=True)
        log.debug("Expected: %s", self.target_schema)
        log.debug("Result: %s", json_result)
        self.assertEqual(result, self.target_schema)

    def test_empty_schema(self):
        self.param_dict = ProtocolParameterDict()
        result = self.param_dict.generate_dict()
        self.assertEqual(result, {})

    def test_bad_descriptions(self):
        self.param_dict._param_dict["foo"].description = None
        self.param_dict._param_dict["foo"].value = None
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.get_init_value, "foo")
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.get_default_value, "foo")
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.set_default, "foo")
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.get_init_value, "foo")
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.get_menu_path_read, "foo")
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.get_submenu_read, "foo")
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.get_menu_path_write, "foo")
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.get_submenu_write, "foo")
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.format, "foo", 1)
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.get_direct_access_list)
        self.assertRaises(InstrumentParameterException,
                          self.param_dict.is_startup_param, "foo")

    def test_set(self):
        """
        Test a simple set of the parameter. Make sure the right values get
        called and the correct exceptions are raised.
        """
        new_param = FunctionParameter("foo",
                                      self.pick_byte2,
                                      lambda x: str(x),
                                      direct_access=True,
                                      startup_param=True,
                                      value=1000,
                                      visibility=ParameterDictVisibility.READ_WRITE)
        self.assertEquals(new_param.get_value(), 1000)
        self.assertEquals(self.param_dict.get("foo"), None)
        # overwrites existing param
        self.param_dict.add_parameter(new_param)
        self.assertEquals(self.param_dict.get("foo"), 1000)
        self.param_dict.set_value("foo", 2000)
        self.assertEquals(self.param_dict.get("foo"), 2000)

    def test_invalid_type(self):
        self.assertRaises(InstrumentParameterException,
                          FunctionParameter,
                          "fn_bar",
                          lambda x: bool(x & 2),  # bit map example
                          lambda x: str(x),
                          direct_access=True,
                          startup_param=True,
                          value=False,
                          type="bad_type",
                          visibility=ParameterDictVisibility.READ_WRITE)

    def test_get(self):
        """
        test getting values with expiration
        """
        # from mi.core.exceptions import InstrumentParameterExpirationException
        pd = ProtocolParameterDict()

        # No expiration, should work just fine
        pd.add('noexp', r'', None, None, expiration=None)
        pd.add('zeroexp', r'', None, None, expiration=0)
        pd.add('lateexp', r'', None, None, expiration=2)

        ###
        # Set and get with no expire
        ###
        pd.set_value('noexp', 1)
        self.assertEqual(pd.get('noexp'), 1)

        ###
        # Set and get with a 0 expire
        ###
        basetime = pd.get_current_timestamp()
        pd.set_value('zeroexp', 2)

        # We should fail because we are calculating exp against current time
        with self.assertRaises(InstrumentParameterExpirationException):
            pd.get('zeroexp')

        # Should succeed because exp is calculated using basetime
        self.assertEqual(pd.get('zeroexp', basetime), 2)

        ###
        # Set and get with a delayed expire
        ###
        basetime = pd.get_current_timestamp()
        futuretime = pd.get_current_timestamp(3)
        self.assertGreater(futuretime - basetime, 3)

        pd.set_value('lateexp', 2)

        # Success because data is not expired
        self.assertEqual(pd.get('lateexp', basetime), 2)

        # Fail because data is expired (simulated three seconds from now)
        with self.assertRaises(InstrumentParameterExpirationException):
            pd.get('lateexp', futuretime)

    def test_regex_flags(self):
        pdv = RegexParameter("foo",
                             r'.+foo=(\d+).+',
                             lambda match: int(match.group(1)),
                             lambda x: str(x),
                             regex_flags=re.DOTALL,
                             value=12)
        # Assert something good with dotall update()
        self.assertTrue(pdv)
        pdv.update("\n\nfoo=1212\n\n")
        self.assertEqual(pdv.get_value(), 1212)

        # negative test with no regex_flags
        pdv = RegexParameter("foo",
                             r'.+foo=(\d+).+',
                             lambda match: int(match.group(1)),
                             lambda x: str(x),
                             value=12)
        # Assert something good with dotall update()
        self.assertTrue(pdv)
        pdv.update("\n\nfoo=1212\n\n")
        self.assertEqual(pdv.get_value(), 12)

        self.assertRaises(TypeError,
                          RegexParameter,
                          "foo",
                          r'.*foo=(\d+).*',
                          lambda match: int(match.group(1)),
                          lambda x: str(x),
                          regex_flags="bad flag",
                          value=12)

    def test_format_current(self):
        self.param_dict.add("test_format", r'.*foo=(\d+).*',
                            lambda match: int(match.group(1)),
                            lambda x: x + 5,
                            value=10)
        self.assertEqual(self.param_dict.format("test_format", 20), 25)
        self.assertEqual(self.param_dict.format("test_format"), 15)
        self.assertRaises(KeyError,
                          self.param_dict.format, "bad_name")

    def _assert_metadata_change(self):
        new_dict = self.param_dict.generate_dict()
        log.debug("Generated dictionary: %s", new_dict)
        self.assertEqual(new_dict["qut"][ParameterDictKey.DESCRIPTION], "QutFileDesc")
        self.assertEqual(new_dict["qut"][ParameterDictKey.DISPLAY_NAME], "QutDisplay")
        self.assertEqual(new_dict["qut"][ParameterDictKey.VALUE][ParameterDictKey.UNITS], "QutFileUnits")
        self.assertEqual(new_dict["qut"][ParameterDictKey.VALUE][ParameterDictKey.DESCRIPTION], "QutFileValueDesc")
        self.assertEqual(new_dict["qut"][ParameterDictKey.VALUE][ParameterDictKey.TYPE], "QutFileType")
        # Should come from hard code
        # self.assertEqual(new_dict["qut"][ParameterDictKey.DISPLAY_NAME], "QutFileName")

        # from base hard code
        new_dict = self.param_dict.generate_dict()
        self.assertEqual(new_dict["baz"][ParameterDictKey.DESCRIPTION],
                         "The baz parameter")
        self.assertEqual(new_dict["baz"][ParameterDictKey.VALUE][ParameterDictKey.UNITS],
                         "nano-bazers")
        self.assertEqual(new_dict["baz"][ParameterDictKey.VALUE][ParameterDictKey.DESCRIPTION],
                         "Should be an integer between 2 and 2000")
        self.assertEqual(new_dict["baz"][ParameterDictKey.VALUE][ParameterDictKey.TYPE],
                         ParameterDictType.INT)
        self.assertEqual(new_dict["baz"][ParameterDictKey.DISPLAY_NAME], "Baz")

        self.assertTrue('extra_param' not in new_dict)
class DataSetDriver(object):
    """
    Base class for data set drivers.  Provides:
    - an interface via callback to publish data
    - an interface via callback to persist driver state
    - an interface via callback to handle exceptions
    - an start and stop sampling
    - a client interface for execute resource

    Subclasses need to include harvesters and parsers and
    be specialized to handle the interaction between the two.
    
    Configurations should contain keys from the DataSetDriverConfigKey class
    and should look something like this example (more full documentation in the
    "Dataset Agent Architecture" page on the OOI wiki):
    {
        'harvester':
        {
            'directory': '/tmp/dsatest',
            'pattern': '*.txt',
            'frequency': 1,
        },
        'parser': {}
        'driver': {
            'records_per_second'
            'harvester_polling_interval'
            'batched_particle_count'
        }
    }
    """
    def __init__(self, config, memento, data_callback, state_callback,
                 exception_callback):
        self._config = config
        self._data_callback = data_callback
        self._state_callback = state_callback
        self._exception_callback = exception_callback
        self._memento = memento
        self._publisher_thread = None

        self._verify_config()
        self._param_dict = ProtocolParameterDict()

        # Updated my set_resource, defaults defined in build_param_dict
        self._polling_interval = None
        self._generate_particle_count = None
        self._particle_count_per_second = None

        self._build_param_dict()

    def shutdown(self):
        self.stop_sampling()

    def start_sampling(self):
        """
        Start a new thread to monitor for data
        """
        self._start_sampling()
        self._start_publisher_thread()

    def stop_sampling(self):
        """
        Stop the sampling thread
        """
        log.debug("Stopping driver now")

        self._stop_sampling()
        self._stop_publisher_thread()

    def _start_sampling(self):
        raise NotImplementedException(
            'virtual methond needs to be specialized')

    def _stop_sampling(self):
        raise NotImplementedException(
            'virtual methond needs to be specialized')

    def cmd_dvr(self, cmd, *args, **kwargs):
        log.warn("DRIVER: cmd_dvr %s", cmd)

        if not cmd in [
                'execute_resource', 'get_resource_capabilities',
                'set_resource', 'get_resource'
        ]:
            log.error("Unhandled command: %s", cmd)
            raise InstrumentStateException("Unhandled command: %s" % cmd)

        resource_cmd = args[0]

        if cmd == 'execute_resource':
            if resource_cmd == DriverEvent.START_AUTOSAMPLE:
                return (ResourceAgentState.STREAMING, None)

            elif resource_cmd == DriverEvent.STOP_AUTOSAMPLE:
                self.stop_sampling()
                return (ResourceAgentState.COMMAND, None)

            else:
                log.error("Unhandled resource command: %s", resource_cmd)
                raise

        elif cmd == 'get_resource_capabilities':
            return self.get_resource_capabilities()

        elif cmd == 'set_resource':
            return self.set_resource(*args, **kwargs)

        elif cmd == 'get_resource':
            return self.get_resource(*args, **kwargs)

    def get_resource_capabilities(self, current_state=True, *args, **kwargs):
        """
        Return driver commands and parameters.
        @param current_state True to retrieve commands available in current
        state, otherwise reutrn all commands.
        @retval list of AgentCapability objects representing the drivers
        capabilities.
        @raises NotImplementedException if not implemented by subclass.
        """
        res_params = self._param_dict.get_keys()
        return [[], res_params]

    def set_resource(self, *args, **kwargs):
        """
        Set the driver parameter
        """
        log.trace("start set_resource")
        try:
            params = args[0]
        except IndexError:
            raise InstrumentParameterException(
                'Set command requires a parameter dict.')

        log.trace("set_resource: iterate through params: %s", params)
        for (key, val) in params.iteritems():
            if key in [
                    DriverParameter.BATCHED_PARTICLE_COUNT,
                    DriverParameter.RECORDS_PER_SECOND
            ]:
                if not isinstance(val, int):
                    raise InstrumentParameterException(
                        "%s must be an integer" % key)
            if key in [DriverParameter.PUBLISHER_POLLING_INTERVAL]:
                if not isinstance(val, (int, float)):
                    raise InstrumentParameterException("%s must be an float" %
                                                       key)

            if val <= 0:
                raise InstrumentParameterException("%s must be > 0" % key)

            self._param_dict.set_value(key, val)

        # Set the driver parameters
        self._generate_particle_count = self._param_dict.get(
            DriverParameter.BATCHED_PARTICLE_COUNT)
        self._particle_count_per_second = self._param_dict.get(
            DriverParameter.RECORDS_PER_SECOND)
        self._polling_interval = self._param_dict.get(
            DriverParameter.PUBLISHER_POLLING_INTERVAL)
        log.trace("Driver Parameters: %s, %s, %s", self._polling_interval,
                  self._particle_count_per_second,
                  self._generate_particle_count)

    def get_resource(self, *args, **kwargs):
        """
        Get driver parameter
        """
        result = {}

        try:
            params = args[0]
        except IndexError:
            raise InstrumentParameterException(
                'Set command requires a parameter list.')

        # If all params requested, retrieve config.
        if params == DriverParameter.ALL:
            result = self._param_dict.get_config()

        # If not all params, confirm a list or tuple of params to retrieve.
        # Raise if not a list or tuple.
        # Retrieve each key in the list, raise if any are invalid.
        else:
            if not isinstance(params, (list, tuple)):
                raise InstrumentParameterException(
                    'Get argument not a list or tuple.')
            result = {}
            for key in params:
                try:
                    val = self._param_dict.get(key)
                    result[key] = val

                except KeyError:
                    raise InstrumentParameterException(
                        ('%s is not a valid parameter.' % key))

        return result

    def _verify_config(self):
        """
        virtual method to verify the supplied driver configuration is value.  Must
        be overloaded in sub classes.

        raises an ConfigurationException when a configuration error is detected.
        """
        raise NotImplementedException(
            'virtual methond needs to be specialized')

    def _build_param_dict(self):
        """
        Setup three common driver parameters
        """
        self._param_dict.add_parameter(
            Parameter(DriverParameter.RECORDS_PER_SECOND,
                      int,
                      value=60,
                      type=ParameterDictType.INT,
                      display_name="Records Per Second",
                      description="Number of records to process per second"))

        self._param_dict.add_parameter(
            Parameter(
                DriverParameter.PUBLISHER_POLLING_INTERVAL,
                float,
                value=1,
                type=ParameterDictType.FLOAT,
                display_name="Harvester Polling Interval",
                description=
                "Duration in minutes to wait before checking for new files."))

        self._param_dict.add_parameter(
            Parameter(
                DriverParameter.BATCHED_PARTICLE_COUNT,
                int,
                value=1,
                type=ParameterDictType.INT,
                display_name="Batched Particle Count",
                description=
                "Number of particles to batch before sending to the agent"))

        config = self._config.get(DataSourceConfigKey.DRIVER, {})
        log.debug("set_resource on startup with: %s", config)
        self.set_resource(config)

    def _start_publisher_thread(self):
        self._publisher_thread = gevent.spawn(self._publisher_loop)
        self._publisher_shutdown = False

    def _stop_publisher_thread(self):
        log.debug("Signal shutdown")
        self._publisher_shutdown = True
        if self._publisher_thread:
            self._publisher_thread.kill(block=False)
        log.debug("shutdown complete")

    def _publisher_loop(self):
        """
        Main loop to listen for new files to parse.  Parse them and move on.
        """
        log.info("Starting main publishing loop")

        try:
            while (not self._publisher_shutdown):
                self._poll()
                gevent.sleep(self._polling_interval)
        except Exception as e:
            log.error("Exception in publisher thread: %s", e)
            self._exception_callback(e)

        log.debug("publisher thread detected shutdown request")

    def _poll(self):
        raise NotImplementedException(
            'virtual methond needs to be specialized')

    def _new_file_exception(self):
        raise NotImplementedException(
            'virtual methond needs to be specialized')
class DataSetDriver(object):
    """
    Base class for data set drivers.  Provides:
    - an interface via callback to publish data
    - an interface via callback to persist driver state
    - an interface via callback to handle exceptions
    - an start and stop sampling
    - a client interface for execute resource

    Subclasses need to include harvesters and parsers and
    be specialized to handle the interaction between the two.
    
    Configurations should contain keys from the DataSetDriverConfigKey class
    and should look something like this example (more full documentation in the
    "Dataset Agent Architecture" page on the OOI wiki):
    {
        'harvester':
        {
            'directory': '/tmp/dsatest',
            'pattern': '*.txt',
            'frequency': 1,
        },
        'parser': {}
        'driver': {
            'records_per_second'
            'harvester_polling_interval'
            'batched_particle_count'
        }
    }
    """
    def __init__(self, config, memento, data_callback, state_callback, exception_callback):
        self._config = config
        self._data_callback = data_callback
        self._state_callback = state_callback
        self._exception_callback = exception_callback
        self._memento = memento
        self._publisher_thread = None

        self._verify_config()

        # Updated my set_resource, defaults defined in build_param_dict
        self._polling_interval = None
        self._generate_particle_count = None
        self._particle_count_per_second = None

        self._param_dict = ProtocolParameterDict()
        self._cmd_dict = ProtocolCommandDict()
        self._driver_dict = DriverDict()

        self._build_command_dict()
        self._build_driver_dict()
        self._build_param_dict()

    def shutdown(self):
        self.stop_sampling()

    def start_sampling(self):
        """
        Start a new thread to monitor for data
        """
        self._start_sampling()
        self._start_publisher_thread()

    def stop_sampling(self):
        """
        Stop the sampling thread
        """
        log.debug("Stopping driver now")

        self._stop_sampling()
        self._stop_publisher_thread()

    def _start_sampling(self):
        raise NotImplementedException('virtual method needs to be specialized')

    def _stop_sampling(self):
        raise NotImplementedException('virtual method needs to be specialized')

    def _is_sampling(self):
        """
        Currently the drivers only have two states, command and streaming and
        all resource commands are common, either start or stop autosample.
        Therefore we didn't implement an enitre state machine to manage states
        and commands.  If it does get more complex than this we should take the
        time to implement a state machine to add some flexibility
        """
        raise NotImplementedException('virtual method needs to be specialized')

    def cmd_dvr(self, cmd, *args, **kwargs):
        log.warn("DRIVER: cmd_dvr %s", cmd)

        if cmd == 'execute_resource':
            resource_cmd = args[0]

            if resource_cmd == DriverEvent.START_AUTOSAMPLE:
                return (ResourceAgentState.STREAMING, None)

            elif resource_cmd == DriverEvent.STOP_AUTOSAMPLE:
                self.stop_sampling()
                return (ResourceAgentState.COMMAND, None)

            else:
                log.error("Unhandled resource command: %s", resource_cmd)
                raise

        elif cmd == 'get_resource_capabilities':
            return self.get_resource_capabilities()

        elif cmd == 'set_resource':
            return self.set_resource(*args, **kwargs)

        elif cmd == 'get_resource':
            return self.get_resource(*args, **kwargs)

        elif cmd == 'get_config_metadata':
            return self.get_config_metadata(*args, **kwargs)

        elif cmd == 'disconnect':
            pass

        elif cmd == 'initialize':
            pass

        else:
            log.error("Unhandled command: %s", cmd)
            raise InstrumentStateException("Unhandled command: %s" % cmd)

    def get_resource_capabilities(self, current_state=True, *args, **kwargs):
        """
        Return driver commands and parameters.
        @param current_state True to retrieve commands available in current
        state, otherwise reutrn all commands.
        @retval list of AgentCapability objects representing the drivers
        capabilities.
        @raises NotImplementedException if not implemented by subclass.
        """
        res_params = self._param_dict.get_keys()
        res_cmds = [DriverEvent.STOP_AUTOSAMPLE, DriverEvent.START_AUTOSAMPLE]

        if current_state and self._is_sampling():
            res_cmds = [DriverEvent.STOP_AUTOSAMPLE]
        elif current_state and not self._is_sampling():
            res_cmds = [DriverEvent.START_AUTOSAMPLE]

        return [res_cmds, res_params]

    def set_resource(self, *args, **kwargs):
        """
        Set the driver parameter
        """
        log.trace("start set_resource")
        try:
            params = args[0]
        except IndexError:
            raise InstrumentParameterException('Set command requires a parameter dict.')

        log.trace("set_resource: iterate through params: %s", params)
        for (key, val) in params.iteritems():
            if key in [DriverParameter.BATCHED_PARTICLE_COUNT, DriverParameter.RECORDS_PER_SECOND]:
                if not isinstance(val, int): raise InstrumentParameterException("%s must be an integer" % key)
            if key in [DriverParameter.PUBLISHER_POLLING_INTERVAL]:
                if not isinstance(val, (int, float)): raise InstrumentParameterException("%s must be an float" % key)

            if val <= 0:
                raise InstrumentParameterException("%s must be > 0" % key)

            self._param_dict.set_value(key, val)

        # Set the driver parameters
        self._generate_particle_count = self._param_dict.get(DriverParameter.BATCHED_PARTICLE_COUNT)
        self._particle_count_per_second = self._param_dict.get(DriverParameter.RECORDS_PER_SECOND)
        self._polling_interval = self._param_dict.get(DriverParameter.PUBLISHER_POLLING_INTERVAL)
        log.trace("Driver Parameters: %s, %s, %s", self._polling_interval, self._particle_count_per_second, self._generate_particle_count)

    def get_resource(self, *args, **kwargs):
        """
        Get driver parameter
        """
        result = {}

        try:
            params = args[0]
        except IndexError:
            raise InstrumentParameterException('Set command requires a parameter list.')

        # If all params requested, retrieve config.
        if params == DriverParameter.ALL:
            result = self._param_dict.get_config()

        # If not all params, confirm a list or tuple of params to retrieve.
        # Raise if not a list or tuple.
        # Retrieve each key in the list, raise if any are invalid.
        else:
            if not isinstance(params, (list, tuple)):
                raise InstrumentParameterException('Get argument not a list or tuple.')
            result = {}
            for key in params:
                try:
                    val = self._param_dict.get(key)
                    result[key] = val

                except KeyError:
                    raise InstrumentParameterException(('%s is not a valid parameter.' % key))

        return result

    def get_config_metadata(self):
        """
        Return the configuration metadata object in JSON format
        @retval The description of the parameters, commands, and driver info
        in a JSON string
        @see https://confluence.oceanobservatories.org/display/syseng/CIAD+MI+SV+Instrument+Driver-Agent+parameter+and+command+metadata+exchange
        """
        log.debug("Getting metadata from driver...")
        log.debug("Getting metadata dict from protocol...")
        return_dict = {}
        return_dict[ConfigMetadataKey.DRIVER] = self._driver_dict.generate_dict()
        return_dict[ConfigMetadataKey.COMMANDS] = self._cmd_dict.generate_dict()
        return_dict[ConfigMetadataKey.PARAMETERS] = self._param_dict.generate_dict()

        return return_dict

    def _verify_config(self):
        """
        virtual method to verify the supplied driver configuration is value.  Must
        be overloaded in sub classes.

        raises an ConfigurationException when a configuration error is detected.
        """
        raise NotImplementedException('virtual methond needs to be specialized')

    def _build_driver_dict(self):
        """
        Populate the driver dictionary with options
        """
        pass

    def _build_command_dict(self):
        """
        Populate the command dictionary with command.
        """
        self._cmd_dict.add(DriverEvent.START_AUTOSAMPLE, display_name="start autosample")
        self._cmd_dict.add(DriverEvent.STOP_AUTOSAMPLE, display_name="stop autosample")

    def _build_param_dict(self):
        """
        Setup three common driver parameters
        """
        self._param_dict.add_parameter(
            Parameter(
                DriverParameter.RECORDS_PER_SECOND,
                int,
                value=60,
                type=ParameterDictType.INT,
                visibility=ParameterDictVisibility.IMMUTABLE,
                display_name="Records Per Second",
                description="Number of records to process per second")
        )

        self._param_dict.add_parameter(
            Parameter(
                DriverParameter.PUBLISHER_POLLING_INTERVAL,
                float,
                value=1,
                type=ParameterDictType.FLOAT,
                visibility=ParameterDictVisibility.IMMUTABLE,
                display_name="Harvester Polling Interval",
                description="Duration in minutes to wait before checking for new files.")
        )

        self._param_dict.add_parameter(
            Parameter(
                DriverParameter.BATCHED_PARTICLE_COUNT,
                int,
                value=1,
                type=ParameterDictType.INT,
                visibility=ParameterDictVisibility.IMMUTABLE,
                display_name="Batched Particle Count",
                description="Number of particles to batch before sending to the agent")
        )

        config = self._config.get(DataSourceConfigKey.DRIVER, {})
        log.debug("set_resource on startup with: %s", config)
        self.set_resource(config)

    def _start_publisher_thread(self):
        self._publisher_thread = gevent.spawn(self._publisher_loop)
        self._publisher_shutdown = False

    def _stop_publisher_thread(self):
        log.debug("Signal shutdown")
        self._publisher_shutdown = True
        if self._publisher_thread:
            self._publisher_thread.kill(block=False)
        log.debug("shutdown complete")

    def _publisher_loop(self):
        """
        Main loop to listen for new files to parse.  Parse them and move on.
        """
        log.info("Starting main publishing loop")

        try:
            while(not self._publisher_shutdown):
                self._poll()
                gevent.sleep(self._polling_interval)
        except Exception as e:
            log.error("Exception in publisher thread: %s", e)
            self._exception_callback(e)

        log.debug("publisher thread detected shutdown request")

    def _poll(self):
        raise NotImplementedException('virtual methond needs to be specialized')

    def _new_file_exception(self):
        raise NotImplementedException('virtual methond needs to be specialized')
예제 #5
0
class DataSetDriver(object):
    """
    Base class for data set drivers.  Provides:
    - an interface via callback to publish data
    - an interface via callback to persist driver state
    - an interface via callback to handle exceptions
    - an start and stop sampling
    - a client interface for execute resource

    Subclasses need to include harvesters and parsers and
    be specialized to handle the interaction between the two.
    
    Configurations should contain keys from the DataSetDriverConfigKey class
    and should look something like this example (more full documentation in the
    "Dataset Agent Architecture" page on the OOI wiki):
    {
        'harvester':
        {
            'directory': '/tmp/dsatest',
            'storage_directory': '/tmp/stored_dsatest',
            'pattern': '*.txt',
            'frequency': 1,
            'file_mod_wait_time': 30,
        },
        'parser': {}
        'driver': {
            'records_per_second'
            'harvester_polling_interval'
            'batched_particle_count'
        }
    }
    """
    def __init__(self, config, memento, data_callback, state_callback, event_callback, exception_callback):
        self._config = copy.deepcopy(config)
        self._data_callback = data_callback
        self._state_callback = state_callback
        self._event_callback = event_callback
        self._exception_callback = exception_callback
        self._memento = memento
        self._publisher_thread = None

        self._verify_config()

        # Updated my set_resource, defaults defined in build_param_dict
        self._polling_interval = None
        self._generate_particle_count = None
        self._particle_count_per_second = None
        self._resource_id = None

        self._param_dict = ProtocolParameterDict()
        self._cmd_dict = ProtocolCommandDict()
        self._driver_dict = DriverDict()

        self._build_command_dict()
        self._build_driver_dict()
        self._build_param_dict()

    def shutdown(self):
        self.stop_sampling()

    def start_sampling(self):
        """
        Start a new thread to monitor for data
        """
        self._start_sampling()
        self._start_publisher_thread()

    def stop_sampling(self):
        """
        Stop the sampling thread
        """
        log.debug("Stopping sampling and publisher now")

        self._stop_sampling()
        self._stop_publisher_thread()

    def _start_sampling(self):
        raise NotImplementedException('virtual method needs to be specialized')

    def _stop_sampling(self):
        raise NotImplementedException('virtual method needs to be specialized')

    def _is_sampling(self):
        """
        Currently the drivers only have two states, command and streaming and
        all resource commands are common, either start or stop autosample.
        Therefore we didn't implement an enitre state machine to manage states
        and commands.  If it does get more complex than this we should take the
        time to implement a state machine to add some flexibility
        """
        raise NotImplementedException('virtual method needs to be specialized')

    def cmd_dvr(self, cmd, *args, **kwargs):
        log.warn("DRIVER: cmd_dvr %s", cmd)

        if cmd == 'execute_resource':
            resource_cmd = args[0]

            if resource_cmd == DriverEvent.START_AUTOSAMPLE:
                return (ResourceAgentState.STREAMING, None)

            elif resource_cmd == DriverEvent.STOP_AUTOSAMPLE:
                self.stop_sampling()
                return (ResourceAgentState.COMMAND, None)

            else:
                log.error("Unhandled resource command: %s", resource_cmd)
                raise

        elif cmd == 'get_resource_capabilities':
            return self.get_resource_capabilities()

        elif cmd == 'set_resource':
            return self.set_resource(*args, **kwargs)

        elif cmd == 'get_resource':
            return self.get_resource(*args, **kwargs)

        elif cmd == 'get_config_metadata':
            return self.get_config_metadata(*args, **kwargs)

        elif cmd == 'disconnect':
            pass

        elif cmd == 'initialize':
            pass

        else:
            log.error("Unhandled command: %s", cmd)
            raise InstrumentStateException("Unhandled command: %s" % cmd)

    def get_resource_capabilities(self, current_state=True, *args, **kwargs):
        """
        Return driver commands and parameters.
        @param current_state True to retrieve commands available in current
        state, otherwise reutrn all commands.
        @retval list of AgentCapability objects representing the drivers
        capabilities.
        @raises NotImplementedException if not implemented by subclass.
        """
        res_params = self._param_dict.get_keys()
        res_cmds = [DriverEvent.STOP_AUTOSAMPLE, DriverEvent.START_AUTOSAMPLE]

        if current_state and self._is_sampling():
            res_cmds = [DriverEvent.STOP_AUTOSAMPLE]
        elif current_state and not self._is_sampling():
            res_cmds = [DriverEvent.START_AUTOSAMPLE]

        return [res_cmds, res_params]

    def set_resource(self, *args, **kwargs):
        """
        Set the driver parameter
        """
        log.trace("start set_resource")
        try:
            params = args[0]
        except IndexError:
            raise InstrumentParameterException('Set command requires a parameter dict.')

        log.trace("set_resource: iterate through params: %s", params)
        for (key, val) in params.iteritems():
            if key in [DriverParameter.BATCHED_PARTICLE_COUNT, DriverParameter.RECORDS_PER_SECOND]:
                if not isinstance(val, int): raise InstrumentParameterException("%s must be an integer" % key)
            if key in [DriverParameter.PUBLISHER_POLLING_INTERVAL]:
                if not isinstance(val, (int, float)): raise InstrumentParameterException("%s must be an float" % key)

            if val <= 0:
                raise InstrumentParameterException("%s must be > 0" % key)

            self._param_dict.set_value(key, val)

        # Set the driver parameters
        self._generate_particle_count = self._param_dict.get(DriverParameter.BATCHED_PARTICLE_COUNT)
        self._particle_count_per_second = self._param_dict.get(DriverParameter.RECORDS_PER_SECOND)
        self._polling_interval = self._param_dict.get(DriverParameter.PUBLISHER_POLLING_INTERVAL)
        log.trace("Driver Parameters: %s, %s, %s", self._polling_interval, self._particle_count_per_second,
                  self._generate_particle_count)


    def get_resource(self, *args, **kwargs):
        """
        Get driver parameter
        """
        result = {}

        try:
            params = args[0]
        except IndexError:
            raise InstrumentParameterException('Set command requires a parameter list.')

        # If all params requested, retrieve config.
        if params == DriverParameter.ALL:
            result = self._param_dict.get_config()

        # If not all params, confirm a list or tuple of params to retrieve.
        # Raise if not a list or tuple.
        # Retrieve each key in the list, raise if any are invalid.
        else:
            if not isinstance(params, (list, tuple)):
                raise InstrumentParameterException('Get argument not a list or tuple.')
            result = {}
            for key in params:
                try:
                    val = self._param_dict.get(key)
                    result[key] = val

                except KeyError:
                    raise InstrumentParameterException(('%s is not a valid parameter.' % key))

        return result

    def get_config_metadata(self):
        """
        Return the configuration metadata object in JSON format
        @retval The description of the parameters, commands, and driver info
        in a JSON string
        @see https://confluence.oceanobservatories.org/display/syseng/CIAD+MI+SV+Instrument+Driver-Agent+parameter+and+command+metadata+exchange
        """
        log.debug("Getting metadata from driver...")
        log.debug("Getting metadata dict from protocol...")
        return_dict = {}
        return_dict[ConfigMetadataKey.DRIVER] = self._driver_dict.generate_dict()
        return_dict[ConfigMetadataKey.COMMANDS] = self._cmd_dict.generate_dict()
        return_dict[ConfigMetadataKey.PARAMETERS] = self._param_dict.generate_dict()

        return return_dict

    def _verify_config(self):
        """
        virtual method to verify the supplied driver configuration is value.  Must
        be overloaded in sub classes.

        raises an ConfigurationException when a configuration error is detected.
        """
        raise NotImplementedException('virtual methond needs to be specialized')

    def _build_driver_dict(self):
        """
        Populate the driver dictionary with options
        """
        pass

    def _build_command_dict(self):
        """
        Populate the command dictionary with command.
        """
        self._cmd_dict.add(DriverEvent.START_AUTOSAMPLE, display_name="start autosample")
        self._cmd_dict.add(DriverEvent.STOP_AUTOSAMPLE, display_name="stop autosample")

    def _build_param_dict(self):
        """
        Setup three common driver parameters
        """
        self._param_dict.add_parameter(
            Parameter(
                DriverParameter.RECORDS_PER_SECOND,
                int,
                value=60,
                type=ParameterDictType.INT,
                visibility=ParameterDictVisibility.IMMUTABLE,
                display_name="Records Per Second",
                description="Number of records to process per second")
        )

        self._param_dict.add_parameter(
            Parameter(
                DriverParameter.PUBLISHER_POLLING_INTERVAL,
                float,
                value=1,
                type=ParameterDictType.FLOAT,
                visibility=ParameterDictVisibility.IMMUTABLE,
                display_name="Harvester Polling Interval",
                description="Duration in minutes to wait before checking for new files.")
        )

        self._param_dict.add_parameter(
            Parameter(
                DriverParameter.BATCHED_PARTICLE_COUNT,
                int,
                value=1,
                type=ParameterDictType.INT,
                visibility=ParameterDictVisibility.IMMUTABLE,
                display_name="Batched Particle Count",
                description="Number of particles to batch before sending to the agent")
        )

        config = self._config.get(DataSourceConfigKey.DRIVER, {})
        log.debug("set_resource on startup with: %s", config)
        self.set_resource(config)

    def _start_publisher_thread(self):
        self._publisher_thread = gevent.spawn(self._publisher_loop)
        self._publisher_shutdown = False

    def _stop_publisher_thread(self):
        log.debug("Signal shutdown")
        self._publisher_shutdown = True
        if self._publisher_thread:
            self._publisher_thread.kill(block=False)
        log.debug("shutdown complete")

    def _publisher_loop(self):
        """
        Main loop to listen for new files to parse.  Parse them and move on.
        """
        log.info("Starting main publishing loop")

        try:
            while(not self._publisher_shutdown):
                self._poll()
                gevent.sleep(self._polling_interval)
        except Exception as e:
            log.error("Exception in publisher thread (resource id: %s): %s", self._resource_id, traceback.format_exc(e))
            self._exception_callback(e)

        log.debug("publisher thread detected shutdown request")

    def _poll(self):
        raise NotImplementedException('virtual methond needs to be specialized')

    def _new_file_exception(self):
        raise NotImplementedException('virtual methond needs to be specialized')

    def _sample_exception_callback(self, exception):
        """
        Publish an event when a sample exception is detected
        """
        self._event_callback(event_type="ResourceAgentErrorEvent", error_msg = "%s" % exception)


    def _raise_new_file_event(self, name):
        """
        Raise a ResourceAgentIOEvent when a new file is detected.  Add file stats
        to the payload of the event.
        """
        s = os.stat(name)
        checksum = ""
        with open(name, 'rb') as filehandle:
            checksum = hashlib.md5(filehandle.read()).hexdigest()

        stats = {
            'name': name,
            'size': s.st_size,
            'mod': s.st_mtime,
            'md5_checksum': checksum
        }

        self._event_callback(event_type="ResourceAgentIOEvent", source_type="new file", stats=stats)
class DataSetDriver(object):
    """
    Base class for data set drivers.  Provides:
    - an interface via callback to publish data
    - an interface via callback to persist driver state
    - an interface via callback to handle exceptions
    - an start and stop sampling
    - a client interface for execute resource

    Subclasses need to include harvesters and parsers and
    be specialized to handle the interaction between the two.
    
    Configurations should contain keys from the DataSetDriverConfigKey class
    and should look something like this example (more full documentation in the
    "Dataset Agent Architecture" page on the OOI wiki):
    {
        'harvester':
        {
            'directory': '/tmp/dsatest',
            'pattern': '*.txt',
            'frequency': 1,
        },
        'parser': {}
        'driver': {
            'records_per_second'
            'harvester_polling_interval'
            'batched_particle_count'
        }
    }
    """
    def __init__(self, config, memento, data_callback, state_callback, exception_callback):
        self._config = config
        self._data_callback = data_callback
        self._state_callback = state_callback
        self._exception_callback = exception_callback
        self._memento = memento

        self._verify_config()
        self._param_dict = ProtocolParameterDict()

        # Updated my set_resource, defaults defined in build_param_dict
        self._polling_interval = None
        self._generate_particle_count = None
        self._particle_count_per_second = None

        self._build_param_dict()

    def start_sampling(self):
        """
        Start a new thread to monitor for data
        """
        self._start_sampling()
        self._start_publisher_thread()

    def stop_sampling(self):
        """
        Stop the sampling thread
        """
        self._stop_publisher_thread()

    def _start_sampling(self):
        raise NotImplementedException('virtual methond needs to be specialized')

    def _stop_sampling(self):
        raise NotImplementedException('virtual methond needs to be specialized')

    def cmd_dvr(self, cmd, *args, **kwargs):
        log.warn("DRIVER: cmd_dvr %s", cmd)

        if not cmd in ['execute_resource', 'get_resource_capabilities', 'set_resource', 'get_resource']:
            log.error("Unhandled command: %s", cmd)
            raise InstrumentStateException("Unhandled command: %s" % cmd)

        resource_cmd = args[0]

        if cmd == 'execute_resource':
            if resource_cmd == DriverEvent.START_AUTOSAMPLE:
                try:
                    log.debug("start autosample")
                    self.start_sampling()
                except:
                    log.error("Failed to start sampling", exc_info=True)
                    raise

                return (ResourceAgentState.STREAMING, None)

            elif resource_cmd == DriverEvent.STOP_AUTOSAMPLE:
                log.debug("stop autosample")
                self.stop_sampling()
                return (ResourceAgentState.COMMAND, None)

            else:
                log.error("Unhandled resource command: %s", resource_cmd)
                raise

        elif cmd == 'get_resource_capabilities':
            return self.get_resource_capabilities()

        elif cmd == 'set_resource':
            return self.set_resource(*args, **kwargs)

        elif cmd == 'get_resource':
            return self.get_resource(*args, **kwargs)

    def get_resource_capabilities(self, current_state=True, *args, **kwargs):
        """
        Return driver commands and parameters.
        @param current_state True to retrieve commands available in current
        state, otherwise reutrn all commands.
        @retval list of AgentCapability objects representing the drivers
        capabilities.
        @raises NotImplementedException if not implemented by subclass.
        """
        res_params = self._param_dict.get_keys()
        return [[], res_params]

    def set_resource(self, *args, **kwargs):
        """
        Set the driver parameter
        """
        log.trace("start set_resource")
        try:
            params = args[0]
        except IndexError:
            raise InstrumentParameterException('Set command requires a parameter dict.')

        log.trace("set_resource: iterate through params: %s", params)
        for (key, val) in params.iteritems():
            if key in [DriverParameter.BATCHED_PARTICLE_COUNT, DriverParameter.RECORDS_PER_SECOND]:
                if not isinstance(val, int): raise InstrumentParameterException("%s must be an integer" % key)
            if key in [DriverParameter.HARVESTER_POLLING_INTERVAL]:
                if not isinstance(val, (int, float)): raise InstrumentParameterException("%s must be an float" % key)

            if val <= 0:
                raise InstrumentParameterException("%s must be > 0" % key)

            self._param_dict.set_value(key, val)

        # Set the driver parameters
        self._generate_particle_count = self._param_dict.get(DriverParameter.BATCHED_PARTICLE_COUNT)
        self._particle_count_per_second = self._param_dict.get(DriverParameter.RECORDS_PER_SECOND)
        self._polling_interval = self._param_dict.get(DriverParameter.HARVESTER_POLLING_INTERVAL)
        log.trace("Driver Parameters: %s, %s, %s", self._polling_interval, self._particle_count_per_second, self._generate_particle_count)

    def get_resource(self, *args, **kwargs):
        """
        Get driver parameter
        """
        result = {}

        try:
            params = args[0]
        except IndexError:
            raise InstrumentParameterException('Set command requires a parameter list.')

        # If all params requested, retrieve config.
        if params == DriverParameter.ALL:
            result = self._param_dict.get_config()

        # If not all params, confirm a list or tuple of params to retrieve.
        # Raise if not a list or tuple.
        # Retrieve each key in the list, raise if any are invalid.
        else:
            if not isinstance(params, (list, tuple)):
                raise InstrumentParameterException('Get argument not a list or tuple.')
            result = {}
            for key in params:
                try:
                    val = self._param_dict.get(key)
                    result[key] = val

                except KeyError:
                    raise InstrumentParameterException(('%s is not a valid parameter.' % key))

        return result

    def _verify_config(self):
        """
        virtual method to verify the supplied driver configuration is value.  Must
        be overloaded in sub classes.

        raises an ConfigurationException when a configuration error is detected.
        """
        raise NotImplementedException('virtual methond needs to be specialized')

    def _build_param_dict(self):
        """
        Setup three common driver parameters
        """
        self._param_dict.add_parameter(
            Parameter(
                DriverParameter.RECORDS_PER_SECOND,
                int,
                value=60,
                type=ParameterDictType.INT,
                display_name="Records Per Second",
                description="Number of records to process per second")
        )

        self._param_dict.add_parameter(
            Parameter(
                DriverParameter.HARVESTER_POLLING_INTERVAL,
                float,
                value=1,
                type=ParameterDictType.FLOAT,
                display_name="Harvester Polling Interval",
                description="Duration in minutes to wait before checking for new files.")
        )

        self._param_dict.add_parameter(
            Parameter(
                DriverParameter.BATCHED_PARTICLE_COUNT,
                int,
                value=1,
                type=ParameterDictType.INT,
                display_name="Batched Particle Count",
                description="Number of particles to batch before sending to the agent")
        )

        config = self._config.get(DataSourceConfigKey.DRIVER, {})
        log.debug("set_resource on startup with: %s", config)
        self.set_resource(config)

    def _start_publisher_thread(self):
        self._publisher_thread = gevent.spawn(self._poll)

    def _stop_publisher_thread(self):
        self._publisher_thread.kill()

    def _poll(self):
        raise NotImplementedException('virtual methond needs to be specialized')

    def _new_file_exception(self):
        raise NotImplementedException('virtual methond needs to be specialized')