def setUp(self):
self.driver_dict = DriverDict()
self.driver_dict.add(DriverDictKey.VENDOR_SW_COMPATIBLE, True)
self.target_dict = {DriverDictKey.VENDOR_SW_COMPATIBLE:True}
self.target_schema = '{"vendor_sw_compatible": true}'
def __init__(self, config, memento, data_callback, state_callback, event_callback, exception_callback):
self._config = copy.deepcopy(config)
self._data_callback = data_callback
self._state_callback = state_callback
self._event_callback = event_callback
self._exception_callback = exception_callback
self._memento = memento
self._publisher_thread = None
self._verify_config()
# Updated my set_resource, defaults defined in build_param_dict
self._polling_interval = None
self._generate_particle_count = None
self._particle_count_per_second = None
self._resource_id = None
self._param_dict = ProtocolParameterDict()
self._cmd_dict = ProtocolCommandDict()
self._driver_dict = DriverDict()
self._build_command_dict()
self._build_driver_dict()
self._build_param_dict()
def __init__(self, config, memento, data_callback, state_callback, exception_callback):
self._config = config
self._data_callback = data_callback
self._state_callback = state_callback
self._exception_callback = exception_callback
self._memento = memento
self._publisher_thread = None
self._verify_config()
# Updated my set_resource, defaults defined in build_param_dict
self._polling_interval = None
self._generate_particle_count = None
self._particle_count_per_second = None
self._param_dict = ProtocolParameterDict()
self._cmd_dict = ProtocolCommandDict()
self._driver_dict = DriverDict()
self._build_command_dict()
self._build_driver_dict()
self._build_param_dict()
def _build_driver_dict(self):
"""
Build a driver dictionary structure, load the strings for the metadata from a file if present.
"""
self._driver_dict = DriverDict()
self._driver_dict.add(DriverDictKey.VENDOR_SW_COMPATIBLE, True)
class SatlanticPARInstrumentProtocol(CommandResponseInstrumentProtocol):
"""The instrument protocol classes to deal with a Satlantic PAR sensor.
The protocol is a very simple command/response protocol with a few show
commands and a few set commands.
Note protocol state machine must be called "self._protocol_fsm"
"""
__metaclass__ = get_logging_metaclass(log_level='debug')
def __init__(self, callback=None):
CommandResponseInstrumentProtocol.__init__(self, Prompt, EOLN, callback)
self._protocol_fsm = InstrumentFSM(PARProtocolState, PARProtocolEvent, PARProtocolEvent.ENTER, PARProtocolEvent.EXIT)
self._protocol_fsm.add_handler(PARProtocolState.UNKNOWN, PARProtocolEvent.ENTER, self._handler_unknown_enter)
self._protocol_fsm.add_handler(PARProtocolState.UNKNOWN, PARProtocolEvent.DISCOVER, self._handler_unknown_discover)
self._protocol_fsm.add_handler(PARProtocolState.COMMAND, PARProtocolEvent.ENTER, self._handler_command_enter)
self._protocol_fsm.add_handler(PARProtocolState.COMMAND, PARProtocolEvent.GET, self._handler_get)
self._protocol_fsm.add_handler(PARProtocolState.COMMAND, PARProtocolEvent.SET, self._handler_command_set)
self._protocol_fsm.add_handler(PARProtocolState.COMMAND, PARProtocolEvent.ACQUIRE_SAMPLE, self._handler_poll_acquire_sample)
self._protocol_fsm.add_handler(PARProtocolState.COMMAND, PARProtocolEvent.ACQUIRE_STATUS, self._handler_acquire_status)
self._protocol_fsm.add_handler(PARProtocolState.COMMAND, PARProtocolEvent.SCHEDULED_ACQUIRE_STATUS, self._handler_acquire_status)
self._protocol_fsm.add_handler(PARProtocolState.COMMAND, PARProtocolEvent.START_AUTOSAMPLE, self._handler_command_start_autosample)
self._protocol_fsm.add_handler(PARProtocolState.COMMAND, PARProtocolEvent.START_DIRECT, self._handler_command_start_direct)
self._protocol_fsm.add_handler(PARProtocolState.AUTOSAMPLE, PARProtocolEvent.ENTER, self._handler_autosample_enter)
self._protocol_fsm.add_handler(PARProtocolState.AUTOSAMPLE, PARProtocolEvent.STOP_AUTOSAMPLE, self._handler_autosample_stop_autosample)
self._protocol_fsm.add_handler(PARProtocolState.AUTOSAMPLE, PARProtocolEvent.SCHEDULED_ACQUIRE_STATUS, self._handler_autosample_acquire_status)
self._protocol_fsm.add_handler(PARProtocolState.DIRECT_ACCESS, PARProtocolEvent.ENTER, self._handler_direct_access_enter)
self._protocol_fsm.add_handler(PARProtocolState.DIRECT_ACCESS, PARProtocolEvent.EXECUTE_DIRECT, self._handler_direct_access_execute_direct)
self._protocol_fsm.add_handler(PARProtocolState.DIRECT_ACCESS, PARProtocolEvent.STOP_DIRECT, self._handler_direct_access_stop_direct)
self._protocol_fsm.start(PARProtocolState.UNKNOWN)
self._add_response_handler(Command.GET, self._parse_get_response)
self._add_response_handler(Command.SET, self._parse_set_response)
self._add_response_handler(Command.SAMPLE, self._parse_response)
# Construct the parameter dictionary containing device parameters,
# current parameter values, and set formatting functions.
self._build_cmd_dict()
self._build_driver_dict()
self._param_dict.add(Parameter.MAXRATE,
MAXRATE_PATTERN,
lambda match: float(match.group(1)),
self._float_or_int_to_string,
direct_access=True,
startup_param=True,
init_value=4,
display_name='Max Rate',
description='Maximum sampling rate (0 (Auto) | 0.125 | 0.5 | 1 | 2 | 4 | 8 | 10 | 12)',
type=ParameterDictType.FLOAT,
units=Units.HERTZ,
visibility=ParameterDictVisibility.READ_WRITE)
self._param_dict.add(Parameter.INSTRUMENT,
HEADER_PATTERN,
lambda match: match.group(1),
str,
visibility=ParameterDictVisibility.IMMUTABLE,
display_name='Instrument Type',
description="",
type=ParameterDictType.STRING,
startup_param=True)
self._param_dict.add(Parameter.SERIAL,
HEADER_PATTERN,
lambda match: match.group(1),
str,
visibility=ParameterDictVisibility.IMMUTABLE,
display_name='Serial Number',
description="",
type=ParameterDictType.STRING,
startup_param=True)
self._param_dict.add(Parameter.FIRMWARE,
HEADER_PATTERN,
lambda match: match.group(1),
str,
visibility=ParameterDictVisibility.IMMUTABLE,
display_name='Firmware Version',
description="",
type=ParameterDictType.STRING,
startup_param=True)
self._param_dict.add(Parameter.ACQUIRE_STATUS_INTERVAL,
INTERVAL_TIME_REGEX,
lambda match: match.group(1),
str,
display_name="Acquire Status Interval",
description='Interval for gathering status particles.',
type=ParameterDictType.STRING,
units=ParameterUnits.TIME_INTERVAL,
visibility=ParameterDictVisibility.READ_WRITE,
default_value='00:00:00',
startup_param=True)
self._chunker = StringChunker(SatlanticPARInstrumentProtocol.sieve_function)
def _build_cmd_dict(self):
"""
Build a command dictionary structure, load the strings for the metadata from a file if present.
"""
self._cmd_dict = ProtocolCommandDict()
self._cmd_dict.add(PARCapability.ACQUIRE_SAMPLE, display_name='Acquire Sample')
self._cmd_dict.add(PARCapability.ACQUIRE_STATUS, display_name='Acquire Status')
self._cmd_dict.add(PARCapability.START_AUTOSAMPLE, display_name='Start Autosample')
self._cmd_dict.add(PARCapability.STOP_AUTOSAMPLE, display_name='Stop Autosample')
self._cmd_dict.add(PARCapability.DISCOVER, display_name='Discover')
def _build_driver_dict(self):
"""
Build a driver dictionary structure, load the strings for the metadata from a file if present.
"""
self._driver_dict = DriverDict()
self._driver_dict.add(DriverDictKey.VENDOR_SW_COMPATIBLE, True)
@staticmethod
def sieve_function(raw_data):
"""
The method that splits samples
"""
matchers = [SAMPLE_REGEX, MAXANDBAUDRATE_REGEX]
return_list = []
for matcher in matchers:
for match in matcher.finditer(raw_data):
return_list.append((match.start(), match.end()))
log.trace("sieve_function: regex found %r", raw_data[match.start():match.end()])
return return_list
def _filter_capabilities(self, events):
"""
"""
events_out = [x for x in events if PARCapability.has(x)]
return events_out
def _do_cmd(self, cmd, *args, **kwargs):
"""
Issue a command to the instrument after clearing of buffers.
@param cmd The command to execute.
@param args positional arguments to pass to the build handler.
@retval The fully built command that was sent
@raises InstrumentProtocolException if command could not be built.
"""
expected_prompt = kwargs.get('expected_prompt', None)
cmd_line = self._build_default_command(cmd, *args)
# Send command.
log.debug('_do_cmd: %s, length=%s' % (repr(cmd_line), len(cmd_line)))
if len(cmd_line) == 1:
self._connection.send(cmd_line)
else:
for char in cmd_line:
starttime = time.time()
self._connection.send(char)
while len(self._promptbuf) == 0 or char not in self._promptbuf[-1]:
time.sleep(0.0015)
if time.time() > starttime + 3:
break
# Keep for reference: This is a reliable alternative, but not fully explained & may not work in the future.
# It somehow corrects bit rate timing issues across the driver-digi-instrument network interface,
# & allows the entire line of a commands to be sent successfully.
if EOLN not in cmd_line: # Note: Direct access commands may already include an EOLN
time.sleep(0.115)
starttime = time.time()
self._connection.send(EOLN)
while EOLN not in self._promptbuf[len(cmd_line):len(cmd_line) + 2] and Prompt.ENTER_EXIT_CMD_MODE \
not in self._promptbuf[len(cmd_line):len(cmd_line) + 2]:
time.sleep(0.0015)
if time.time() > starttime + 3:
break
# Limit resend_check_value from expected_prompt to one of the two below
resend_check_value = None
if expected_prompt is not None:
for check in (Prompt.COMMAND, Prompt.SAMPLES):
if check in expected_prompt:
log.trace('_do_cmd: command: %s, check=%s' % (cmd_line, check))
resend_check_value = check
# Resend the EOLN if it did not go through the first time
starttime = time.time()
if resend_check_value is not None:
while True:
time.sleep(0.1)
if time.time() > starttime + 2:
log.debug("Sending eoln again.")
self._connection.send(EOLN)
starttime = time.time()
if resend_check_value in self._promptbuf:
break
if PARProtocolError.INVALID_COMMAND in self._promptbuf:
break
return cmd_line
def _do_cmd_no_resp(self, cmd, *args, **kwargs):
"""
Issue a command to the instrument after clearing of buffers. No response is handled as a result of the command.
Overridden: special "write delay" & command resending
reliability improvements, no need for wakeup, default build command used for all commands
@param cmd The command to execute.
@param args positional arguments to pass to the build handler.
@raises InstrumentProtocolException if command could not be built.
"""
self._do_cmd(cmd, *args, **kwargs)
def _do_cmd_resp(self, cmd, *args, **kwargs):
"""
Perform a command-response on the device. Overridden: special "write delay" & command resending
reliability improvements, no need for wakeup, default build command used for all commands
@param cmd The command to execute.
@param args positional arguments to pass to the build handler.
@param expected_prompt kwarg offering a specific prompt to look for
other than the ones in the protocol class itself.
@param response_regex kwarg with a compiled regex for the response to
match. Groups that match will be returned as a string.
Cannot be supplied with expected_prompt. May be helpful for instruments that do not have a prompt.
@retval resp_result The (possibly parsed) response result including the
first instance of the prompt matched. If a regex was used, the prompt
will be an empty string and the response will be the joined collection of matched groups.
@raises InstrumentTimeoutException if the response did not occur in time.
@raises InstrumentProtocolException if command could not be built or if response was not recognized.
"""
timeout = kwargs.get('timeout', DEFAULT_CMD_TIMEOUT)
expected_prompt = kwargs.get('expected_prompt', None)
response_regex = kwargs.get('response_regex', None)
if response_regex and not isinstance(response_regex, RE_PATTERN):
raise InstrumentProtocolException('Response regex is not a compiled pattern!')
if expected_prompt and response_regex:
raise InstrumentProtocolException('Cannot supply both regex and expected prompt!')
retry_count = 5
retry_num = 0
cmd_line = ""
result = ""
prompt = ""
for retry_num in xrange(retry_count):
# Clear line and prompt buffers for result.
self._linebuf = ''
self._promptbuf = ''
cmd_line = self._do_cmd(cmd, *args, **kwargs)
# Wait for the prompt, prepare result and return, timeout exception
if response_regex:
result_tuple = self._get_response(timeout, response_regex=response_regex,
expected_prompt=expected_prompt)
result = "".join(result_tuple)
else:
(prompt, result) = self._get_response(timeout, expected_prompt=expected_prompt)
# Confirm the entire command was sent, otherwise resend retry_count number of times
if len(cmd_line) > 1 and \
(expected_prompt is not None or
(response_regex is not None))\
and not result.startswith(cmd_line):
# and cmd_line not in result:
log.debug("_do_cmd_resp: Send command: %s failed %s attempt, result = %s.", cmd, retry_num, result)
if retry_num >= retry_count:
raise InstrumentCommandException('_do_cmd_resp: Failed %s attempts sending command: %s' %
(retry_count, cmd))
else:
break
log.debug("_do_cmd_resp: Sent command: %s, %s reattempts, expected_prompt=%s, result=%s.",
cmd_line, retry_num, expected_prompt, result)
resp_handler = self._response_handlers.get((self.get_current_state(), cmd), None) or \
self._response_handlers.get(cmd, None)
resp_result = None
if resp_handler:
resp_result = resp_handler(result, prompt)
time.sleep(0.3) # give some time for the instrument connection to keep up
return resp_result
########################################################################
# Unknown handlers.
########################################################################
def _handler_unknown_enter(self):
"""
Enter unknown state.
"""
self._driver_event(DriverAsyncEvent.STATE_CHANGE)
def _handler_unknown_discover(self):
"""
Discover current state; can be COMMAND or AUTOSAMPLE.
@retval (next_state, result), (PARProtocolState.COMMAND or PARProtocolState.AUTOSAMPLE, None).
"""
try:
probe_resp = self._do_cmd_resp(Command.SAMPLE, timeout=2,
expected_prompt=[Prompt.SAMPLES, PARProtocolError.INVALID_COMMAND])
except InstrumentTimeoutException:
self._do_cmd_resp(Command.SWITCH_TO_AUTOSAMPLE, expected_prompt=Prompt.SAMPLES, timeout=15)
return PARProtocolState.AUTOSAMPLE, ResourceAgentState.STREAMING
log.trace("_handler_unknown_discover: returned: %s", probe_resp)
if probe_resp == PARProtocolError.INVALID_COMMAND:
return PARProtocolState.COMMAND, ResourceAgentState.IDLE
else:
# Put the instrument into full autosample in case it isn't already (could be in polled mode)
self._do_cmd_resp(Command.SWITCH_TO_AUTOSAMPLE, expected_prompt=Prompt.SAMPLES, timeout=15)
return PARProtocolState.AUTOSAMPLE, ResourceAgentState.STREAMING
########################################################################
# Command handlers.
########################################################################
def _handler_command_enter(self):
"""
Enter command state.
"""
# Command device to update parameters and send a config change event.
if self._init_type != InitializationType.NONE:
self._update_params()
self._init_params()
self._driver_event(DriverAsyncEvent.STATE_CHANGE)
def _update_params(self):
"""
Fetch the parameters from the device, and update the param dict.
"""
max_rate_response = self._do_cmd_resp(Command.GET, Parameter.MAXRATE, expected_prompt=Prompt.COMMAND)
self._param_dict.update(max_rate_response)
def _set_params(self, params, startup=False, *args, **kwargs):
"""
Issue commands to the instrument to set various parameters
Also called when setting parameters during startup and direct access
Issue commands to the instrument to set various parameters. If
startup is set to true that means we are setting startup values
and immutable parameters can be set. Otherwise only READ_WRITE
parameters can be set.
@param params dictionary containing parameter name and value
@param startup bool True is we are initializing, False otherwise
@raise InstrumentParameterException
"""
# Retrieve required parameter from args.
# Raise exception if no parameter provided, or not a dict.
scheduling_interval_changed = False
instrument_params_changed = False
old_config = self._param_dict.get_all()
if not isinstance(params, dict):
raise InstrumentParameterException('Set params requires a parameter dict.')
self._verify_not_readonly(params, startup)
for name, value in params.iteritems():
old_val = self._param_dict.format(name)
new_val = self._param_dict.format(name, params[name])
log.debug('Changing param %r OLD = %r, NEW %r', name, old_val, new_val)
if name == Parameter.MAXRATE:
if value not in VALID_MAXRATES:
raise InstrumentParameterException("Maxrate %s out of range" % value)
if old_val != new_val:
if self._do_cmd_resp(Command.SET, name, new_val, expected_prompt=Prompt.COMMAND):
instrument_params_changed = True
elif name == Parameter.ACQUIRE_STATUS_INTERVAL:
if old_val != new_val:
self._param_dict.set_value(name, new_val)
scheduling_interval_changed = True
elif name in [Parameter.FIRMWARE, Parameter.INSTRUMENT, Parameter.SERIAL]:
self._param_dict.set_value(name, new_val)
else:
raise InstrumentParameterException("Parameter not in dictionary: %s" % name)
if instrument_params_changed:
self._do_cmd_resp(Command.SAVE, expected_prompt=Prompt.COMMAND)
self._update_params()
if scheduling_interval_changed and not startup:
self._setup_scheduler_config()
new_config = self._param_dict.get_all()
log.debug("Updated parameter dict: old_config = %s, new_config = %s", old_config, new_config)
if new_config != old_config:
self._driver_event(DriverAsyncEvent.CONFIG_CHANGE)
for name in params.keys():
if self._param_dict.format(name, params[name]) != self._param_dict.format(name):
raise InstrumentParameterException('Failed to update parameter: %s' % name)
def _handle_scheduling_params_changed(self):
"""
Required actions when scheduling parameters change
"""
self._setup_scheduler_config()
def _setup_scheduler_config(self):
"""
Set up auto scheduler configuration.
"""
interval = self._param_dict.format(Parameter.ACQUIRE_STATUS_INTERVAL).split(':')
hours = int(interval[0])
minutes = int(interval[1])
seconds = int(interval[2])
log.debug("Setting scheduled interval to: %s %s %s", hours, minutes, seconds)
if DriverConfigKey.SCHEDULER in self._startup_config:
self._startup_config[DriverConfigKey.SCHEDULER][ScheduledJob.ACQUIRE_STATUS] = {
DriverSchedulerConfigKey.TRIGGER: {
DriverSchedulerConfigKey.TRIGGER_TYPE: TriggerType.INTERVAL,
DriverSchedulerConfigKey.HOURS: int(hours),
DriverSchedulerConfigKey.MINUTES: int(minutes),
DriverSchedulerConfigKey.SECONDS: int(seconds)}
}
else:
self._startup_config[DriverConfigKey.SCHEDULER] = {
ScheduledJob.ACQUIRE_STATUS: {
DriverSchedulerConfigKey.TRIGGER: {
DriverSchedulerConfigKey.TRIGGER_TYPE: TriggerType.INTERVAL,
DriverSchedulerConfigKey.HOURS: int(hours),
DriverSchedulerConfigKey.MINUTES: int(minutes),
DriverSchedulerConfigKey.SECONDS: int(seconds)}
},
}
# Start the scheduler if it is not running
if not self._scheduler:
self.initialize_scheduler()
# First remove the scheduler, if it exists
if not self._scheduler_callback.get(ScheduledJob.ACQUIRE_STATUS) is None:
self._remove_scheduler(ScheduledJob.ACQUIRE_STATUS)
log.debug("Removed scheduler for acquire status")
# Now Add the scheduler
if hours > 0 or minutes > 0 or seconds > 0:
self._add_scheduler_event(ScheduledJob.ACQUIRE_STATUS, PARProtocolEvent.SCHEDULED_ACQUIRE_STATUS)
def _handler_command_set(self, *args, **kwargs):
"""
Handle setting data from command mode.
@param params Dict of the parameters and values to pass to the state
@retval return (next state, result)
"""
self._set_params(*args, **kwargs)
return None, None
def _handler_command_start_autosample(self):
"""
Handle getting a start autosample event when in command mode
@retval return (next state, result)
"""
self._do_cmd_resp(Command.EXIT, expected_prompt=Prompt.SAMPLES, timeout=15)
time.sleep(0.115)
self._do_cmd_resp(Command.SWITCH_TO_AUTOSAMPLE, expected_prompt=Prompt.SAMPLES, timeout=15)
return PARProtocolState.AUTOSAMPLE, (ResourceAgentState.STREAMING, None)
def _handler_command_start_direct(self):
"""
"""
return PARProtocolState.DIRECT_ACCESS, (ResourceAgentState.DIRECT_ACCESS, None)
########################################################################
# Autosample handlers.
########################################################################
def _handler_autosample_enter(self):
"""
Handle PARProtocolState.AUTOSAMPLE PARProtocolEvent.ENTER
@retval return (next state, result)
"""
if self._init_type != InitializationType.NONE:
self._handler_autosample_stop_autosample()
self._update_params()
self._handler_command_start_autosample()
self._init_params()
self._driver_event(DriverAsyncEvent.STATE_CHANGE)
return None, None
def _handler_autosample_stop_autosample(self):
"""
Handle PARProtocolState.AUTOSAMPLE stop
@retval return (next state, result)
@throw InstrumentProtocolException For hardware error
"""
try:
self._send_break()
except InstrumentException, e:
log.debug("_handler_autosample_stop_autosample error: %s", e)
raise InstrumentProtocolException(error_code=InstErrorCode.HARDWARE_ERROR,
msg="Couldn't break from autosample!")
return PARProtocolState.COMMAND, (ResourceAgentState.COMMAND, None)
class DataSetDriver(object):
"""
Base class for data set drivers. Provides:
- an interface via callback to publish data
- an interface via callback to persist driver state
- an interface via callback to handle exceptions
- an start and stop sampling
- a client interface for execute resource
Subclasses need to include harvesters and parsers and
be specialized to handle the interaction between the two.
Configurations should contain keys from the DataSetDriverConfigKey class
and should look something like this example (more full documentation in the
"Dataset Agent Architecture" page on the OOI wiki):
{
'harvester':
{
'directory': '/tmp/dsatest',
'pattern': '*.txt',
'frequency': 1,
},
'parser': {}
'driver': {
'records_per_second'
'harvester_polling_interval'
'batched_particle_count'
}
}
"""
def __init__(self, config, memento, data_callback, state_callback, exception_callback):
self._config = config
self._data_callback = data_callback
self._state_callback = state_callback
self._exception_callback = exception_callback
self._memento = memento
self._publisher_thread = None
self._verify_config()
# Updated my set_resource, defaults defined in build_param_dict
self._polling_interval = None
self._generate_particle_count = None
self._particle_count_per_second = None
self._param_dict = ProtocolParameterDict()
self._cmd_dict = ProtocolCommandDict()
self._driver_dict = DriverDict()
self._build_command_dict()
self._build_driver_dict()
self._build_param_dict()
def shutdown(self):
self.stop_sampling()
def start_sampling(self):
"""
Start a new thread to monitor for data
"""
self._start_sampling()
self._start_publisher_thread()
def stop_sampling(self):
"""
Stop the sampling thread
"""
log.debug("Stopping driver now")
self._stop_sampling()
self._stop_publisher_thread()
def _start_sampling(self):
raise NotImplementedException('virtual method needs to be specialized')
def _stop_sampling(self):
raise NotImplementedException('virtual method needs to be specialized')
def _is_sampling(self):
"""
Currently the drivers only have two states, command and streaming and
all resource commands are common, either start or stop autosample.
Therefore we didn't implement an enitre state machine to manage states
and commands. If it does get more complex than this we should take the
time to implement a state machine to add some flexibility
"""
raise NotImplementedException('virtual method needs to be specialized')
def cmd_dvr(self, cmd, *args, **kwargs):
log.warn("DRIVER: cmd_dvr %s", cmd)
if cmd == 'execute_resource':
resource_cmd = args[0]
if resource_cmd == DriverEvent.START_AUTOSAMPLE:
return (ResourceAgentState.STREAMING, None)
elif resource_cmd == DriverEvent.STOP_AUTOSAMPLE:
self.stop_sampling()
return (ResourceAgentState.COMMAND, None)
else:
log.error("Unhandled resource command: %s", resource_cmd)
raise
elif cmd == 'get_resource_capabilities':
return self.get_resource_capabilities()
elif cmd == 'set_resource':
return self.set_resource(*args, **kwargs)
elif cmd == 'get_resource':
return self.get_resource(*args, **kwargs)
elif cmd == 'get_config_metadata':
return self.get_config_metadata(*args, **kwargs)
elif cmd == 'disconnect':
pass
elif cmd == 'initialize':
pass
else:
log.error("Unhandled command: %s", cmd)
raise InstrumentStateException("Unhandled command: %s" % cmd)
def get_resource_capabilities(self, current_state=True, *args, **kwargs):
"""
Return driver commands and parameters.
@param current_state True to retrieve commands available in current
state, otherwise reutrn all commands.
@retval list of AgentCapability objects representing the drivers
capabilities.
@raises NotImplementedException if not implemented by subclass.
"""
res_params = self._param_dict.get_keys()
res_cmds = [DriverEvent.STOP_AUTOSAMPLE, DriverEvent.START_AUTOSAMPLE]
if current_state and self._is_sampling():
res_cmds = [DriverEvent.STOP_AUTOSAMPLE]
elif current_state and not self._is_sampling():
res_cmds = [DriverEvent.START_AUTOSAMPLE]
return [res_cmds, res_params]
def set_resource(self, *args, **kwargs):
"""
Set the driver parameter
"""
log.trace("start set_resource")
try:
params = args[0]
except IndexError:
raise InstrumentParameterException('Set command requires a parameter dict.')
log.trace("set_resource: iterate through params: %s", params)
for (key, val) in params.iteritems():
if key in [DriverParameter.BATCHED_PARTICLE_COUNT, DriverParameter.RECORDS_PER_SECOND]:
if not isinstance(val, int): raise InstrumentParameterException("%s must be an integer" % key)
if key in [DriverParameter.PUBLISHER_POLLING_INTERVAL]:
if not isinstance(val, (int, float)): raise InstrumentParameterException("%s must be an float" % key)
if val <= 0:
raise InstrumentParameterException("%s must be > 0" % key)
self._param_dict.set_value(key, val)
# Set the driver parameters
self._generate_particle_count = self._param_dict.get(DriverParameter.BATCHED_PARTICLE_COUNT)
self._particle_count_per_second = self._param_dict.get(DriverParameter.RECORDS_PER_SECOND)
self._polling_interval = self._param_dict.get(DriverParameter.PUBLISHER_POLLING_INTERVAL)
log.trace("Driver Parameters: %s, %s, %s", self._polling_interval, self._particle_count_per_second, self._generate_particle_count)
def get_resource(self, *args, **kwargs):
"""
Get driver parameter
"""
result = {}
try:
params = args[0]
except IndexError:
raise InstrumentParameterException('Set command requires a parameter list.')
# If all params requested, retrieve config.
if params == DriverParameter.ALL:
result = self._param_dict.get_config()
# If not all params, confirm a list or tuple of params to retrieve.
# Raise if not a list or tuple.
# Retrieve each key in the list, raise if any are invalid.
else:
if not isinstance(params, (list, tuple)):
raise InstrumentParameterException('Get argument not a list or tuple.')
result = {}
for key in params:
try:
val = self._param_dict.get(key)
result[key] = val
except KeyError:
raise InstrumentParameterException(('%s is not a valid parameter.' % key))
return result
def get_config_metadata(self):
"""
Return the configuration metadata object in JSON format
@retval The description of the parameters, commands, and driver info
in a JSON string
@see https://confluence.oceanobservatories.org/display/syseng/CIAD+MI+SV+Instrument+Driver-Agent+parameter+and+command+metadata+exchange
"""
log.debug("Getting metadata from driver...")
log.debug("Getting metadata dict from protocol...")
return_dict = {}
return_dict[ConfigMetadataKey.DRIVER] = self._driver_dict.generate_dict()
return_dict[ConfigMetadataKey.COMMANDS] = self._cmd_dict.generate_dict()
return_dict[ConfigMetadataKey.PARAMETERS] = self._param_dict.generate_dict()
return return_dict
def _verify_config(self):
"""
virtual method to verify the supplied driver configuration is value. Must
be overloaded in sub classes.
raises an ConfigurationException when a configuration error is detected.
"""
raise NotImplementedException('virtual methond needs to be specialized')
def _build_driver_dict(self):
"""
Populate the driver dictionary with options
"""
pass
def _build_command_dict(self):
"""
Populate the command dictionary with command.
"""
self._cmd_dict.add(DriverEvent.START_AUTOSAMPLE, display_name="start autosample")
self._cmd_dict.add(DriverEvent.STOP_AUTOSAMPLE, display_name="stop autosample")
def _build_param_dict(self):
"""
Setup three common driver parameters
"""
self._param_dict.add_parameter(
Parameter(
DriverParameter.RECORDS_PER_SECOND,
int,
value=60,
type=ParameterDictType.INT,
visibility=ParameterDictVisibility.IMMUTABLE,
display_name="Records Per Second",
description="Number of records to process per second")
)
self._param_dict.add_parameter(
Parameter(
DriverParameter.PUBLISHER_POLLING_INTERVAL,
float,
value=1,
type=ParameterDictType.FLOAT,
visibility=ParameterDictVisibility.IMMUTABLE,
display_name="Harvester Polling Interval",
description="Duration in minutes to wait before checking for new files.")
)
self._param_dict.add_parameter(
Parameter(
DriverParameter.BATCHED_PARTICLE_COUNT,
int,
value=1,
type=ParameterDictType.INT,
visibility=ParameterDictVisibility.IMMUTABLE,
display_name="Batched Particle Count",
description="Number of particles to batch before sending to the agent")
)
config = self._config.get(DataSourceConfigKey.DRIVER, {})
log.debug("set_resource on startup with: %s", config)
self.set_resource(config)
def _start_publisher_thread(self):
self._publisher_thread = gevent.spawn(self._publisher_loop)
self._publisher_shutdown = False
def _stop_publisher_thread(self):
log.debug("Signal shutdown")
self._publisher_shutdown = True
if self._publisher_thread:
self._publisher_thread.kill(block=False)
log.debug("shutdown complete")
def _publisher_loop(self):
"""
Main loop to listen for new files to parse. Parse them and move on.
"""
log.info("Starting main publishing loop")
try:
while(not self._publisher_shutdown):
self._poll()
gevent.sleep(self._polling_interval)
except Exception as e:
log.error("Exception in publisher thread: %s", e)
self._exception_callback(e)
log.debug("publisher thread detected shutdown request")
def _poll(self):
raise NotImplementedException('virtual methond needs to be specialized')
def _new_file_exception(self):
raise NotImplementedException('virtual methond needs to be specialized')
def test_empty_schema(self):
self.driver_dict = DriverDict()
result = self.driver_dict.generate_dict()
self.assertEqual(result, {})
class TestUnitProtocolCommandDict(MiUnitTestCase):
"""
Test cases for instrument driver class. Functions in this class provide
instrument driver unit tests and provide a tutorial on use of
the driver interface.
"""
def setUp(self):
self.driver_dict = DriverDict()
self.driver_dict.add(DriverDictKey.VENDOR_SW_COMPATIBLE, True)
self.target_dict = {DriverDictKey.VENDOR_SW_COMPATIBLE:True}
self.target_schema = '{"vendor_sw_compatible": true}'
def test_schema_generation(self):
result = self.driver_dict.generate_dict()
self.assertEqual(result, self.target_dict)
self.assertEqual(json.dumps(result), self.target_schema)
def test_empty_schema(self):
self.driver_dict = DriverDict()
result = self.driver_dict.generate_dict()
self.assertEqual(result, {})
def test_add_get(self):
self.assertEquals(self.driver_dict.get_value("vendor_sw_compatible"),
True)
self.driver_dict.add("good_val", 12)
self.assertEquals(self.driver_dict.get_value("good_val"), 12)
self.driver_dict.add("good_val", 21)
self.assertEquals(self.driver_dict.get_value("good_val"), 21)
self.driver_dict.add("good_val")
self.assertEquals(self.driver_dict.get_value("good_val"), None)
self.assertRaises(KeyError,
self.driver_dict.get_value, "missing_val")
self.assertRaises(KeyError,
self.driver_dict.get_value, None)
class DataSetDriver(object):
"""
Base class for data set drivers. Provides:
- an interface via callback to publish data
- an interface via callback to persist driver state
- an interface via callback to handle exceptions
- an start and stop sampling
- a client interface for execute resource
Subclasses need to include harvesters and parsers and
be specialized to handle the interaction between the two.
Configurations should contain keys from the DataSetDriverConfigKey class
and should look something like this example (more full documentation in the
"Dataset Agent Architecture" page on the OOI wiki):
{
'harvester':
{
'directory': '/tmp/dsatest',
'storage_directory': '/tmp/stored_dsatest',
'pattern': '*.txt',
'frequency': 1,
'file_mod_wait_time': 30,
},
'parser': {}
'driver': {
'records_per_second'
'harvester_polling_interval'
'batched_particle_count'
}
}
"""
def __init__(self, config, memento, data_callback, state_callback, event_callback, exception_callback):
self._config = copy.deepcopy(config)
self._data_callback = data_callback
self._state_callback = state_callback
self._event_callback = event_callback
self._exception_callback = exception_callback
self._memento = memento
self._publisher_thread = None
self._verify_config()
# Updated my set_resource, defaults defined in build_param_dict
self._polling_interval = None
self._generate_particle_count = None
self._particle_count_per_second = None
self._resource_id = None
self._param_dict = ProtocolParameterDict()
self._cmd_dict = ProtocolCommandDict()
self._driver_dict = DriverDict()
self._build_command_dict()
self._build_driver_dict()
self._build_param_dict()
def shutdown(self):
self.stop_sampling()
def start_sampling(self):
"""
Start a new thread to monitor for data
"""
self._start_sampling()
self._start_publisher_thread()
def stop_sampling(self):
"""
Stop the sampling thread
"""
log.debug("Stopping sampling and publisher now")
self._stop_sampling()
self._stop_publisher_thread()
def _start_sampling(self):
raise NotImplementedException('virtual method needs to be specialized')
def _stop_sampling(self):
raise NotImplementedException('virtual method needs to be specialized')
def _is_sampling(self):
"""
Currently the drivers only have two states, command and streaming and
all resource commands are common, either start or stop autosample.
Therefore we didn't implement an enitre state machine to manage states
and commands. If it does get more complex than this we should take the
time to implement a state machine to add some flexibility
"""
raise NotImplementedException('virtual method needs to be specialized')
def cmd_dvr(self, cmd, *args, **kwargs):
log.warn("DRIVER: cmd_dvr %s", cmd)
if cmd == 'execute_resource':
resource_cmd = args[0]
if resource_cmd == DriverEvent.START_AUTOSAMPLE:
return (ResourceAgentState.STREAMING, None)
elif resource_cmd == DriverEvent.STOP_AUTOSAMPLE:
self.stop_sampling()
return (ResourceAgentState.COMMAND, None)
else:
log.error("Unhandled resource command: %s", resource_cmd)
raise
elif cmd == 'get_resource_capabilities':
return self.get_resource_capabilities()
elif cmd == 'set_resource':
return self.set_resource(*args, **kwargs)
elif cmd == 'get_resource':
return self.get_resource(*args, **kwargs)
elif cmd == 'get_config_metadata':
return self.get_config_metadata(*args, **kwargs)
elif cmd == 'disconnect':
pass
elif cmd == 'initialize':
pass
else:
log.error("Unhandled command: %s", cmd)
raise InstrumentStateException("Unhandled command: %s" % cmd)
def get_resource_capabilities(self, current_state=True, *args, **kwargs):
"""
Return driver commands and parameters.
@param current_state True to retrieve commands available in current
state, otherwise reutrn all commands.
@retval list of AgentCapability objects representing the drivers
capabilities.
@raises NotImplementedException if not implemented by subclass.
"""
res_params = self._param_dict.get_keys()
res_cmds = [DriverEvent.STOP_AUTOSAMPLE, DriverEvent.START_AUTOSAMPLE]
if current_state and self._is_sampling():
res_cmds = [DriverEvent.STOP_AUTOSAMPLE]
elif current_state and not self._is_sampling():
res_cmds = [DriverEvent.START_AUTOSAMPLE]
return [res_cmds, res_params]
def set_resource(self, *args, **kwargs):
"""
Set the driver parameter
"""
log.trace("start set_resource")
try:
params = args[0]
except IndexError:
raise InstrumentParameterException('Set command requires a parameter dict.')
log.trace("set_resource: iterate through params: %s", params)
for (key, val) in params.iteritems():
if key in [DriverParameter.BATCHED_PARTICLE_COUNT, DriverParameter.RECORDS_PER_SECOND]:
if not isinstance(val, int): raise InstrumentParameterException("%s must be an integer" % key)
if key in [DriverParameter.PUBLISHER_POLLING_INTERVAL]:
if not isinstance(val, (int, float)): raise InstrumentParameterException("%s must be an float" % key)
if val <= 0:
raise InstrumentParameterException("%s must be > 0" % key)
self._param_dict.set_value(key, val)
# Set the driver parameters
self._generate_particle_count = self._param_dict.get(DriverParameter.BATCHED_PARTICLE_COUNT)
self._particle_count_per_second = self._param_dict.get(DriverParameter.RECORDS_PER_SECOND)
self._polling_interval = self._param_dict.get(DriverParameter.PUBLISHER_POLLING_INTERVAL)
log.trace("Driver Parameters: %s, %s, %s", self._polling_interval, self._particle_count_per_second,
self._generate_particle_count)
def get_resource(self, *args, **kwargs):
"""
Get driver parameter
"""
result = {}
try:
params = args[0]
except IndexError:
raise InstrumentParameterException('Set command requires a parameter list.')
# If all params requested, retrieve config.
if params == DriverParameter.ALL:
result = self._param_dict.get_config()
# If not all params, confirm a list or tuple of params to retrieve.
# Raise if not a list or tuple.
# Retrieve each key in the list, raise if any are invalid.
else:
if not isinstance(params, (list, tuple)):
raise InstrumentParameterException('Get argument not a list or tuple.')
result = {}
for key in params:
try:
val = self._param_dict.get(key)
result[key] = val
except KeyError:
raise InstrumentParameterException(('%s is not a valid parameter.' % key))
return result
def get_config_metadata(self):
"""
Return the configuration metadata object in JSON format
@retval The description of the parameters, commands, and driver info
in a JSON string
@see https://confluence.oceanobservatories.org/display/syseng/CIAD+MI+SV+Instrument+Driver-Agent+parameter+and+command+metadata+exchange
"""
log.debug("Getting metadata from driver...")
log.debug("Getting metadata dict from protocol...")
return_dict = {}
return_dict[ConfigMetadataKey.DRIVER] = self._driver_dict.generate_dict()
return_dict[ConfigMetadataKey.COMMANDS] = self._cmd_dict.generate_dict()
return_dict[ConfigMetadataKey.PARAMETERS] = self._param_dict.generate_dict()
return return_dict
def _verify_config(self):
"""
virtual method to verify the supplied driver configuration is value. Must
be overloaded in sub classes.
raises an ConfigurationException when a configuration error is detected.
"""
raise NotImplementedException('virtual methond needs to be specialized')
def _build_driver_dict(self):
"""
Populate the driver dictionary with options
"""
pass
def _build_command_dict(self):
"""
Populate the command dictionary with command.
"""
self._cmd_dict.add(DriverEvent.START_AUTOSAMPLE, display_name="start autosample")
self._cmd_dict.add(DriverEvent.STOP_AUTOSAMPLE, display_name="stop autosample")
def _build_param_dict(self):
"""
Setup three common driver parameters
"""
self._param_dict.add_parameter(
Parameter(
DriverParameter.RECORDS_PER_SECOND,
int,
value=60,
type=ParameterDictType.INT,
visibility=ParameterDictVisibility.IMMUTABLE,
display_name="Records Per Second",
description="Number of records to process per second")
)
self._param_dict.add_parameter(
Parameter(
DriverParameter.PUBLISHER_POLLING_INTERVAL,
float,
value=1,
type=ParameterDictType.FLOAT,
visibility=ParameterDictVisibility.IMMUTABLE,
display_name="Harvester Polling Interval",
description="Duration in minutes to wait before checking for new files.")
)
self._param_dict.add_parameter(
Parameter(
DriverParameter.BATCHED_PARTICLE_COUNT,
int,
value=1,
type=ParameterDictType.INT,
visibility=ParameterDictVisibility.IMMUTABLE,
display_name="Batched Particle Count",
description="Number of particles to batch before sending to the agent")
)
config = self._config.get(DataSourceConfigKey.DRIVER, {})
log.debug("set_resource on startup with: %s", config)
self.set_resource(config)
def _start_publisher_thread(self):
self._publisher_thread = gevent.spawn(self._publisher_loop)
self._publisher_shutdown = False
def _stop_publisher_thread(self):
log.debug("Signal shutdown")
self._publisher_shutdown = True
if self._publisher_thread:
self._publisher_thread.kill(block=False)
log.debug("shutdown complete")
def _publisher_loop(self):
"""
Main loop to listen for new files to parse. Parse them and move on.
"""
log.info("Starting main publishing loop")
try:
while(not self._publisher_shutdown):
self._poll()
gevent.sleep(self._polling_interval)
except Exception as e:
log.error("Exception in publisher thread (resource id: %s): %s", self._resource_id, traceback.format_exc(e))
self._exception_callback(e)
log.debug("publisher thread detected shutdown request")
def _poll(self):
raise NotImplementedException('virtual methond needs to be specialized')
def _new_file_exception(self):
raise NotImplementedException('virtual methond needs to be specialized')
def _sample_exception_callback(self, exception):
"""
Publish an event when a sample exception is detected
"""
self._event_callback(event_type="ResourceAgentErrorEvent", error_msg = "%s" % exception)
def _raise_new_file_event(self, name):
"""
Raise a ResourceAgentIOEvent when a new file is detected. Add file stats
to the payload of the event.
"""
s = os.stat(name)
checksum = ""
with open(name, 'rb') as filehandle:
checksum = hashlib.md5(filehandle.read()).hexdigest()
stats = {
'name': name,
'size': s.st_size,
'mod': s.st_mtime,
'md5_checksum': checksum
}
self._event_callback(event_type="ResourceAgentIOEvent", source_type="new file", stats=stats)
