def _store_package_files(self):
"""
@brief Store all files in zip archive and add them to the manifest file
"""
# make sure metadata is up to date
self.metadata = Metadata(self.metadata.driver_make,
self.metadata.driver_model,
self.metadata.driver_name,
REPODIR + '/marine-integrations')
self.generator = DriverGenerator(self.metadata)
egg_generator = EggGenerator(self.metadata)
egg_file = egg_generator.save()
# Add egg
self._add_file(egg_file, 'egg', 'python driver egg package')
# Add the package metadata file
self._add_file(self.metadata.metadata_path(), description = 'package metadata')
# Add the qualification test log
self._add_file(self.log_path(), description = 'qualification tests results')
# Store parameter/command string description file
str_path = "%s/%s" % (self.generator.resource_dir(), self.string_file())
if os.path.exists(str_path):
self._add_file(str_path, 'resource', 'driver string file')
# Store additional resource files
self._store_resource_files()
# Finally save the manifest file. This must be last of course
self._add_file(self.manifest().manifest_path(), description = 'package manifest file')
def overwrite(self):
"""
@brief Overwrite the current files with what is stored in the current metadata file.
"""
self.metadata = Metadata()
config_path = "%s/%s" % (self.metadata.driver_dir(), CommConfig.config_filename())
self.comm_config = CommConfig.get_config_from_file(config_path)
self.generate_code(force = True)
class StartDriver():
"""
Main class for running the start driver process.
"""
def fetch_metadata(self):
"""
@brief collect metadata from the user
"""
self.metadata = Metadata()
self.metadata.get_from_console()
def fetch_comm_config(self):
"""
@brief collect connection information for the logger from the user
"""
config_path = "%s/%s" % (self.metadata.driver_dir(), CommConfig.config_filename())
self.comm_config = CommConfig.get_config_from_console(config_path)
self.comm_config.get_from_console()
def generate_code(self, force = False):
"""
@brief generate the directory structure, code and tests for the new driver.
"""
driver = DriverGenerator( self.metadata, force = force )
driver.generate()
def overwrite(self):
"""
@brief Overwrite the current files with what is stored in the current metadata file.
"""
self.metadata = Metadata()
config_path = "%s/%s" % (self.metadata.driver_dir(), CommConfig.config_filename())
self.comm_config = CommConfig.get_config_from_file(config_path)
self.generate_code(force = True)
def run(self):
"""
@brief Run it.
"""
print( "*** Starting Driver Creation Process***" )
self.fetch_metadata()
self.fetch_comm_config()
self.generate_code()
def fetch_metadata(self):
"""
@brief collect metadata from the user
"""
self.metadata = Metadata()
self.driver_make = self.metadata.driver_make
self.driver_model = self.metadata.driver_model
self.driver_name = self.metadata.driver_name
if not (self.driver_make and self.driver_model and self.driver_name):
self.driver_make = prompt.text( 'Driver Make', self.driver_make )
self.driver_model = prompt.text( 'Driver Model', self.driver_model )
self.driver_name = prompt.text( 'Driver Name', self.driver_name )
if not (self.driver_class):
self.driver_class = prompt.text( 'Driver Class', self.driver_class )
self.metadata = Metadata(self.driver_make, self.driver_model, self.driver_name)
def fetch_metadata(self):
"""
@brief collect metadata from the user
"""
if not (self.driver_make and self.driver_model and self.driver_name):
self.driver_make = prompt.text( 'Driver Make', self.driver_make )
self.driver_model = prompt.text( 'Driver Model', self.driver_model )
self.driver_name = prompt.text( 'Driver Name', self.driver_name )
self.metadata = Metadata(self.driver_make, self.driver_model, self.driver_name)
self.driver_version = prompt.text('Driver Version', self.metadata.version)
def _get_file():
"""
build the data file name. Then loop until the file can be open successfully
@return: file pointer to the data file
"""
metadata = Metadata()
config_path = "%s/%s" % (metadata.driver_dir(), CommConfig.config_filename())
comm_config = CommConfig.get_config_from_file(config_path)
date = time.strftime("%Y%m%d")
filename = "%s/port_agent_%d.%s.data" % (DATADIR, comm_config.command_port, date)
file = None
while(not file):
try:
file = open(filename)
except Exception as e:
sys.stderr.write("file open failed: %s\n" % e)
time.sleep(SLEEP)
return file
def __init__(self):
"""
@brief ctor
"""
self.metadata = Metadata()
self._zipfile = None
self._manifest = None
self._compression = None
self.generator = DriverGenerator(self.metadata)
# Set compression level
self.zipfile_compression()
def get_metadata(self):
# get which dataset agent is selected from the current metadata, use
# this to get metadata from the cloned repo
tmp_metadata = Metadata()
# read metadata from the cloned repo
self.metadata = Metadata(tmp_metadata.driver_make,
tmp_metadata.driver_model,
tmp_metadata.driver_name,
REPODIR + '/marine-integrations')
return self.metadata
def run(self):
print "*** Starting Driver Packaging Process***"
# store the original directory since we will be navigating away from it
original_dir = os.getcwd()
# first create a temporary clone of ooici to work with
self.clone_repo()
# get which dataset agent is selected from the current metadata, use
# this to get metadata from the cloned repo
tmp_metadata = Metadata()
# read metadata from the cloned repo
self.metadata = Metadata(
tmp_metadata.driver_make,
tmp_metadata.driver_model,
tmp_metadata.driver_name,
REPODIR + "/marine-integrations",
)
if self.test_mode:
# sys.argv.append("--repackage")
sys.argv.append("--no-test")
sys.argv.append("--no-push")
if "--repackage" in sys.argv:
self.get_repackage_version(self.build_name())
else:
new_version = self.update_version()
base_name = self.build_name() + "_" + new_version.replace(".", "_")
self.make_branch(base_name)
if "--no-test" in sys.argv:
f = open(self.log_path(), "w")
f.write("Tests manually bypassed with --no-test option\n")
f.close()
self.package_driver()
else:
if self.run_qualification_tests():
self.package_driver()
if not "--no-push" in sys.argv and not "--repackage" in sys.argv:
cmd = "git push %s" % WRITE_REPO_URL
output = subprocess.check_output(cmd, shell=True)
if len(output) > 0:
log.debug("git push returned: %s", output)
# go back to the original directory
os.chdir(original_dir)
print "Package Created: " + self.archive_path()
def checkout_version(self):
"""
@brief Check out this driver version from the repository if it exists
"""
base_name = '%s_%s_%s_%s' % (self.driver_make,
self.driver_model,
self.driver_name,
self.driver_version.replace('.', '_'))
cmd = 'git tag -l ' + 'release_' + base_name
output = subprocess.check_output(cmd, shell=True)
if len(output) > 0:
# this tag exists, check out the branch
#(tag is the branch name with 'release_' in front)
# checkout the branch so changes can be saved
cmd = 'git checkout ' + base_name
output = subprocess.check_output(cmd, shell=True)
# re-read metadata file since it has changed
self.metadata = Metadata(self.driver_make, self.driver_model, self.driver_name)
else:
raise DriverDoesNotExist("Driver version %s does not exist", self.driver_version)
def fetch_metadata(self):
"""
@brief collect metadata from the user
"""
self.metadata = Metadata()
self.metadata.get_from_console()
def get_metadata(self):
self.metadata = Metadata(self.driver_make, self.driver_model, self.driver_name)
return self.metadata
except IOError as e:
print "Encountered problem writing strings template, complete by hand"
def display_report(self):
"""
@brief Display a report of the files created to STDOUT
"""
print("*** Generation Complete ***")
print(" - Driver File: " + self.driver_dir() + "/" +
self.driver_filename())
print(" - Test File: " + self.driver_test_dir() + "/" +
self.driver_test_filename())
print(" - Resource Directory: " + self.resource_dir())
def generate(self):
"""
@brief Main method for generating drivers. Assumption: this is run from the console.
"""
print("*** Generating Driver Code ***")
self.build_directories()
self.generate_code()
self.display_report()
if __name__ == '__main__':
metadata = Metadata()
driver = DriverGenerator(metadata)
driver.generate()
class SwitchDriver():
"""
Main class for running the switch driver process.
"""
def __init__(self, make=None, model=None, name=None, version=None):
self.driver_make = make
self.driver_model = model
self.driver_name = name
self.driver_version = version
def fetch_metadata(self):
"""
@brief collect metadata from the user
"""
if not (self.driver_make and self.driver_model and self.driver_name):
self.driver_make = prompt.text( 'Driver Make', self.driver_make )
self.driver_model = prompt.text( 'Driver Model', self.driver_model )
self.driver_name = prompt.text( 'Driver Name', self.driver_name )
self.metadata = Metadata(self.driver_make, self.driver_model, self.driver_name)
self.driver_version = prompt.text('Driver Version', self.metadata.version)
def fetch_comm_config(self):
"""
@brief collect connection information for the logger from the user
"""
config_path = "%s/%s" % (self.metadata.driver_dir(), CommConfig.config_filename())
self.comm_config = CommConfig.get_config_from_console(config_path)
self.comm_config.get_from_console()
def checkout_version(self):
"""
@brief Check out this driver version from the repository if it exists
"""
base_name = '%s_%s_%s_%s' % (self.driver_make,
self.driver_model,
self.driver_name,
self.driver_version.replace('.', '_'))
cmd = 'git tag -l ' + 'release_' + base_name
output = subprocess.check_output(cmd, shell=True)
if len(output) > 0:
# this tag exists, check out the branch
#(tag is the branch name with 'release_' in front)
# checkout the branch so changes can be saved
cmd = 'git checkout ' + base_name
output = subprocess.check_output(cmd, shell=True)
# re-read metadata file since it has changed
self.metadata = Metadata(self.driver_make, self.driver_model, self.driver_name)
else:
raise DriverDoesNotExist("Driver version %s does not exist", self.driver_version)
def run(self):
"""
@brief Run it.
"""
print( "*** Starting Switch Driver Process***" )
self.fetch_metadata()
if not exists(self.metadata.driver_dir()):
raise DriverDoesNotExist( "%s", self.metadata.driver_dir() )
# if this version does not match the requested one, make sure the version exists,
# then checkout the branch with that version
if self.driver_version != self.metadata.version:
self.checkout_version()
self.fetch_comm_config()
self.metadata.link_current_metadata()
@staticmethod
def list_drivers():
"""
@brief Print a list of all the drivers and their versions
"""
driver_dir = join(Config().get("working_repo"), 'mi', 'instrument')
log.debug("Driver Dir: %s", driver_dir)
drivers = SwitchDriver.get_drivers()
for make in sorted(drivers.keys()):
for model in sorted(drivers[make].keys()):
for name in sorted(drivers[make][model].keys()):
for version in sorted(drivers[make][model][name]):
print "%s %s %s %s" % (make, model, name, version)
@staticmethod
def get_drivers():
"""
@brief Get a list of all drivers and their versions
"""
driver_dir = join(Config().get("working_repo"), 'mi', 'instrument')
log.debug("Driver Dir: %s", driver_dir)
drivers = {}
for make in listdir(driver_dir):
make_dir = join(driver_dir, make)
if isdir(make_dir) and not make == 'test':
for model in listdir(make_dir):
model_dir = join(make_dir, model)
if isdir(model_dir) and not model == 'test':
for name in listdir(model_dir):
name_dir = join(model_dir, name)
if isdir(name_dir) and not name == 'test':
log.debug("found driver: %s %s %s", make, model, name)
if not drivers.get(make): drivers[make] = {}
if not drivers[make].get(model): drivers[make][model] = {}
drivers[make][model][name] = SwitchDriver.get_versions(make,model,name)
return drivers
@staticmethod
def get_versions(make, model, name):
"""
@brief Get the version for this instrument driver from the tags
@param make
@param model
@param name
"""
full_name = 'release_%s_%s_%s' % (make, model, name)
# get all tags that start with this instrument
cmd = 'git tag -l ' + full_name + '*'
output = subprocess.check_output(cmd, shell=True)
version_list = []
if len(output) > 0:
tag_regex = re.compile(r'release_[a-z0-9_]+(\d+_\d+_\d+)')
tag_iter = tag_regex.finditer(output)
for tag_match in tag_iter:
version_list.append(tag_match.group(1))
return version_list
class RunInstrument(MiIntTestCase):
"""
Main class for communicating with an instrument
"""
def __init__(self, monitor=False, subscriber=False):
self.driver_make = None
self.driver_model = None
self.driver_name = None
self.driver_class = DRIVER_CLASS
self.ip_address = None
self.data_port = None
self.command_port = None
self.driver_version = None
self._pagent = None
self.monitor_window = monitor
self.subcriber_window = subscriber
self.stream_config = {}
self._cleanups = []
def _initialize(self):
"""
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
try:
"""
Get the information for the driver. This can be read from the yml
files; the user can run switch_driver to change the current driver.
"""
self.fetch_metadata()
self.fetch_driver_class()
self.fetch_comm_config()
if not exists(PIPE_PATH):
mkfifo(PIPE_PATH)
if not exists(self.metadata.driver_dir()):
raise DriverDoesNotExist( "%s/%s/$%s" % (self.metadata.driver_make,
self.metadata.driver_model,
self.driver_name))
driver_module = DRIVER_MODULE_ROOT + self.metadata.driver_make + '.' + self.metadata.driver_model + '.' + self.metadata.driver_name + DRIVER_MODULE_LEAF
log.info('driver module: %s', driver_module)
log.info('driver class: %s', self.driver_class)
log.info('device address: %s', self.ip_address)
log.info('device data port: %s', self.data_port)
log.info('device command port: %s', self.command_port)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
DVR_CONFIG.update({'dvr_mod' : driver_module, 'dvr_cls' : self.driver_class})
"""
self._support = DriverIntegrationTestSupport(driver_module,
self.driver_class,
self.ip_address,
self.data_port,
DELIM,
WORK_DIR)
"""
# Start port agent, add stop to cleanup (not sure if that's
# necessary yet).
print( "------------------>>>> Starting Port Agent <<<<------------------" )
self.start_pagent()
# Start a monitor window if specified.
if self.monitor_window:
self.monitor_file = self._pagent.port_agent.logfname
strXterm = "xterm -T InstrumentMonitor -sb -rightbar"
#pOpenString = "xterm -T InstrumentMonitor -e tail -f " + self.monitor_file
pOpenString = strXterm + " -e tail -f " + self.monitor_file
x = subprocess.Popen(pOpenString, shell=True)
"""
DHE: Added self._cleanups to make base classes happy
"""
self.addCleanup(self.stop_pagent)
# Start container.
print( "------------------>>>> Starting Capability Container <<<<------------------" )
self._start_container()
# Bring up services in a deploy file (no need to message)
print( "------------------>>>> Starting Deploy Services <<<<------------------" )
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Setup stream config.
self._build_stream_config()
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
self._ia_pid = None
print( "------------------>>>> Starting Instrument Agent <<<<------------------" )
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID,
process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
if self.subcriber_window:
self._start_data_subscribers(6)
#self.addCleanup(self._stop_data_subscribers)
except:
log.error("initialize(): Exception occurred; shutting down.", exc_info=True)
return False
else:
return True
###############################################################################
# Port agent helpers.
###############################################################################
def start_pagent(self):
"""
Construct and start the port agent.
@retval port Port that was used for connection to agent
"""
# Create port agent object.
comm_config = self.comm_config
config = {
'device_addr' : comm_config.device_addr,
'device_port' : comm_config.device_port,
'command_port': comm_config.command_port,
'data_port': comm_config.data_port,
'process_type': PortAgentProcessType.UNIX,
'log_level': 5,
}
self._pagent = PortAgentProcess.launch_process(config, timeout = 60, test_mode = True)
pid = self._pagent.get_pid()
port = self._pagent.get_data_port()
log.info('Started port agent pid %d listening at port %d', pid, port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port
}
return port
def stop_pagent(self):
"""
Stop the port agent.
"""
if self._pagent:
pid = self._pagent.get_pid()
if pid:
log.info('Stopping pagent pid %i', pid)
self._pagent.stop()
else:
log.info('No port agent running.')
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
streams = {
'parsed' : 'ctd_parsed_param_dict',
'raw' : 'ctd_raw_param_dict'
}
for (stream_name, param_dict_name) in streams.iteritems():
pd_id = dataset_management.read_parameter_dictionary_by_name(DEFAULT_PARAM_DICT, id_only=True)
if (not pd_id):
log.error("No pd_id found for param_dict '%s'" % DEFAULT_PARAM_DICT)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name,
parameter_dictionary_id=pd_id)
pd = None
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self.stream_config[stream_name] = stream_config
def _start_data_subscribers(self, count):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
#self._async_data_result = AsyncResult()
strXterm = "xterm -T InstrumentScienceData -sb -rightbar "
pOpenString = strXterm + " -e tail -f " + PIPE_PATH
subprocess.Popen(['xterm', '-T', 'InstrumentScienceData', '-e', 'tail', '-f', PIPE_PATH])
#subprocess.Popen(pOpenString)
#self.pipeData = open(PIPE_PATH, "w", 1)
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
print 'Received message on ' + str(stream_id) + ' (' + str(stream_route.exchange_point) + ',' + str(stream_route.routing_key) + ')'
log.info('Received message on %s (%s,%s)', stream_id, stream_route.exchange_point, stream_route.routing_key)
self.pipeData = open(PIPE_PATH, "w", 1)
self.pipeData.write(str(message))
self.pipeData.flush()
self.pipeData.close()
self._samples_received.append(message)
#if len(self._samples_received) == count:
#self._async_data_result.set()
for (stream_name, stream_config) in self._stream_config.iteritems():
stream_id = stream_config['stream_id']
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
print 'stream_id: %s' % stream_id
sub_id = pubsub_client.create_subscription(name=exchange_name, stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
for subscriber in self._data_subscribers:
pubsub_client = PubsubManagementServiceClient()
if hasattr(subscriber,'subscription_id'):
try:
pubsub_client.deactivate_subscription(subscriber.subscription_id)
except:
pass
pubsub_client.delete_subscription(subscriber.subscription_id)
subscriber.stop()
def bring_instrument_active(self):
"""
@brief Bring the agent up to COMMAND state,
"""
"""
DHE: Don't have an event subscriber yet
# Set up a subscriber to collect error events.
#self._start_event_subscriber('ResourceAgentResourceStateEvent', 6)
#self.addCleanup(self._stop_event_subscriber)
"""
try:
state = self._ia_client.get_agent_state()
print "AgentState: " + str(state)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
print "AgentState: " + str(state)
res_state = self._ia_client.get_resource_state()
print "DriverState: " + str(res_state)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
print "AgentState: " + str(state)
res_state = self._ia_client.get_resource_state()
print "DriverState: " + str(res_state)
"""
If the agent is in STREAMING state, it will not accept the run
command.
"""
if state != ResourceAgentState.STREAMING:
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
print "AgentState: " + str(state)
res_state = self._ia_client.get_resource_state()
print "DriverState: " + str(res_state)
except:
log.error("bring_instrument_active(): Exception occurred; shutting down.", exc_info=True)
return False
else:
return True
"""
DHE: Don't have an event subscriber yet so we've received no events.
self._async_event_result.get(timeout=2)
print "Received: " + str(len(self._events_received)) + " events."
"""
###############################################################################
# RunInstrument helpers.
###############################################################################
def get_capabilities(self):
"""
@brief Get exposed capabilities in current state.
"""
retval = self._ia_client.get_capabilities()
# Validate capabilities for state UNINITIALIZED.
self.agt_cmds = [x.name for x in retval if x.cap_type==CapabilityType.AGT_CMD]
self.agt_pars = [x.name for x in retval if x.cap_type==CapabilityType.AGT_PAR]
self.res_cmds = [x.name for x in retval if x.cap_type==CapabilityType.RES_CMD]
self.res_pars = [x.name for x in retval if x.cap_type==CapabilityType.RES_PAR]
print "\n------------------>>>> Current Capabilities <<<<------------------"
print "Agent Commands: " + str(self.agt_cmds)
#print "Agent Parameters: " + str(self.agt_pars)
print "Resource Commands: " + str(self.res_cmds)
#print "Resource Parameters: " + str(self.res_pars)
def send_agent_command(self, command):
"""
@brief Send a command to the agent.
"""
DA_WAIT_PERIOD = 60
waiting = False
print "Input command: " + str(command)
if command == 'RESOURCE_AGENT_EVENT_GO_DIRECT_ACCESS':
cmd = AgentCommand(command = command,
kwargs={'session_type': DirectAccessTypes.telnet,
'session_timeout':600,
'inactivity_timeout':600})
waiting = True
else:
cmd = AgentCommand(command = command)
retval = self._ia_client.execute_agent(cmd)
print "Results of command: " + str(retval)
while waiting:
print "Waiting " + str(DA_WAIT_PERIOD) + " seconds for you to test direct access."
gevent.sleep(DA_WAIT_PERIOD)
still_waiting = prompt.text('Still waiting? (y/n)')
if still_waiting is 'n':
waiting = False
def send_driver_command(self, command):
"""
@brief Send a command to the instrument through the instrument agent.
First determine whether it's a get or set, which are handled separately.
"""
if command == DriverEvent.GET:
self._get_param()
elif command == DriverEvent.SET:
self._set_param()
else:
print "Input command: " + str(command)
cmd = AgentCommand(command = command)
retval = self._ia_client.execute_resource(cmd)
print "Results of command: " + str(retval)
def _get_param(self):
"""
@brief Get a single parameter from the instrument (will be updated to get
multiple later).
"""
_all_params = self._ia_client.get_resource('DRIVER_PARAMETER_ALL')
print "Parameters you can get are: " + str(_all_params)
_param_valid = False
while _param_valid is False:
_param = prompt.text('\nEnter a single parameter')
if _param in _all_params:
_param_valid = True
else:
print 'Invalid parameter: ' + _param
reply = self._ia_client.get_resource([_param])
print 'Reply is :' + str(reply)
def _set_param(self):
"""
@brief Set a single parameter
"""
_all_params = self._ia_client.get_resource('DRIVER_PARAMETER_ALL')
print "Parameters you can set are: " + str(_all_params)
_param_valid = False
while _param_valid is False:
_param = prompt.text('\nEnter a single parameter')
if _param in _all_params:
_param_valid = True
else:
print 'Invalid parameter: ' + _param
_value = prompt.text('Enter value')
_value = _value.lower()
"""
DHE: Need to convert to native types here; can't be string; this is a
problem for the UI because we need a way to get the metadata about
each param to the UI.
"""
if _value == 'true':
_value = True
elif _value == 'false':
_value = False
param_dict = {_param: _value}
self._ia_client.set_resource(param_dict)
def fetch_metadata(self):
"""
@brief collect metadata from the user
"""
self.metadata = Metadata()
self.driver_make = self.metadata.driver_make
self.driver_model = self.metadata.driver_model
self.driver_name = self.metadata.driver_name
if not (self.driver_make and self.driver_model and self.driver_name):
self.driver_make = prompt.text( 'Driver Make', self.driver_make )
self.driver_model = prompt.text( 'Driver Model', self.driver_model )
self.driver_name = prompt.text( 'Driver Name', self.driver_name )
if not (self.driver_class):
self.driver_class = prompt.text( 'Driver Class', self.driver_class )
self.metadata = Metadata(self.driver_make, self.driver_model, self.driver_name)
def fetch_comm_config(self):
"""
@brief collect connection information for the logger from the user
"""
config_path = "%s/%s" % (self.metadata.driver_dir(), CommConfig.config_filename())
self.comm_config = CommConfig.get_config_from_console(config_path)
self.comm_config.display_config()
#self.comm_config.get_from_console()
self.ip_address = self.comm_config.device_addr
self.data_port = self.comm_config.data_port
self.command_port = self.comm_config.command_port
if not (self.ip_address):
self.ip_address = prompt.text( 'Instrument IP Address', self.ip_address )
if not (self.data_port):
continuing = True
while continuing:
sport = prompt.text( 'Instrument Port', self.data_port )
try:
self.data_port = int(sport)
continuing = False
except ValueError as e:
print "Error converting port to number: " + str(e)
print "Please enter a valid port number.\n"
def fetch_driver_class(self):
self.driver_class = prompt.text( 'Driver Class', self.driver_class )
def get_user_command(self, text='Enter command'):
command = prompt.text(text)
return command
def run(self):
"""
@brief Run it.
"""
print( "------------------>>>> Starting RunInstrument <<<<------------------" )
"""
initialize; returns True if successful, else False.
"""
continuing = self._initialize()
"""
bring_instrument_active; returns True if successful, else False
"""
if (continuing):
continuing = self.bring_instrument_active()
PROMPT = 'Enter command (\'quit\' to exit)'
text = PROMPT
while continuing:
try:
"""
Get a list of the currently available capabilities
"""
self.get_capabilities()
command = self.get_user_command(text)
text = PROMPT
if command == 'quit':
continuing = False
elif command in self.agt_cmds:
self.send_agent_command(command)
elif command in self.res_cmds:
self.send_driver_command(command)
else:
text = 'Invalid Command: ' + command + '\n' + PROMPT
except:
log.error("run(): Exception occurred; shutting down.", exc_info=True)
continuing = False
self.stop_pagent()
print( "------------------>>>> Stopping RunInstrument <<<<------------------" )
class PackageDriver(object):
"""
Main class for running the package driver process.
"""
###
# Configuration
###
def log_file(self):
return "qualification.log"
def log_path(self):
return "%s/%s" % (self.metadata.idk_dir(), self.log_file())
def archive_file(self):
return "%s_%s_%s-%s-driver.zip" % (self.metadata.driver_make,
self.metadata.driver_model,
self.metadata.driver_name,
self.metadata.version)
def archive_path(self):
return os.path.join(os.path.expanduser("~"),self.archive_file())
###
# Public Methods
###
def __init__(self):
"""
@brief ctor
"""
self.metadata = Metadata()
self._zipfile = None
self._manifest = None
self._compression = None
self.generator = DriverGenerator(self.metadata)
# Set compression level
self.zipfile_compression()
def run_qualification_tests(self):
"""
@brief Run all qualification tests for the driver and store the results for packaging
"""
log.info("-- Running qualification tests")
test = NoseTest(self.metadata, log_file=self.log_path())
test.report_header()
result = test.run_qualification()
if(test.run_qualification()):
log.info(" ++ Qualification tests passed")
return True
else:
log.error("Qualification tests have fail! No package created.")
return False
def package_driver(self):
"""
@brief Store driver files in a zip package
"""
log.info("-- Building driver package")
self._store_package_files()
def run(self):
print "*** Starting Driver Packaging Process***"
if( self.run_qualification_tests() ):
self.package_driver()
print "Package Created: " + self.archive_path()
else:
sys.exit()
def zipfile(self):
"""
@brief Return the ZipFile object. Create the file if it isn't already open
@retval ZipFile object
"""
if(not self._zipfile):
self._zipfile = zipfile.ZipFile(self.archive_path(), mode="w")
return self._zipfile
def zipfile_compression(self):
"""
@brief What type of compression should we use for the package file. If we have access to zlib, we will compress
@retval Compression type
"""
if(self._compression): return self._compression
try:
import zlib
self._compression = zipfile.ZIP_DEFLATED
log.info("Setting compression level to deflated")
except:
log.info("Setting compression level to store only")
self._compression = zipfile.ZIP_STORED
def manifest(self):
"""
@brief Return the PackageManifest object. Create it if it doesn't already exist
@retval PackageManifest object
"""
if(not self._manifest):
self._manifest = PackageManifest(self.metadata)
return self._manifest
###
# Private Methods
###
def _store_package_files(self):
"""
@brief Store all files in zip archive and add them to the manifest file
"""
egg_generator = EggGenerator(self.metadata)
egg_file = egg_generator.save()
# Add egg
self._add_file(egg_file, 'egg', 'python driver egg package')
# Add the package metadata file
self._add_file(self.metadata.metadata_path(), description = 'package metadata')
# Add the qualification test log
self._add_file(self.log_path(), description = 'qualification tests results')
# Store additional resource files
self._store_resource_files()
# Finally save the manifest file. This must be last of course
self._add_file(self.manifest().manifest_path(), description = 'package manifest file')
def _store_resource_files(self):
"""
@brief Store additional files added by the driver developer. These files life in the driver resource dir.
"""
log.debug( " -- Searching for developer added resource files." )
for file in os.listdir(self.generator.resource_dir()):
log.debug(" ++ found: " + file)
desc = prompt.text( 'Describe ' + file )
self._add_file(self.generator.resource_dir() + "/" + file, 'resource', desc)
def _add_file(self, source, destdir=None, description=None):
"""
@brief Add a file to the zip package and store the file in the manifest.
"""
filename = os.path.basename(source)
dest = filename
if(destdir):
dest = "%s/%s" % (destdir, filename)
log.debug( "archive %s to %s" % (filename, dest) )
self.manifest().add_file(dest, description);
self.zipfile().write(source, dest, self.zipfile_compression())
class PackageDriver(object):
"""
Main class for running the package driver process.
"""
###
# Configuration
###
def string_file(self):
return "strings.yml"
def log_file(self):
return "qualification.log"
def log_path(self):
return "%s/%s" % (self.metadata.idk_dir(), self.log_file())
def build_name(self):
return "%s_%s_%s" % (self.metadata.driver_make,
self.metadata.driver_model,
self.metadata.driver_name)
def archive_file(self):
return "%s-%s-driver.zip" % (self.build_name(), self.metadata.version)
def archive_path(self):
return os.path.join(os.path.expanduser("~"), self.archive_file())
def get_metadata(self):
# get which dataset agent is selected from the current metadata, use
# this to get metadata from the cloned repo
tmp_metadata = Metadata()
# read metadata from the cloned repo
self.metadata = Metadata(tmp_metadata.driver_make,
tmp_metadata.driver_model,
tmp_metadata.driver_name,
REPODIR + '/marine-integrations')
return self.metadata
def get_nose_test(self):
return NoseTest(self.metadata, log_file=self.log_path())
def get_driver_generator(self):
return DriverGenerator(self.metadata)
def get_egg_generator(self):
return EggGenerator(self.metadata)
###
# Public Methods
###
def __init__(self):
"""
@brief ctor
"""
self._zipfile = None
self._manifest = None
self._compression = None
# Set compression level
self.zipfile_compression()
def run_qualification_tests(self):
"""
@brief Run all qualification tests for the driver and store the results for packaging
"""
log.info("-- Running qualification tests")
test = self.get_nose_test(self.metadata, log_file=self.log_path())
test.report_header()
if (test.run_qualification()):
log.info(" ++ Qualification tests passed")
return True
else:
log.error("Qualification tests have fail! No package created.")
return False
def clone_repo(self):
"""
clone the ooici repository into a temp location and navigate to it
"""
# make a temp dir to put the clone in
if not os.path.exists(REPODIR):
os.mkdir(REPODIR)
os.chdir(REPODIR)
# remove an old clone if one exists, start clean
if os.path.exists(REPODIR + '/marine-integrations'):
shutil.rmtree(REPODIR + '/marine-integrations')
# clone the ooici repository into a temporary location
log.debug('Attempting to clone repository into %s, REPODIR set to %s',
os.getcwd(), REPODIR)
ret = os.system(
'git clone [email protected]:ooici/marine-integrations.git')
if ret < 0:
raise GitCommandException("Bad return from git command")
# if the directory doesn't exist, something went wrong with cloning
if not os.path.exists(REPODIR + '/marine-integrations'):
raise GitCommandException(
'Error creating ooici repository clone with base: %s' %
REPODIR)
# navigate into the cloned repository
os.chdir(REPODIR + '/marine-integrations')
log.debug('in cloned repository')
def get_repackage_version(self, tag_base):
"""
Get the driver version the user wants to repackage
"""
# suggest the current driver version as default
repkg_version = prompt.text('Driver Version to re-package',
self.metadata.version)
# confirm this version has the correct format
self._verify_version(repkg_version)
# check to make sure this driver version exists
tag_name = 'release_' + tag_base + '_' + repkg_version.replace(
'.', '_')
cmd = 'git tag -l ' + tag_name
# find out if this tag name exists
output = subprocess.check_output(cmd, shell=True)
if len(output) > 0:
# this tag exists, check it out
os.system('git checkout tags/' + tag_name)
else:
log.error('No driver version %s found', tag_name)
raise InvalidParameters('No driver version %s found', tag_name)
def make_branch(self, base_name):
"""
Make a new branch for this release and tag it with the same name so we
can get back to it
@param base_name - the base name for this instrument used to make the
branch and tag names. The base should have the form:
'<driver_name>_<driver_version>', where the version has the format X_X_X.
This is equal to the branch name, and the tag will have 'release_'
prepended to the base name.
"""
# create a new branch name and check it out
cmd = 'git checkout -b ' + base_name
output = subprocess.check_output(cmd, shell=True)
log.debug('created new branch %s: %s', base_name, output)
# tag the initial branch so that we can get back to it later
cmd = 'git tag ' + 'release_' + base_name
output = subprocess.check_output(cmd, shell=True)
log.debug('created new tag %s: %s', 'release_' + base_name, output)
def update_version(self):
"""
Update the driver version for this package. By default increment by one.
After updating the metadata file, commit the change to git.
"""
last_dot = self.metadata.version.rfind('.')
last_version = int(self.metadata.version[last_dot + 1:])
suggest_version = self.metadata.version[:last_dot +
1] + str(last_version + 1)
new_version = prompt.text('Update Driver Version', suggest_version)
# confirm this version has the correct format
self._verify_version(new_version)
if new_version != self.metadata.version:
# search for the tag for this version, find out if it already exists
cmd = 'git tag -l ' + 'release_' + self.build_name(
) + '_' + new_version.replace('.', '_')
# find out if this tag name exists
output = subprocess.check_output(cmd, shell=True)
if len(output) > 0:
# this tag already exists and we are not repackaging
raise InvalidParameters(
"Version %s already exists. To repackage, run package driver with the --repackage option",
new_version)
# set the new driver version in the metadata
self.metadata.set_driver_version(new_version)
# commit the changed file to git
cmd = 'git commit ' + str(self.metadata.metadata_path(
)) + ' -m \'Updated metadata driver version\''
os.system(cmd)
return new_version
def package_driver(self):
"""
@brief Store driver files in a zip package
"""
log.info("-- Building driver package")
self._store_package_files()
def run(self):
print "*** Starting Driver Packaging Process***"
# store the original directory since we will be navigating away from it
original_dir = os.getcwd()
# first create a temporary clone of ooici to work with
self.clone_repo()
self.metadata = self.get_metadata()
if "--repackage" in sys.argv:
self.get_repackage_version(self.build_name())
else:
new_version = self.update_version()
base_name = self.build_name() + '_' + new_version.replace('.', '_')
self.make_branch(base_name)
if "--no-test" in sys.argv:
f = open(self.log_path(), "w")
f.write("Tests manually bypassed with --no-test option\n")
f.close()
self.package_driver()
else:
if (self.run_qualification_tests()):
self.package_driver()
if not "--no-push" in sys.argv and not "--repackage" in sys.argv:
cmd = 'git push'
output = subprocess.check_output(cmd, shell=True)
if len(output) > 0:
log.debug('git push returned: %s', output)
# go back to the original directory
os.chdir(original_dir)
print "Package Created: " + self.archive_path()
def zipfile(self):
"""
@brief Return the ZipFile object. Create the file if it isn't already open
@retval ZipFile object
"""
if (not self._zipfile):
self._zipfile = zipfile.ZipFile(self.archive_path(), mode="w")
return self._zipfile
def zipfile_compression(self):
"""
@brief What type of compression should we use for the package file. If we have access to zlib, we will compress
@retval Compression type
"""
if (self._compression): return self._compression
try:
import zlib
self._compression = zipfile.ZIP_DEFLATED
log.info("Setting compression level to deflated")
except:
log.info("Setting compression level to store only")
self._compression = zipfile.ZIP_STORED
def manifest(self):
"""
@brief Return the PackageManifest object. Create it if it doesn't already exist
@retval PackageManifest object
"""
if (not self._manifest):
self._manifest = PackageManifest(self.metadata)
return self._manifest
###
# Private Methods
###
def _store_package_files(self):
"""
@brief Store all files in zip archive and add them to the manifest file
"""
# make sure metadata is up to date
self.metadata = self.get_metadata()
self.generator = self.get_driver_generator()
egg_generator = self.get_egg_generator()
egg_file = egg_generator.save()
# Add egg
self._add_file(egg_file, 'egg', 'python driver egg package')
# Add the package metadata file
self._add_file(self.metadata.metadata_path(),
description='package metadata')
# Add the qualification test log
self._add_file(self.log_path(),
description='qualification tests results')
# Store parameter/command string description file
str_path = "%s/%s" % (self.generator.resource_dir(),
self.string_file())
if os.path.exists(str_path):
self._add_file(str_path, 'resource', 'driver string file')
# Store additional resource files
self._store_resource_files()
# Finally save the manifest file. This must be last of course
self._add_file(self.manifest().manifest_path(),
description='package manifest file')
def _store_resource_files(self):
"""
@brief Store additional files added by the driver developer. These
files live in the driver resource dir.
"""
resource_dir = os.path.join(self.metadata.relative_driver_path(),
"resource")
log.debug(
" -- Searching for developer added resource files in dir: %s",
resource_dir)
stringfile = self.string_file()
if os.path.exists(resource_dir):
for file in os.listdir(resource_dir):
if file != stringfile:
log.debug(" ++ found: " + file)
desc = prompt.text('Describe ' + file)
self._add_file(resource_dir + "/" + file, 'resource', desc)
else:
log.debug(" --- No resource directory found, skipping...")
def _add_file(self, source, destdir=None, description=None):
"""
@brief Add a file to the zip package and store the file in the manifest.
"""
filename = os.path.basename(source)
dest = filename
if (destdir):
dest = "%s/%s" % (destdir, filename)
log.debug("archive %s to %s" % (filename, dest))
self.manifest().add_file(dest, description)
self.zipfile().write(source, dest, self.zipfile_compression())
def _verify_version(self, version=None):
"""
Ensure we have a good version number and that it has not already been packaged and published
"""
if version == None:
version = self.metadata.version
if not version:
raise ValidationFailure("Driver version required in metadata")
p = re.compile("^\d+\.\d+\.\d+$")
if not p.findall("%s" % version):
raise ValidationFailure(
"Version format incorrect '%s', should be x.x.x" % version)
class SwitchDriver():
"""
Main class for running the switch driver process.
"""
def __init__(self, make=None, model=None, name=None, version=None):
self.driver_make = make
self.driver_model = model
self.driver_name = name
self.driver_version = version
def fetch_metadata(self):
"""
@brief collect metadata from the user
"""
if not (self.driver_make and self.driver_model and self.driver_name):
self.driver_make = prompt.text( 'Driver Make', self.driver_make )
self.driver_model = prompt.text( 'Driver Model', self.driver_model )
self.driver_name = prompt.text( 'Driver Name', self.driver_name )
self.metadata = Metadata(self.driver_make, self.driver_model, self.driver_name)
def fetch_comm_config(self):
"""
@brief collect connection information for the logger from the user
"""
config_path = "%s/%s" % (self.metadata.driver_dir(), CommConfig.config_filename())
self.comm_config = CommConfig.get_config_from_console(config_path)
self.comm_config.get_from_console()
def run(self):
"""
@brief Run it.
"""
print( "*** Starting Switch Driver Process***" )
self.fetch_metadata()
if not exists(self.metadata.driver_dir()):
raise DriverDoesNotExist( "%s", self.metadata.driver_dir() )
self.fetch_comm_config()
self.metadata.link_current_metadata()
@staticmethod
def list_drivers():
driver_dir = join(Config().get("working_repo"), 'mi', 'instrument')
log.debug("Driver Dir: %s", driver_dir)
drivers = SwitchDriver.get_drivers()
for make in sorted(drivers.keys()):
for model in sorted(drivers[make].keys()):
for name in sorted(drivers[make][model].keys()):
for version in sorted(drivers[make][model][name]):
print "%s %s %s %s" % (make, model, name, version)
@staticmethod
def get_drivers():
driver_dir = join(Config().get("working_repo"), 'mi', 'instrument')
log.debug("Driver Dir: %s", driver_dir)
drivers = {}
for make in listdir(driver_dir):
make_dir = join(driver_dir, make)
if isdir(make_dir) and not make == 'test':
for model in listdir(make_dir):
model_dir = join(make_dir, model)
if isdir(model_dir) and not model == 'test':
for name in listdir(model_dir):
name_dir = join(model_dir, name)
if isdir(name_dir) and not name == 'test':
log.debug("found driver: %s %s %s", make, model, name)
if not drivers.get(make): drivers[make] = {}
if not drivers[make].get(model): drivers[make][model] = {}
drivers[make][model][name] = SwitchDriver.get_versions(make,model,name)
return drivers
@staticmethod
def get_versions(make, model, name):
return ['master']
class RunInstrument(MiIntTestCase):
"""
Main class for communicating with an instrument
"""
def __init__(self, monitor=False, subscriber=False):
self.driver_make = None
self.driver_model = None
self.driver_name = None
self.driver_class = DRIVER_CLASS
self.ip_address = None
self.data_port = None
self.command_port = None
self.driver_version = None
self._pagent = None
self.monitor_window = monitor
self.subcriber_window = subscriber
self.stream_config = {}
self._cleanups = []
def _initialize(self):
"""
Start port agent, add port agent cleanup.
Start container.
Start deploy services.
Define agent config, start agent.
Start agent client.
"""
try:
"""
Get the information for the driver. This can be read from the yml
files; the user can run switch_driver to change the current driver.
"""
self.fetch_metadata()
self.fetch_driver_class()
self.fetch_comm_config()
if not exists(PIPE_PATH):
mkfifo(PIPE_PATH)
if not exists(self.metadata.driver_dir()):
raise DriverDoesNotExist(
"%s/%s/$%s" %
(self.metadata.driver_make, self.metadata.driver_model,
self.driver_name))
driver_module = DRIVER_MODULE_ROOT + self.metadata.driver_make + '.' + self.metadata.driver_model + '.' + self.metadata.driver_name + DRIVER_MODULE_LEAF
log.info('driver module: %s', driver_module)
log.info('driver class: %s', self.driver_class)
log.info('device address: %s', self.ip_address)
log.info('device data port: %s', self.data_port)
log.info('device command port: %s', self.command_port)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
DVR_CONFIG.update({
'dvr_mod': driver_module,
'dvr_cls': self.driver_class
})
"""
self._support = DriverIntegrationTestSupport(driver_module,
self.driver_class,
self.ip_address,
self.data_port,
DELIM,
WORK_DIR)
"""
# Start port agent, add stop to cleanup (not sure if that's
# necessary yet).
print(
"------------------>>>> Starting Port Agent <<<<------------------"
)
self.start_pagent()
# Start a monitor window if specified.
if self.monitor_window:
self.monitor_file = self._pagent.port_agent.logfname
strXterm = "xterm -T InstrumentMonitor -sb -rightbar"
#pOpenString = "xterm -T InstrumentMonitor -e tail -f " + self.monitor_file
pOpenString = strXterm + " -e tail -f " + self.monitor_file
x = subprocess.Popen(pOpenString, shell=True)
"""
DHE: Added self._cleanups to make base classes happy
"""
self.addCleanup(self.stop_pagent)
# Start container.
print(
"------------------>>>> Starting Capability Container <<<<------------------"
)
self._start_container()
# Bring up services in a deploy file (no need to message)
print(
"------------------>>>> Starting Deploy Services <<<<------------------"
)
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Setup stream config.
self._build_stream_config()
# Create agent config.
agent_config = {
'driver_config': DVR_CONFIG,
'stream_config': self.stream_config,
'agent': {
'resource_id': IA_RESOURCE_ID
},
'test_mode': True
}
# Start instrument agent.
self._ia_pid = None
print(
"------------------>>>> Starting Instrument Agent <<<<------------------"
)
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID,
process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
if self.subcriber_window:
self._start_data_subscribers(6)
#self.addCleanup(self._stop_data_subscribers)
except:
log.error("initialize(): Exception occurred; shutting down.",
exc_info=True)
return False
else:
return True
###############################################################################
# Port agent helpers.
###############################################################################
def start_pagent(self):
"""
Construct and start the port agent.
@retval port Port that was used for connection to agent
"""
# Create port agent object.
comm_config = self.comm_config
config = {
'device_addr': comm_config.device_addr,
'device_port': comm_config.device_port,
'command_port': comm_config.command_port,
'data_port': comm_config.data_port,
'process_type': PortAgentProcessType.UNIX,
'log_level': 5,
}
self._pagent = PortAgentProcess.launch_process(config,
timeout=60,
test_mode=True)
pid = self._pagent.get_pid()
port = self._pagent.get_data_port()
cmd_port = self._pagent.get_command_port()
log.info('Started port agent pid %d listening at port %d', pid, port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr': 'localhost',
'port': port,
'cmd_port': cmd_port
}
return port
def stop_pagent(self):
"""
Stop the port agent.
"""
if self._pagent:
pid = self._pagent.get_pid()
if pid:
log.info('Stopping pagent pid %i', pid)
self._pagent.stop()
else:
log.info('No port agent running.')
###############################################################################
# Data stream helpers.
###############################################################################
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self.stream_config = {}
streams = {
'parsed': 'ctd_parsed_param_dict',
'raw': 'ctd_raw_param_dict'
}
for (stream_name, param_dict_name) in streams.iteritems():
pd_id = dataset_management.read_parameter_dictionary_by_name(
DEFAULT_PARAM_DICT, id_only=True)
if (not pd_id):
log.error("No pd_id found for param_dict '%s'" %
DEFAULT_PARAM_DICT)
stream_def_id = pubsub_client.create_stream_definition(
name=stream_name, parameter_dictionary_id=pd_id)
pd = None
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self.stream_config[stream_name] = stream_config
def _start_data_subscribers(self, count):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._data_subscribers = []
self._samples_received = []
#self._async_data_result = AsyncResult()
strXterm = "xterm -T InstrumentScienceData -sb -rightbar "
pOpenString = strXterm + " -e tail -f " + PIPE_PATH
subprocess.Popen([
'xterm', '-T', 'InstrumentScienceData', '-e', 'tail', '-f',
PIPE_PATH
])
#subprocess.Popen(pOpenString)
#self.pipeData = open(PIPE_PATH, "w", 1)
# A callback for processing subscribed-to data.
def recv_data(message, stream_route, stream_id):
print 'Received message on ' + str(stream_id) + ' (' + str(
stream_route.exchange_point) + ',' + str(
stream_route.routing_key) + ')'
log.info('Received message on %s (%s,%s)', stream_id,
stream_route.exchange_point, stream_route.routing_key)
self.pipeData = open(PIPE_PATH, "w", 1)
self.pipeData.write(str(message))
self.pipeData.flush()
self.pipeData.close()
self._samples_received.append(message)
#if len(self._samples_received) == count:
#self._async_data_result.set()
for (stream_name, stream_config) in self._stream_config.iteritems():
stream_id = stream_config['stream_id']
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
self._purge_queue(exchange_name)
sub = StandaloneStreamSubscriber(exchange_name, recv_data)
sub.start()
self._data_subscribers.append(sub)
print 'stream_id: %s' % stream_id
sub_id = pubsub_client.create_subscription(name=exchange_name,
stream_ids=[stream_id])
pubsub_client.activate_subscription(sub_id)
sub.subscription_id = sub_id # Bind the subscription to the standalone subscriber (easier cleanup, not good in real practice)
def _purge_queue(self, queue):
xn = self.container.ex_manager.create_xn_queue(queue)
xn.purge()
def _stop_data_subscribers(self):
for subscriber in self._data_subscribers:
pubsub_client = PubsubManagementServiceClient()
if hasattr(subscriber, 'subscription_id'):
try:
pubsub_client.deactivate_subscription(
subscriber.subscription_id)
except:
pass
pubsub_client.delete_subscription(subscriber.subscription_id)
subscriber.stop()
def bring_instrument_active(self):
"""
@brief Bring the agent up to COMMAND state,
"""
"""
DHE: Don't have an event subscriber yet
# Set up a subscriber to collect error events.
#self._start_event_subscriber('ResourceAgentResourceStateEvent', 6)
#self.addCleanup(self._stop_event_subscriber)
"""
try:
state = self._ia_client.get_agent_state()
print "AgentState: " + str(state)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
print "AgentState: " + str(state)
res_state = self._ia_client.get_resource_state()
print "DriverState: " + str(res_state)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
print "AgentState: " + str(state)
res_state = self._ia_client.get_resource_state()
print "DriverState: " + str(res_state)
"""
If the agent is in STREAMING state, it will not accept the run
command.
"""
if state != ResourceAgentState.STREAMING:
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
print "AgentState: " + str(state)
res_state = self._ia_client.get_resource_state()
print "DriverState: " + str(res_state)
except:
log.error(
"bring_instrument_active(): Exception occurred; shutting down.",
exc_info=True)
return False
else:
return True
"""
DHE: Don't have an event subscriber yet so we've received no events.
self._async_event_result.get(timeout=2)
print "Received: " + str(len(self._events_received)) + " events."
"""
###############################################################################
# RunInstrument helpers.
###############################################################################
def get_capabilities(self):
"""
@brief Get exposed capabilities in current state.
"""
retval = self._ia_client.get_capabilities()
# Validate capabilities for state UNINITIALIZED.
self.agt_cmds = [
x.name for x in retval if x.cap_type == CapabilityType.AGT_CMD
]
self.agt_pars = [
x.name for x in retval if x.cap_type == CapabilityType.AGT_PAR
]
self.res_cmds = [
x.name for x in retval if x.cap_type == CapabilityType.RES_CMD
]
self.res_pars = [
x.name for x in retval if x.cap_type == CapabilityType.RES_PAR
]
print "\n------------------>>>> Current Capabilities <<<<------------------"
print "Agent Commands: " + str(self.agt_cmds)
#print "Agent Parameters: " + str(self.agt_pars)
print "Resource Commands: " + str(self.res_cmds)
#print "Resource Parameters: " + str(self.res_pars)
def send_agent_command(self, command):
"""
@brief Send a command to the agent.
"""
DA_WAIT_PERIOD = 60
waiting = False
print "Input command: " + str(command)
if command == 'RESOURCE_AGENT_EVENT_GO_DIRECT_ACCESS':
cmd = AgentCommand(command=command,
kwargs={
'session_type': DirectAccessTypes.telnet,
'session_timeout': 600,
'inactivity_timeout': 600
})
waiting = True
else:
cmd = AgentCommand(command=command)
retval = self._ia_client.execute_agent(cmd)
print "Results of command: " + str(retval)
while waiting:
print "Waiting " + str(
DA_WAIT_PERIOD) + " seconds for you to test direct access."
gevent.sleep(DA_WAIT_PERIOD)
still_waiting = prompt.text('Still waiting? (y/n)')
if still_waiting is 'n':
waiting = False
def send_driver_command(self, command):
"""
@brief Send a command to the instrument through the instrument agent.
First determine whether it's a get or set, which are handled separately.
"""
if command == DriverEvent.GET:
self._get_param()
elif command == DriverEvent.SET:
self._set_param()
else:
print "Input command: " + str(command)
cmd = AgentCommand(command=command)
retval = self._ia_client.execute_resource(cmd)
print "Results of command: " + str(retval)
def _get_param(self):
"""
@brief Get a single parameter from the instrument (will be updated to get
multiple later).
"""
_all_params = self._ia_client.get_resource('DRIVER_PARAMETER_ALL')
print "Parameters you can get are: " + str(_all_params)
_param_valid = False
while _param_valid is False:
_param = prompt.text('\nEnter a single parameter')
if _param in _all_params:
_param_valid = True
else:
print 'Invalid parameter: ' + _param
reply = self._ia_client.get_resource([_param])
print 'Reply is :' + str(reply)
def _set_param(self):
"""
@brief Set a single parameter
"""
_all_params = self._ia_client.get_resource('DRIVER_PARAMETER_ALL')
print "Parameters you can set are: " + str(_all_params)
_param_valid = False
while _param_valid is False:
_param = prompt.text('\nEnter a single parameter')
if _param in _all_params:
_param_valid = True
else:
print 'Invalid parameter: ' + _param
_value = prompt.text('Enter value')
_value = _value.lower()
"""
DHE: Need to convert to native types here; can't be string; this is a
problem for the UI because we need a way to get the metadata about
each param to the UI.
"""
if _value == 'true':
_value = True
elif _value == 'false':
_value = False
param_dict = {_param: _value}
self._ia_client.set_resource(param_dict)
def fetch_metadata(self):
"""
@brief collect metadata from the user
"""
self.metadata = Metadata()
self.driver_make = self.metadata.driver_make
self.driver_model = self.metadata.driver_model
self.driver_name = self.metadata.driver_name
if not (self.driver_make and self.driver_model and self.driver_name):
self.driver_make = prompt.text('Driver Make', self.driver_make)
self.driver_model = prompt.text('Driver Model', self.driver_model)
self.driver_name = prompt.text('Driver Name', self.driver_name)
if not (self.driver_class):
self.driver_class = prompt.text('Driver Class', self.driver_class)
self.metadata = Metadata(self.driver_make, self.driver_model,
self.driver_name)
def fetch_comm_config(self):
"""
@brief collect connection information for the logger from the user
"""
config_path = "%s/%s" % (self.metadata.driver_dir(),
CommConfig.config_filename())
self.comm_config = CommConfig.get_config_from_console(config_path)
self.comm_config.display_config()
#self.comm_config.get_from_console()
self.ip_address = self.comm_config.device_addr
self.data_port = self.comm_config.data_port
self.command_port = self.comm_config.command_port
if not (self.ip_address):
self.ip_address = prompt.text('Instrument IP Address',
self.ip_address)
if not (self.data_port):
continuing = True
while continuing:
sport = prompt.text('Instrument Port', self.data_port)
try:
self.data_port = int(sport)
continuing = False
except ValueError as e:
print "Error converting port to number: " + str(e)
print "Please enter a valid port number.\n"
def fetch_driver_class(self):
self.driver_class = prompt.text('Driver Class', self.driver_class)
def get_user_command(self, text='Enter command'):
command = prompt.text(text)
return command
def run(self):
"""
@brief Run it.
"""
print(
"------------------>>>> Starting RunInstrument <<<<------------------"
)
"""
initialize; returns True if successful, else False.
"""
continuing = self._initialize()
"""
bring_instrument_active; returns True if successful, else False
"""
if (continuing):
continuing = self.bring_instrument_active()
PROMPT = 'Enter command (\'quit\' to exit)'
text = PROMPT
while continuing:
try:
"""
Get a list of the currently available capabilities
"""
self.get_capabilities()
command = self.get_user_command(text)
text = PROMPT
if command == 'quit':
continuing = False
elif command in self.agt_cmds:
self.send_agent_command(command)
elif command in self.res_cmds:
self.send_driver_command(command)
else:
text = 'Invalid Command: ' + command + '\n' + PROMPT
except:
log.error("run(): Exception occurred; shutting down.",
exc_info=True)
continuing = False
self.stop_pagent()
print(
"------------------>>>> Stopping RunInstrument <<<<------------------"
)
class SwitchDriver():
"""
Main class for running the switch driver process.
"""
def __init__(self, make=None, model=None, name=None, version=None):
self.driver_make = make
self.driver_model = model
self.driver_name = name
self.driver_version = version
def fetch_metadata(self):
"""
@brief collect metadata from the user
"""
if not (self.driver_make and self.driver_model and self.driver_name):
self.driver_make = prompt.text('Driver Make', self.driver_make)
self.driver_model = prompt.text('Driver Model', self.driver_model)
self.driver_name = prompt.text('Driver Name', self.driver_name)
self.metadata = Metadata(self.driver_make, self.driver_model,
self.driver_name)
self.driver_version = prompt.text('Driver Version',
self.metadata.version)
def fetch_comm_config(self):
"""
@brief collect connection information for the logger from the user
"""
config_path = "%s/%s" % (self.metadata.driver_dir(),
CommConfig.config_filename())
self.comm_config = CommConfig.get_config_from_console(config_path)
self.comm_config.get_from_console()
def checkout_version(self):
"""
@brief Check out this driver version from the repository if it exists
"""
base_name = '%s_%s_%s_%s' % (self.driver_make, self.driver_model,
self.driver_name,
self.driver_version.replace('.', '_'))
cmd = 'git tag -l ' + 'release_' + base_name
output = subprocess.check_output(cmd, shell=True)
if len(output) > 0:
# this tag exists, check out the branch
#(tag is the branch name with 'release_' in front)
# checkout the branch so changes can be saved
cmd = 'git checkout ' + base_name
output = subprocess.check_output(cmd, shell=True)
# re-read metadata file since it has changed
self.metadata = Metadata(self.driver_make, self.driver_model,
self.driver_name)
else:
raise DriverDoesNotExist("Driver version %s does not exist",
self.driver_version)
def run(self):
"""
@brief Run it.
"""
print("*** Starting Switch Driver Process***")
self.fetch_metadata()
if not exists(self.metadata.driver_dir()):
raise DriverDoesNotExist("%s", self.metadata.driver_dir())
# if this version does not match the requested one, make sure the version exists,
# then checkout the branch with that version
if self.driver_version != self.metadata.version:
self.checkout_version()
self.fetch_comm_config()
self.metadata.link_current_metadata()
@staticmethod
def list_drivers():
"""
@brief Print a list of all the drivers and their versions
"""
driver_dir = join(Config().get("working_repo"), 'mi', 'instrument')
log.debug("Driver Dir: %s", driver_dir)
drivers = SwitchDriver.get_drivers()
for make in sorted(drivers.keys()):
for model in sorted(drivers[make].keys()):
for name in sorted(drivers[make][model].keys()):
for version in sorted(drivers[make][model][name]):
print "%s %s %s %s" % (make, model, name, version)
@staticmethod
def get_drivers():
"""
@brief Get a list of all drivers and their versions
"""
driver_dir = join(Config().get("working_repo"), 'mi', 'instrument')
log.debug("Driver Dir: %s", driver_dir)
drivers = {}
for make in listdir(driver_dir):
make_dir = join(driver_dir, make)
if isdir(make_dir) and not make == 'test':
for model in listdir(make_dir):
model_dir = join(make_dir, model)
if isdir(model_dir) and not model == 'test':
for name in listdir(model_dir):
name_dir = join(model_dir, name)
if isdir(name_dir) and not name == 'test':
log.debug("found driver: %s %s %s", make,
model, name)
if not drivers.get(make): drivers[make] = {}
if not drivers[make].get(model):
drivers[make][model] = {}
drivers[make][model][
name] = SwitchDriver.get_versions(
make, model, name)
return drivers
@staticmethod
def get_versions(make, model, name):
"""
@brief Get the version for this instrument driver from the tags
@param make
@param model
@param name
"""
full_name = 'release_%s_%s_%s' % (make, model, name)
# get all tags that start with this instrument
cmd = 'git tag -l ' + full_name + '*'
output = subprocess.check_output(cmd, shell=True)
version_list = []
if len(output) > 0:
tag_regex = re.compile(r'release_[a-z0-9_]+(\d+_\d+_\d+)')
tag_iter = tag_regex.finditer(output)
for tag_match in tag_iter:
version_list.append(tag_match.group(1))
return version_list
class PackageDriver(object):
"""
Main class for running the package driver process.
"""
###
# Configuration
###
def string_file(self):
return "strings.yml"
def log_file(self):
return "qualification.log"
def log_path(self):
return "%s/%s" % (self.metadata.idk_dir(), self.log_file())
def build_name(self):
return "%s_%s_%s" % (self.metadata.driver_make,
self.metadata.driver_model,
self.metadata.driver_name)
def archive_file(self):
return "%s-%s-driver.zip" % (self.build_name(),
self.metadata.version)
def archive_path(self):
return os.path.join(os.path.expanduser("~"),self.archive_file())
###
# Public Methods
###
def __init__(self):
"""
@brief ctor
"""
self._zipfile = None
self._manifest = None
self._compression = None
# Set compression level
self.zipfile_compression()
def run_qualification_tests(self):
"""
@brief Run all qualification tests for the driver and store the results for packaging
"""
log.info("-- Running qualification tests")
test = NoseTest(self.metadata, log_file=self.log_path())
test.report_header()
if(test.run_qualification()):
log.info(" ++ Qualification tests passed")
return True
else:
log.error("Qualification tests have fail! No package created.")
return False
def clone_repo(self):
"""
clone the ooici repository into a temp location and navigate to it
"""
# make a temp dir to put the clone in
if not os.path.exists(REPODIR):
os.mkdir(REPODIR)
os.chdir(REPODIR)
# remove an old clone if one exists, start clean
if os.path.exists(REPODIR + '/marine-integrations'):
shutil.rmtree(REPODIR + '/marine-integrations')
# clone the ooici repository into a temporary location
log.debug('Attempting to clone repository into %s, REPODIR set to %s',
os.getcwd(), REPODIR)
ret = os.system('git clone [email protected]:ooici/marine-integrations.git')
if ret < 0:
raise GitCommandException("Bad return from git command")
# if the directory doesn't exist, something went wrong with cloning
if not os.path.exists(REPODIR + '/marine-integrations'):
raise GitCommandException('Error creating ooici repository clone with base: %s' % REPODIR)
# navigate into the cloned repository
os.chdir(REPODIR + '/marine-integrations')
log.debug('in cloned repository')
def get_repackage_version(self, tag_base):
"""
Get the driver version the user wants to repackage
"""
# suggest the current driver version as default
repkg_version = prompt.text( 'Driver Version to re-package', self.metadata.version )
# confirm this version has the correct format
self._verify_version(repkg_version)
# check to make sure this driver version exists
tag_name = 'release_' + tag_base + '_' + repkg_version.replace('.', '_')
cmd = 'git tag -l ' + tag_name
# find out if this tag name exists
output = subprocess.check_output(cmd, shell=True)
if len(output) > 0:
# this tag exists, check it out
os.system('git checkout tags/' + tag_name)
else:
log.error('No driver version %s found', tag_name)
raise InvalidParameters('No driver version %s found', tag_name)
def make_branch(self, base_name):
"""
Make a new branch for this release and tag it with the same name so we
can get back to it
@param base_name - the base name for this instrument used to make the
branch and tag names. The base should have the form:
'<driver_name>_<driver_version>', where the version has the format X_X_X.
This is equal to the branch name, and the tag will have 'release_'
prepended to the base name.
"""
# create a new branch name and check it out
cmd = 'git checkout -b ' + base_name
output = subprocess.check_output(cmd, shell=True)
log.debug('created new branch %s: %s', base_name, output)
# tag the initial branch so that we can get back to it later
cmd = 'git tag ' + 'release_' + base_name
output = subprocess.check_output(cmd, shell=True)
log.debug('created new tag %s: %s', 'release_' + base_name, output)
def update_version(self):
"""
Update the driver version for this package. By default increment by one.
After updating the metadata file, commit the change to git.
"""
last_dot = self.metadata.version.rfind('.')
last_version = int(self.metadata.version[last_dot+1:])
suggest_version = self.metadata.version[:last_dot+1] + str(last_version + 1)
new_version = prompt.text('Update Driver Version', suggest_version )
# confirm this version has the correct format
self._verify_version(new_version)
if new_version != self.metadata.version:
# search for the tag for this version, find out if it already exists
cmd = 'git tag -l ' + 'release_' + self.build_name() + '_' + new_version.replace('.', '_')
# find out if this tag name exists
output = subprocess.check_output(cmd, shell=True)
if len(output) > 0:
# this tag already exists and we are not repackaging
raise InvalidParameters("Version %s already exists. To repackage, run package driver with the --repackage option", new_version)
# set the new driver version in the metadata
self.metadata.set_driver_version(new_version)
# commit the changed file to git
cmd = 'git commit ' + str(self.metadata.metadata_path()) + ' -m \'Updated metadata driver version\''
os.system(cmd)
return new_version
def package_driver(self):
"""
@brief Store driver files in a zip package
"""
log.info("-- Building driver package")
self._store_package_files()
def run(self):
print "*** Starting Driver Packaging Process***"
# store the original directory since we will be navigating away from it
original_dir = os.getcwd()
# first create a temporary clone of ooici to work with
self.clone_repo()
# get which dataset agent is selected from the current metadata, use
# this to get metadata from the cloned repo
tmp_metadata = Metadata()
# read metadata from the cloned repo
self.metadata = Metadata(tmp_metadata.driver_make,
tmp_metadata.driver_model,
tmp_metadata.driver_name,
REPODIR + '/marine-integrations')
if "--repackage" in sys.argv:
self.get_repackage_version(self.build_name())
else:
new_version = self.update_version()
base_name = self.build_name() + '_' + new_version.replace('.', '_')
self.make_branch(base_name)
if "--no-test" in sys.argv:
f = open(self.log_path(), "w")
f.write("Tests manually bypassed with --no-test option\n")
f.close()
self.package_driver()
else:
if(self.run_qualification_tests()):
self.package_driver()
if not "--no-push" in sys.argv and not "--repackage" in sys.argv:
cmd = 'git push'
output = subprocess.check_output(cmd, shell=True)
if len(output) > 0:
log.debug('git push returned: %s', output)
# go back to the original directory
os.chdir(original_dir)
print "Package Created: " + self.archive_path()
def zipfile(self):
"""
@brief Return the ZipFile object. Create the file if it isn't already open
@retval ZipFile object
"""
if(not self._zipfile):
self._zipfile = zipfile.ZipFile(self.archive_path(), mode="w")
return self._zipfile
def zipfile_compression(self):
"""
@brief What type of compression should we use for the package file. If we have access to zlib, we will compress
@retval Compression type
"""
if(self._compression): return self._compression
try:
import zlib
self._compression = zipfile.ZIP_DEFLATED
log.info("Setting compression level to deflated")
except:
log.info("Setting compression level to store only")
self._compression = zipfile.ZIP_STORED
def manifest(self):
"""
@brief Return the PackageManifest object. Create it if it doesn't already exist
@retval PackageManifest object
"""
if(not self._manifest):
self._manifest = PackageManifest(self.metadata)
return self._manifest
###
# Private Methods
###
def _store_package_files(self):
"""
@brief Store all files in zip archive and add them to the manifest file
"""
# make sure metadata is up to date
self.metadata = Metadata(self.metadata.driver_make,
self.metadata.driver_model,
self.metadata.driver_name,
REPODIR + '/marine-integrations')
self.generator = DriverGenerator(self.metadata)
egg_generator = EggGenerator(self.metadata)
egg_file = egg_generator.save()
# Add egg
self._add_file(egg_file, 'egg', 'python driver egg package')
# Add the package metadata file
self._add_file(self.metadata.metadata_path(), description = 'package metadata')
# Add the qualification test log
self._add_file(self.log_path(), description = 'qualification tests results')
# Store parameter/command string description file
str_path = "%s/%s" % (self.generator.resource_dir(), self.string_file())
if os.path.exists(str_path):
self._add_file(str_path, 'resource', 'driver string file')
# Store additional resource files
self._store_resource_files()
# Finally save the manifest file. This must be last of course
self._add_file(self.manifest().manifest_path(), description = 'package manifest file')
def _store_resource_files(self):
"""
@brief Store additional files added by the driver developer. These
files live in the driver resource dir.
"""
resource_dir = os.path.join(self.metadata.relative_driver_path(), "resource")
log.debug(" -- Searching for developer added resource files in dir: %s",
resource_dir)
stringfile = self.string_file()
if os.path.exists(resource_dir):
for file in os.listdir(resource_dir):
if file != stringfile:
log.debug(" ++ found: " + file)
desc = prompt.text('Describe ' + file)
self._add_file(resource_dir + "/" + file, 'resource', desc)
else:
log.debug(" --- No resource directory found, skipping...")
def _add_file(self, source, destdir=None, description=None):
"""
@brief Add a file to the zip package and store the file in the manifest.
"""
filename = os.path.basename(source)
dest = filename
if(destdir):
dest = "%s/%s" % (destdir, filename)
log.debug("archive %s to %s" % (filename, dest))
self.manifest().add_file(dest, description);
self.zipfile().write(source, dest, self.zipfile_compression())
def _verify_version(self, version = None):
"""
Ensure we have a good version number and that it has not already been packaged and published
"""
if version == None:
version = self.metadata.version
if not version:
raise ValidationFailure("Driver version required in metadata")
p = re.compile("^\d+\.\d+\.\d+$")
if not p.findall("%s" % version):
raise ValidationFailure("Version format incorrect '%s', should be x.x.x" % version)
