def test_build_granule_and_load_from_granule_with_taxonomy(self): #Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle tx = TaxyTool() tx.add_taxonomy_set('temp', 'long_temp_name') tx.add_taxonomy_set('cond', 'long_cond_name') tx.add_taxonomy_set('pres', 'long_pres_name') tx.add_taxonomy_set('rdt') # map is {<local name>: <granule name or path>} #Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect rdt = RecordDictionaryTool(taxonomy=tx) #Create some arrays and fill them with random values temp_array = np.random.standard_normal(100) cond_array = np.random.standard_normal(100) pres_array = np.random.standard_normal(100) #Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc. rdt['temp'] = temp_array rdt['cond'] = cond_array rdt['pres'] = pres_array #You can also add in another RecordDictionaryTool, providing the taxonomies are the same. rdt2 = RecordDictionaryTool(taxonomy=tx) rdt2['temp'] = temp_array rdt['rdt'] = rdt2 g = build_granule(data_producer_id='john', taxonomy=tx, record_dictionary=rdt) l_tx = TaxyTool.load_from_granule(g) l_rd = RecordDictionaryTool.load_from_granule(g) # Make sure we got back the same Taxonomy Object self.assertEquals(l_tx._t, tx._t) self.assertEquals(l_tx.get_handles('temp'), tx.get_handles('temp')) self.assertEquals(l_tx.get_handles('testing_2'), tx.get_handles('testing_2')) # Now test the record dictionary object self.assertEquals(l_rd._rd, rdt._rd) self.assertEquals(l_rd._tx._t, rdt._tx._t) for k, v in l_rd.iteritems(): self.assertIn(k, rdt) if isinstance(v, np.ndarray): self.assertTrue( (v == rdt[k]).all()) else: self.assertEquals(v._rd, rdt[k]._rd)
def build_granule(): tt = TaxyTool() tt.add_taxonomy_set('c') tt.add_taxonomy_set('t') tt.add_taxonomy_set('d') rdt = RecordDictionaryTool(taxonomy=tt) rdt['c'] = np.array([0,1]) rdt['t'] = np.array([0,1]) rdt['d'] = np.array([0,1]) granule = bg(data_producer_id='test_identifier', taxonomy=tt, record_dictionary=rdt) return granule
def _get_taxy_tool(self, taxonomy): """ helper to get the TaxyTool version of the given argument. @param taxonomy Either a TaxyTool, a Taxonomy, or some value that can be processed by TaxyTool.load to create the TaxyTool instance. """ if isinstance(taxonomy, TaxyTool): tx = taxonomy elif isinstance(taxonomy, Taxonomy): tx = TaxyTool(taxonomy) else: tx = TaxyTool.load(taxonomy) return tx
def __init__(self, taxonomy=None, param_dictionary=None, shape=None): """ @brief Initialize a new instance of Record Dictionary Tool with a taxonomy and an optional fixed length @param taxonomy is an instance of a TaxonomyTool or Taxonomy (IonObject) used in this record dictionary @param length is an optional fixed length for the value sequences of this record dictionary """ if not isinstance(shape, (_NoneType, int, tuple)): raise TypeError('Invalid shape argument, received type "%s"; should be None or int or tuple' % type(shape)) self._rd = {} self._shp = shape if isinstance(self._shp, int): self._shp = (self._shp,) # hold onto the taxonomy - we need it to build the granule... #Eventually Taxonomy and TaxyTool will go away, to be replaced with ParameterDictionary from coverage-model #For now, keep Taxonomy stuff in until everyone can re-work their code. if param_dictionary: self._param_dict = param_dictionary self._tx = None elif isinstance(taxonomy, TaxyTool): self._tx = taxonomy self._param_dict = None elif isinstance(taxonomy, Taxonomy): self._tx = TaxyTool(taxonomy) self._param_dict = None else: raise TypeError('Invalid taxonomy argument, received type "%s"; should be ParameterDictionary, Taxonomy or TaxyTool' % type(taxonomy))
def validate_mpl_graphs_transform_results(self, results): cc = self.container assertions = self.assertTrue # if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases if isinstance(results,Granule): results =[results] for g in results: if isinstance(g,Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) graphs = get_safe(rdt, 'matplotlib_graphs') if graphs == None: continue for graph in graphs[0]: # At this point only dictionaries containing image data should be passed # For some reason non dictionary values are filtering through. if not isinstance(graph, dict): continue assertions(graph['viz_product_type'] == 'matplotlib_graphs' ) # check to see if the list (numpy array) contains actual images assertions(imghdr.what(graph['image_name'], h = graph['image_obj']) == 'png')
def validate_google_dt_transform_results(self, results): cc = self.container assertions = self.assertTrue # if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases if isinstance(results,Granule): results =[results] for g in results: if isinstance(g,Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) gdt_data = get_safe(rdt, 'google_dt_components') # IF this granule does not contains google dt, skip if gdt_data == None: continue gdt = gdt_data[0] assertions(gdt['viz_product_type'] == 'google_dt' ) assertions(len(gdt['data_description']) >= 0) # Need to come up with a better check assertions(len(gdt['data_content']) >= 0)
def test_combine_granule(self): tt = TaxyTool() tt.add_taxonomy_set('a') rdt = RecordDictionaryTool(tt) rdt['a'] = np.array([1,2,3]) granule1 = build_granule('test',tt,rdt) rdt = RecordDictionaryTool(tt) rdt['a'] = np.array([4,5,6]) granule2 = build_granule('test',tt,rdt) granule3 = combine_granules(granule1,granule2) rdt = RecordDictionaryTool.load_from_granule(granule3) self.assertTrue(np.allclose(rdt['a'],np.array([1,2,3,4,5,6])))
def combine_granules(granule_a, granule_b): """ This is a method that combines granules in a very naive manner """ validate_is_instance(granule_a,Granule, 'granule_a is not a proper Granule') validate_is_instance(granule_b,Granule, 'granule_b is not a proper Granule') tt_a = TaxyTool.load_from_granule(granule_a) tt_b = TaxyTool.load_from_granule(granule_b) if tt_a != tt_b: raise BadRequest('Can\'t combine the two granules, they do not have the same taxonomy.') rdt_new = RecordDictionaryTool(tt_a) rdt_a = RecordDictionaryTool.load_from_granule(granule_a) rdt_b = RecordDictionaryTool.load_from_granule(granule_b) for k in rdt_a.iterkeys(): rdt_new[k] = np.append(rdt_a[k], rdt_b[k]) return build_granule(granule_a.data_producer_id, tt_a, rdt_new)
def setUp(self): self._tx = TaxyTool() self._tx.add_taxonomy_set('temp', 'long_temp_name') self._tx.add_taxonomy_set('cond', 'long_cond_name') self._tx.add_taxonomy_set('pres', 'long_pres_name') self._tx.add_taxonomy_set('rdt') self._tx.add_taxonomy_set('rdt2') # map is {<local name>: <granule name or path>} self._rdt = RecordDictionaryTool(taxonomy=self._tx) self._pdict = ParameterDictionary() t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) t_ctxt.reference_frame = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' self._pdict.add_context(t_ctxt) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lat_ctxt.reference_frame = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' self._pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lon_ctxt.reference_frame = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' self._pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' self._pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) cond_ctxt.uom = 'unknown' self._pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext('pres', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) pres_ctxt.uom = 'unknown' self._pdict.add_context(pres_ctxt) self._rdt_pdict = RecordDictionaryTool(param_dictionary=self._pdict)
from ion.services.dm.utility.granule.taxonomy import TaxyTool from ion.services.dm.utility.granule.record_dictionary import RecordDictionaryTool from pyon.util.containers import get_safe from prototype.sci_data.stream_defs import SBE37_CDM_stream_definition, SBE37_RAW_stream_definition from prototype.sci_data.stream_parser import PointSupplementStreamParser from prototype.sci_data.constructor_apis import PointSupplementConstructor, RawSupplementConstructor import StringIO from numpy import array, append # Matplotlib related imports from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas from matplotlib.figure import Figure tx = TaxyTool() tx.add_taxonomy_set("matplotlib_graphs", "Matplotlib generated graphs for a particular data product") class VizTransformMatplotlibGraphs(TransformFunction): """ This class is used for instantiating worker processes that have subscriptions to data streams and convert incoming data from CDM format to Matplotlib graphs """ # Need to extract the incoming stream def automatically outgoing_stream_def = SBE37_RAW_stream_definition() incoming_stream_def = SBE37_CDM_stream_definition()
class RecordDictionaryTool(object): """ A granule is a unit of information which conveys part of a coverage. It is composed of a taxonomy and a nested structure of record dictionaries. The record dictionary is composed of named value sequences and other nested record dictionaries. The fact that all of the keys in the record dictionary are handles mapped by the taxonomy should never be exposed. The application user refers to everything in the record dictionary by the unique nick name from the taxonomy. """ def __init__(self, taxonomy=None, param_dictionary=None, shape=None): """ @brief Initialize a new instance of Record Dictionary Tool with a taxonomy and an optional fixed length @param taxonomy is an instance of a TaxonomyTool or Taxonomy (IonObject) used in this record dictionary @param length is an optional fixed length for the value sequences of this record dictionary """ if not isinstance(shape, (_NoneType, int, tuple)): raise TypeError('Invalid shape argument, received type "%s"; should be None or int or tuple' % type(shape)) self._rd = {} self._shp = shape if isinstance(self._shp, int): self._shp = (self._shp,) # hold onto the taxonomy - we need it to build the granule... #Eventually Taxonomy and TaxyTool will go away, to be replaced with ParameterDictionary from coverage-model #For now, keep Taxonomy stuff in until everyone can re-work their code. if param_dictionary: self._param_dict = param_dictionary self._tx = None elif isinstance(taxonomy, TaxyTool): self._tx = taxonomy self._param_dict = None elif isinstance(taxonomy, Taxonomy): self._tx = TaxyTool(taxonomy) self._param_dict = None else: raise TypeError('Invalid taxonomy argument, received type "%s"; should be ParameterDictionary, Taxonomy or TaxyTool' % type(taxonomy)) @classmethod def load_from_granule(cls, g): """ @brief return an instance of Record Dictionary Tool from a granule. Used when a granule is received in a message """ if g.param_dictionary: result = cls(param_dictionary=ParameterDictionary.load(g.param_dictionary)) else: result = cls(TaxyTool(g.taxonomy)) result._rd = g.record_dictionary if result._rd.has_key(0): result._shp = result._rd[0].shape return result def __setitem__(self, name, vals): """ Set an item by nick name in the record dictionary """ #This if block is used if RecordDictionaryTool was initialized using a ParameterDictionary. #This is the long term solution using the coverage-model. if self._param_dict: if isinstance(vals, RecordDictionaryTool): assert vals._param_dict == self._param_dict self._rd[self._param_dict.ord_from_key(param_name=name)] = vals._rd elif isinstance(vals, numpy.ndarray): #Otherwise it is a value sequence which should have the correct length # Matthew says: Assert only equal shape 5/17/12 if vals.ndim == 0: raise ValueError('The rank of a value sequence array in a record dictionary must be greater than zero. Got name "%s" with rank "%d"' % (name, vals.ndim)) # Set _shp if it is None if self._shp is None: self._shp = vals.shape # # Test new value sequence length # if self._shp != vals.shape: # raise ValueError('Invalid array shape "%s" for name "%s"; Record dictionary defined shape is "%s"' % (vals.shape, name, self._shp)) self._rd[self._param_dict.ord_from_key(param_name=name)] = vals else: raise TypeError('Invalid type "%s" in Record Dictionary Tool setitem with name "%s". Valid types are numpy.ndarray and RecordDictionaryTool' % (type(vals),name)) #This if block is used if RecordDictionaryTool was initialized using a Taxonomy or TaxyTool #This block will eventually be removed. if self._tx: if isinstance(vals, RecordDictionaryTool): assert vals._tx == self._tx self._rd[self._tx.get_handle(name)] = vals._rd elif isinstance(vals, numpy.ndarray): #Otherwise it is a value sequence which should have the correct length # Matthew says: Assert only equal shape 5/17/12 if vals.ndim == 0: raise ValueError('The rank of a value sequence array in a record dictionary must be greater than zero. Got name "%s" with rank "%d"' % (name, vals.ndim)) # Set _shp if it is None if self._shp is None: self._shp = vals.shape # # Test new value sequence length # if self._shp != vals.shape: # raise ValueError('Invalid array shape "%s" for name "%s"; Record dictionary defined shape is "%s"' % (vals.shape, name, self._shp)) self._rd[self._tx.get_handle(name)] = vals else: raise TypeError('Invalid type "%s" in Record Dictionary Tool setitem with name "%s". Valid types are numpy.ndarray and RecordDictionaryTool' % (type(vals),name)) def __getitem__(self, name): """ Get an item by nick name from the record dictionary. """ #This if block is used if RecordDictionaryTool was initialized using a ParameterDictionary. #This is the long term solution using the coverage-model. if self._param_dict: if isinstance(self._rd[self._param_dict.ord_from_key(param_name=name)], dict): result = RecordDictionaryTool(taxonomy=self._param_dict) result._rd = self._rd[self._param_dict.ord_from_key(param_name=name)] return result else: return self._rd[self._param_dict.ord_from_key(param_name=name)] #This if block is used if RecordDictionaryTool was initialized using a Taxonomy or TaxyTool #This block will eventually be removed. if self._tx: if isinstance(self._rd[self._tx.get_handle(name)], dict): result = RecordDictionaryTool(taxonomy=self._tx) result._rd = self._rd[self._tx.get_handle(name)] return result else: return self._rd[self._tx.get_handle(name)] def iteritems(self): """ D.iteritems() -> an iterator over the (key, value) items of D """ #This if block is used if RecordDictionaryTool was initialized using a ParameterDictionary. #This is the long term solution using the coverage-model. if self._param_dict: for k, v in self._rd.iteritems(): if isinstance(v, dict): result = RecordDictionaryTool(param_dictionary=self._param_dict) result._rd = v yield self._param_dict.key_from_ord(k), result else: yield self._param_dict.key_from_ord(k), v #This if block is used if RecordDictionaryTool was initialized using a Taxonomy or TaxyTool #This block will eventually be removed. if self._tx: for k, v in self._rd.iteritems(): if isinstance(v, dict): result = RecordDictionaryTool(taxonomy=self._tx) result._rd = v yield self._tx.get_nick_name(k), result else: yield self._tx.get_nick_name(k), v def iterkeys(self): """ D.iterkeys() -> an iterator over the keys of D """ #This if block is used if RecordDictionaryTool was initialized using a ParameterDictionary. #This is the long term solution using the coverage-model. if self._param_dict: for k in self._rd.iterkeys(): yield self._param_dict.key_from_ord(k) #This if block is used if RecordDictionaryTool was initialized using a Taxonomy or TaxyTool #This block will eventually be removed. if self._tx: for k in self._rd.iterkeys(): yield self._tx.get_nick_name(k) def itervalues(self): """ D.itervalues() -> an iterator over the values of D """ #This if block is used if RecordDictionaryTool was initialized using a ParameterDictionary. #This is the long term solution using the coverage-model. if self._param_dict: for v in self._rd.itervalues(): if isinstance(v, dict): result = RecordDictionaryTool(taxonomy=self._param_dict) result._rd = v yield result else: yield v #This if block is used if RecordDictionaryTool was initialized using a Taxonomy or TaxyTool #This block will eventually be removed. if self._tx: for v in self._rd.itervalues(): if isinstance(v, dict): result = RecordDictionaryTool(taxonomy=self._tx) result._rd = v yield result else: yield v #TJG - This may need to be updated def update(self, E=None, **F): """ @brief Dictionary update method exposed for Record Dictionaries @param E is another record dictionary @param F is a dictionary of nicknames and value sequences """ if E: if hasattr(E, "keys"): for k in E: self[k] = E[k] else: for k, v in E.iteritems(): self[k] = v if F: for k in F.keys(): self[k] = F[k] def __contains__(self, nick_name): """ D.__contains__(k) -> True if D has a key k, else False """ handle = '' try: if self._param_dict: handle = self._param_dict.ord_from_key(nick_name) if self._tx: handle = self._tx.get_handle(nick_name) except KeyError as ke: # if the nick_name is not in the taxonomy, it is certainly not in the record dictionary return False return handle in self._rd def __delitem__(self, y): """ x.__delitem__(y) <==> del x[y] """ #not sure if this is right, might just have to remove the name, not the whole handle if self._param_dict: del self._rd[self._param_dict.ord_from_key(y)] if self._tx: del self._rd[self._tx.get_handle(y)] #will probably need to delete the name from _tx def __iter__(self): """ x.__iter__() <==> iter(x) """ return self.iterkeys() def __len__(self): """ x.__len__() <==> len(x) """ return len(self._rd) def __repr__(self): """ x.__repr__() <==> repr(x) """ result = "{" for k, v in self.iteritems(): result += "\'{0}\': {1},".format(k, v) if len(result) > 1: result = result[:-1] + "}" return result def __str__(self): result = "{" for k, v in self.iteritems(): result += "\'{0}\': {1},".format(k, v) if len(result) > 1: result = result[:-1] + "}" return result __hash__ = None def pretty_print(self): """ @brief Pretty Print the record dictionary for debug or log purposes. """ pass # fid = StringIO.StringIO() # # Use string IO inside a try block in case of exceptions or a large return value. # try: # fid.write('Start Pretty Print Record Dictionary:\n') # self._pprint(fid,offset='') # fid.write('End of Pretty Print') # except Exception, ex: # log.exception('Unexpected Exception in Pretty Print Wrapper!') # fid.write('Exception! %s' % ex) # # finally: # result = fid.getvalue() # fid.close() # # # return result def _pprint(self, fid, offset=None): pass
import random import math from prototype.sci_data.stream_defs import ctd_stream_packet, SBE37_CDM_stream_definition, ctd_stream_definition from prototype.sci_data.constructor_apis import PointSupplementConstructor from interface.services.dm.ipubsub_management_service import PubsubManagementServiceClient from ion.processes.data.ctd_stream_publisher import SimpleCtdPublisher from ion.services.dm.utility.granule.record_dictionary import RecordDictionaryTool from ion.services.dm.utility.granule.taxonomy import TaxyTool from ion.services.dm.utility.granule.granule import build_granule import numpy ### Taxonomies are defined before hand out of band... somehow. tx = TaxyTool() tx.add_taxonomy_set('temp','long name for temp') tx.add_taxonomy_set('cond','long name for cond') tx.add_taxonomy_set('lat','long name for latitude') tx.add_taxonomy_set('lon','long name for longitude') tx.add_taxonomy_set('pres','long name for pres') tx.add_taxonomy_set('time','long name for time') tx.add_taxonomy_set('height','long name for height') class SinusoidalCtdPublisher(SimpleCtdPublisher): def __init__(self, *args, **kwargs): super(SinusoidalCtdPublisher, self).__init__(*args,**kwargs) #@todo Init stuff
def get_taxonomy(stream_name): """ Gets the taxonomy for a given stream. @param stream_name Stream name. @retval corresponding taxonomy. """ if stream_name in _cache: log.debug("taxonomy for stream_name=%s found in cache" % stream_name) return _cache[stream_name] taxy = None # TODO determine and use appropriate mechanism for definition of # taxonomies. The following hard-coded definitions are taken from # preliminary code in instrument_management_service.py as of 7/15/12. if "parsed" == stream_name: taxy = TaxyTool() taxy.add_taxonomy_set('temp', 't', 'long name for temp') taxy.add_taxonomy_set('cond', 'c', 'long name for cond') taxy.add_taxonomy_set('pres', 'p', 'long name for pres') taxy.add_taxonomy_set('lat','long name for latitude') taxy.add_taxonomy_set('lon','long name for longitude') taxy.add_taxonomy_set('time','long name for time') taxy.add_taxonomy_set('height','long name for height') # This is an example of using groups it is not a normative statement about how to use groups taxy.add_taxonomy_set('coordinates','This group contains coordinates...') taxy.add_taxonomy_set('data','This group contains data...') elif "raw" == stream_name: taxy = TaxyTool() taxy.add_taxonomy_set('blob','bytes in an array') taxy.add_taxonomy_set('lat','long name for latitude') taxy.add_taxonomy_set('lon','long name for longitude') taxy.add_taxonomy_set('time','long name for time') taxy.add_taxonomy_set('height','long name for height') taxy.add_taxonomy_set('coordinates','This group contains coordinates...') taxy.add_taxonomy_set('data','This group contains data...') else: log.warn("Undefined taxonomy for stream_name=%s" % stream_name) if taxy: _cache[stream_name] = taxy log.debug("created taxonomy for stream_name=%s " % stream_name) return taxy
def setUp(self): """ Setup test cases. """ parsed_tx = TaxyTool() parsed_tx.add_taxonomy_set('temp', 'long name for temp', 't') parsed_tx.add_taxonomy_set('cond', 'long name for cond', 'c') parsed_tx.add_taxonomy_set('pres', 'long name for pres', 'd') parsed_tx.add_taxonomy_set('lat', 'long name for latitude', 'lt') parsed_tx.add_taxonomy_set('lon', 'long name for longitude', 'ln') parsed_tx.add_taxonomy_set('time', 'long name for time', 'tm') # This is an example of using groups it is not a normative statement about how to use groups parsed_tx.add_taxonomy_set('coordinates', 'This group contains coordinates...') parsed_tx.add_taxonomy_set('data', 'This group contains data...') self._parsed_taxonomy = parsed_tx raw_tx = TaxyTool() raw_tx.add_taxonomy_set('raw_fixed', 'Fixed length bytes in an array of records') raw_tx.add_taxonomy_set('raw_blob', 'Unlimited length bytes in an array') self._raw_taxonomy = raw_tx log.debug("_parsed_taxonomy = %s" % self._parsed_taxonomy.dump()) log.debug("_raw_taxonomy = %s" % self._raw_taxonomy.dump()) self.packet_factory = PacketFactory.get_packet_factory( PacketFactoryType.R2LCAFormat) self.assertIsInstance(self.packet_factory, LCAPacketFactory)
def test_activateDatasetAgent(self): # Create ExternalDatasetModel datsetModel_obj = IonObject( RT.ExternalDatasetModel, name="ExampleDatasetModel", description="ExampleDatasetModel", datset_type="FibSeries", ) try: datasetModel_id = self.damsclient.create_external_dataset_model(datsetModel_obj) except BadRequest as ex: self.fail("failed to create new ExternalDatasetModel: %s" % ex) log.debug("TestExternalDatasetAgentMgmt: new ExternalDatasetModel id = %s", str(datasetModel_id)) # Create ExternalDatasetAgent datasetAgent_obj = IonObject( RT.ExternalDatasetAgent, name="datasetagent007", description="datasetagent007", handler_module=EDA_MOD, handler_class=EDA_CLS, ) try: datasetAgent_id = self.damsclient.create_external_dataset_agent(datasetAgent_obj, datasetModel_id) except BadRequest as ex: self.fail("failed to create new ExternalDatasetAgent: %s" % ex) log.debug("TestExternalDatasetAgentMgmt: new ExternalDatasetAgent id = %s", str(datasetAgent_id)) # Create ExternalDataset log.debug("TestExternalDatasetAgentMgmt: Create external dataset resource ") extDataset_obj = IonObject(RT.ExternalDataset, name="ExtDataset", description="ExtDataset") try: extDataset_id = self.damsclient.create_external_dataset(extDataset_obj, datasetModel_id) except BadRequest as ex: self.fail("failed to create new external dataset resource: %s" % ex) log.debug("TestExternalDatasetAgentMgmt: new ExternalDataset id = %s ", str(extDataset_id)) # register the dataset as a data producer dproducer_id = self.damsclient.register_external_data_set(extDataset_id) # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name="SBE37_CDM", parameter_dictionary_id=pdict_id) log.debug("TestExternalDatasetAgentMgmt: new Stream Definition id = %s", str(ctd_stream_def_id)) log.debug("TestExternalDatasetAgentMgmt: Creating new data product with a stream definition") dp_obj = IonObject(RT.DataProduct, name="eoi dataset data", description=" stream test") tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject( RT.DataProduct, name="DP1", description="some new dp", temporal_domain=tdom, spatial_domain=sdom ) data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id) log.debug("TestExternalDatasetAgentMgmt: new dp_id = %s", str(data_product_id1)) self.damsclient.assign_data_product(input_resource_id=extDataset_id, data_product_id=data_product_id1) # todo fix the problem here.... self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug("TestExternalDatasetAgentMgmt: Data product streams1 = %s", str(stream_ids)) stream_id = stream_ids[0] # Build a taxonomy for the dataset tx = TaxyTool() tx.add_taxonomy_set("data", "external_data") # Augment the DVR_CONFIG with the necessary pieces self.DVR_CONFIG["dh_cfg"] = { "TESTING": True, "stream_id": stream_id, # TODO: This should probably be a 'stream_config' dict with stream_name:stream_id members "data_producer_id": dproducer_id, # 'external_dataset_res':extDataset_obj, # Not needed - retrieved by EDA based on resource_id "taxonomy": tx.dump(), # TODO: Currently does not support sets "max_records": 4, } # Create agent config. self._stream_config = {} agent_config = { "driver_config": self.DVR_CONFIG, "stream_config": self._stream_config, "agent": {"resource_id": EDA_RESOURCE_ID}, "test_mode": True, } extDatasetAgentInstance_obj = IonObject( RT.ExternalDatasetAgentInstance, name="DatasetAgentInstance", description="DatasetAgentInstance", dataset_driver_config=self.DVR_CONFIG, dataset_agent_config=agent_config, ) extDatasetAgentInstance_id = self.damsclient.create_external_dataset_agent_instance( external_dataset_agent_instance=extDatasetAgentInstance_obj, external_dataset_agent_id=datasetAgent_id, external_dataset_id=extDataset_id, ) log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(extDatasetAgentInstance_obj)) log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s", str(extDatasetAgentInstance_id)) # Check that the instance is currently not active id, active = self.damsclient.retrieve_external_dataset_agent_instance(extDataset_id) log.debug( "TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 1 = %s ", str(id), str(active) ) self.damsclient.start_external_dataset_agent_instance(extDatasetAgentInstance_id) dataset_agent_instance_obj = self.damsclient.read_external_dataset_agent_instance(extDatasetAgentInstance_id) log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(dataset_agent_instance_obj)) # now the instance process should be active id, active = self.damsclient.retrieve_external_dataset_agent_instance(extDataset_id) log.debug( "TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 2 = %s ", str(id), str(active) ) # Start a resource agent client to talk with the instrument agent. self._dsa_client = ResourceAgentClient(extDataset_id, process=FakeProcess()) print "TestExternalDatasetAgentMgmt: got ia client %s", self._dsa_client log.debug("TestExternalDatasetAgentMgmt: got dataset client %s", str(self._dsa_client)) # cmd=AgentCommand(command='initialize') # _ = self._dsa_client.execute_agent(cmd) # # cmd = AgentCommand(command='go_active') # _ = self._dsa_client.execute_agent(cmd) # # cmd = AgentCommand(command='run') # _ = self._dsa_client.execute_agent(cmd) # # log.info('Send an unconstrained request for data (\'new data\')') # cmd = AgentCommand(command='acquire_data') # self._dsa_client.execute(cmd) # # log.info('Send a second unconstrained request for data (\'new data\'), should be rejected') # cmd = AgentCommand(command='acquire_data') # self._dsa_client.execute(cmd) # # cmd = AgentCommand(command='reset') # _ = self._dsa_client.execute_agent(cmd) # cmd = AgentCommand(command='get_current_state') # retval = self._dsa_client.execute_agent(cmd) # state = retval.result # TODO: Think about what we really should be testing at this point # The following is taken from ion.agents.data.test.test_external_dataset_agent.ExternalDatasetAgentTestBase.test_states() # TODO: Do we also need to show data retrieval? cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) cmd = AgentCommand(command="initialize") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) cmd = AgentCommand(command="go_active") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command="run") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command="pause") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STOPPED) cmd = AgentCommand(command="resume") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command="clear") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command="run") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command="pause") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STOPPED) cmd = AgentCommand(command="clear") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command="run") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command="reset") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) # ------------------------------- # Deactivate InstrumentAgentInstance # ------------------------------- self.damsclient.stop_external_dataset_agent_instance(extDatasetAgentInstance_id)
from pyon.core.exception import BadRequest from pyon.public import IonObject, RT, log from datetime import datetime import numpy from ion.services.dm.utility.granule.granule import build_granule from ion.services.dm.utility.granule.taxonomy import TaxyTool from ion.services.dm.utility.granule.record_dictionary import RecordDictionaryTool from prototype.sci_data.stream_defs import SBE37_CDM_stream_definition, SBE37_RAW_stream_definition from prototype.sci_data.stream_parser import PointSupplementStreamParser from prototype.sci_data.constructor_apis import PointSupplementConstructor, RawSupplementConstructor from pyon.util.containers import get_safe tx = TaxyTool() tx.add_taxonomy_set('google_dt_components','Google datatable') class VizTransformGoogleDT(TransformFunction): """ This class is used for converting incoming data from CDM format to JSON style Google DataTables Note: One behaviour that this class is expected to achieve specifically is to determine if its supposed to work as a realtime transform (exists indefinitely and maintains a sliding window of data) or as a replay transform (one-shot). [2] This transform behaves as an instantaneous forwarder. There is no waiting for the entire stream to create the complete datatable. As the granules come in, they are translated to the datatable 'components'. Components, because we are not creating the actual datatable in this code. That's the job
def setUp(self): """ Setup test cases. """ parsed_tx = TaxyTool() parsed_tx.add_taxonomy_set('temp','long name for temp', 't') parsed_tx.add_taxonomy_set('cond','long name for cond', 'c') parsed_tx.add_taxonomy_set('pres','long name for pres', 'd') parsed_tx.add_taxonomy_set('lat','long name for latitude', 'lt') parsed_tx.add_taxonomy_set('lon','long name for longitude', 'ln') parsed_tx.add_taxonomy_set('time','long name for time', 'tm') # This is an example of using groups it is not a normative statement about how to use groups parsed_tx.add_taxonomy_set('coordinates','This group contains coordinates...') parsed_tx.add_taxonomy_set('data','This group contains data...') self._parsed_taxonomy = parsed_tx raw_tx = TaxyTool() raw_tx.add_taxonomy_set('raw_fixed','Fixed length bytes in an array of records') raw_tx.add_taxonomy_set('raw_blob','Unlimited length bytes in an array') self._raw_taxonomy = raw_tx log.debug("_parsed_taxonomy = %s" % self._parsed_taxonomy.dump()) log.debug("_raw_taxonomy = %s" % self._raw_taxonomy.dump()) self.packet_factory = PacketFactory.get_packet_factory( PacketFactoryType.R2LCAFormat) self.assertIsInstance(self.packet_factory, LCAPacketFactory)
def test_activateDatasetAgent(self): # Create ExternalDatasetModel datsetModel_obj = IonObject(RT.ExternalDatasetModel, name='ExampleDatasetModel', description="ExampleDatasetModel", datset_type="FibSeries") try: datasetModel_id = self.damsclient.create_external_dataset_model( datsetModel_obj) except BadRequest as ex: self.fail("failed to create new ExternalDatasetModel: %s" % ex) log.debug( "TestExternalDatasetAgentMgmt: new ExternalDatasetModel id = %s", str(datasetModel_id)) # Create ExternalDatasetAgent datasetAgent_obj = IonObject(RT.ExternalDatasetAgent, name='datasetagent007', description="datasetagent007", handler_module=EDA_MOD, handler_class=EDA_CLS) try: datasetAgent_id = self.damsclient.create_external_dataset_agent( datasetAgent_obj, datasetModel_id) except BadRequest as ex: self.fail("failed to create new ExternalDatasetAgent: %s" % ex) log.debug( "TestExternalDatasetAgentMgmt: new ExternalDatasetAgent id = %s", str(datasetAgent_id)) # Create ExternalDataset log.debug( 'TestExternalDatasetAgentMgmt: Create external dataset resource ') extDataset_obj = IonObject(RT.ExternalDataset, name='ExtDataset', description="ExtDataset") try: extDataset_id = self.damsclient.create_external_dataset( extDataset_obj, datasetModel_id) except BadRequest as ex: self.fail("failed to create new external dataset resource: %s" % ex) log.debug( "TestExternalDatasetAgentMgmt: new ExternalDataset id = %s ", str(extDataset_id)) #register the dataset as a data producer dproducer_id = self.damsclient.register_external_data_set( extDataset_id) # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='SBE37_CDM', parameter_dictionary_id=pdict_id) log.debug( "TestExternalDatasetAgentMgmt: new Stream Definition id = %s", str(ctd_stream_def_id)) log.debug( "TestExternalDatasetAgentMgmt: Creating new data product with a stream definition" ) dp_obj = IonObject(RT.DataProduct, name='eoi dataset data', description=' stream test') dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') data_product_id1 = self.dpclient.create_data_product( dp_obj, ctd_stream_def_id) log.debug("TestExternalDatasetAgentMgmt: new dp_id = %s", str(data_product_id1)) self.damsclient.assign_data_product(input_resource_id=extDataset_id, data_product_id=data_product_id1) #todo fix the problem here.... self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug("TestExternalDatasetAgentMgmt: Data product streams1 = %s", str(stream_ids)) stream_id = stream_ids[0] # Build a taxonomy for the dataset tx = TaxyTool() tx.add_taxonomy_set('data', 'external_data') # Augment the DVR_CONFIG with the necessary pieces self.DVR_CONFIG['dh_cfg'] = { 'TESTING': True, 'stream_id': stream_id, #TODO: This should probably be a 'stream_config' dict with stream_name:stream_id members 'data_producer_id': dproducer_id, # 'external_dataset_res':extDataset_obj, # Not needed - retrieved by EDA based on resource_id 'taxonomy': tx.dump(), #TODO: Currently does not support sets 'max_records': 4, } # Create agent config. self._stream_config = {} agent_config = { 'driver_config': self.DVR_CONFIG, 'stream_config': self._stream_config, 'agent': { 'resource_id': EDA_RESOURCE_ID }, 'test_mode': True } extDatasetAgentInstance_obj = IonObject( RT.ExternalDatasetAgentInstance, name='DatasetAgentInstance', description="DatasetAgentInstance", dataset_driver_config=self.DVR_CONFIG, dataset_agent_config=agent_config) extDatasetAgentInstance_id = self.damsclient.create_external_dataset_agent_instance( external_dataset_agent_instance=extDatasetAgentInstance_obj, external_dataset_agent_id=datasetAgent_id, external_dataset_id=extDataset_id) log.debug( "TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(extDatasetAgentInstance_obj)) log.debug( "TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s", str(extDatasetAgentInstance_id)) #Check that the instance is currently not active id, active = self.damsclient.retrieve_external_dataset_agent_instance( extDataset_id) log.debug( "TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 1 = %s ", str(id), str(active)) self.damsclient.start_external_dataset_agent_instance( extDatasetAgentInstance_id) dataset_agent_instance_obj = self.damsclient.read_external_dataset_agent_instance( extDatasetAgentInstance_id) log.debug( "TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(dataset_agent_instance_obj)) # now the instance process should be active id, active = self.damsclient.retrieve_external_dataset_agent_instance( extDataset_id) log.debug( "TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 2 = %s ", str(id), str(active)) # Start a resource agent client to talk with the instrument agent. self._dsa_client = ResourceAgentClient(extDataset_id, process=FakeProcess()) print 'TestExternalDatasetAgentMgmt: got ia client %s', self._dsa_client log.debug("TestExternalDatasetAgentMgmt: got dataset client %s", str(self._dsa_client)) # cmd=AgentCommand(command='initialize') # _ = self._dsa_client.execute_agent(cmd) # # cmd = AgentCommand(command='go_active') # _ = self._dsa_client.execute_agent(cmd) # # cmd = AgentCommand(command='run') # _ = self._dsa_client.execute_agent(cmd) # # log.info('Send an unconstrained request for data (\'new data\')') # cmd = AgentCommand(command='acquire_data') # self._dsa_client.execute(cmd) # # log.info('Send a second unconstrained request for data (\'new data\'), should be rejected') # cmd = AgentCommand(command='acquire_data') # self._dsa_client.execute(cmd) # # cmd = AgentCommand(command='reset') # _ = self._dsa_client.execute_agent(cmd) # cmd = AgentCommand(command='get_current_state') # retval = self._dsa_client.execute_agent(cmd) # state = retval.result # TODO: Think about what we really should be testing at this point # The following is taken from ion.agents.data.test.test_external_dataset_agent.ExternalDatasetAgentTestBase.test_states() # TODO: Do we also need to show data retrieval? cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) cmd = AgentCommand(command='initialize') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) cmd = AgentCommand(command='go_active') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command='pause') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STOPPED) cmd = AgentCommand(command='resume') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command='clear') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command='pause') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STOPPED) cmd = AgentCommand(command='clear') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command='reset') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.damsclient.stop_external_dataset_agent_instance( extDatasetAgentInstance_id)
def setUp(self): """ Setup test cases. """ parsed_tx = TaxyTool() parsed_tx.add_taxonomy_set("temp", "long name for temp", "t") parsed_tx.add_taxonomy_set("cond", "long name for cond", "c") parsed_tx.add_taxonomy_set("pres", "long name for pres", "d") parsed_tx.add_taxonomy_set("lat", "long name for latitude", "lt") parsed_tx.add_taxonomy_set("lon", "long name for longitude", "ln") parsed_tx.add_taxonomy_set("time", "long name for time", "tm") # This is an example of using groups it is not a normative statement about how to use groups parsed_tx.add_taxonomy_set("coordinates", "This group contains coordinates...") parsed_tx.add_taxonomy_set("data", "This group contains data...") self._parsed_taxonomy = parsed_tx raw_tx = TaxyTool() raw_tx.add_taxonomy_set("raw_fixed", "Fixed length bytes in an array of records") raw_tx.add_taxonomy_set("raw_blob", "Unlimited length bytes in an array") self._raw_taxonomy = raw_tx log.debug("_parsed_taxonomy = %s" % self._parsed_taxonomy.dump()) log.debug("_raw_taxonomy = %s" % self._raw_taxonomy.dump()) self.packet_factory = PacketFactory.get_packet_factory(PacketFactoryType.R2LCAFormat) self.assertIsInstance(self.packet_factory, LCAPacketFactory)
def get_taxonomy(stream_name): """ Gets the taxonomy for a given stream. @param stream_name Stream name. @retval corresponding taxonomy. """ if stream_name in _cache: log.debug("taxonomy for stream_name=%s found in cache" % stream_name) return _cache[stream_name] taxy = None # TODO determine and use appropriate mechanism for definition of # taxonomies. The following hard-coded definitions are taken from # preliminary code in instrument_management_service.py as of 7/15/12. if "parsed" == stream_name: taxy = TaxyTool() taxy.add_taxonomy_set('temp', 't', 'long name for temp') taxy.add_taxonomy_set('cond', 'c', 'long name for cond') taxy.add_taxonomy_set('pres', 'p', 'long name for pres') taxy.add_taxonomy_set('lat', 'long name for latitude') taxy.add_taxonomy_set('lon', 'long name for longitude') taxy.add_taxonomy_set('time', 'long name for time') taxy.add_taxonomy_set('height', 'long name for height') # This is an example of using groups it is not a normative statement about how to use groups taxy.add_taxonomy_set('coordinates', 'This group contains coordinates...') taxy.add_taxonomy_set('data', 'This group contains data...') elif "raw" == stream_name: taxy = TaxyTool() taxy.add_taxonomy_set('blob', 'bytes in an array') taxy.add_taxonomy_set('lat', 'long name for latitude') taxy.add_taxonomy_set('lon', 'long name for longitude') taxy.add_taxonomy_set('time', 'long name for time') taxy.add_taxonomy_set('height', 'long name for height') taxy.add_taxonomy_set('coordinates', 'This group contains coordinates...') taxy.add_taxonomy_set('data', 'This group contains data...') else: log.warn("Undefined taxonomy for stream_name=%s" % stream_name) if taxy: _cache[stream_name] = taxy log.debug("created taxonomy for stream_name=%s " % stream_name) return taxy
import numpy import gevent from prototype.sci_data.stream_defs import ctd_stream_packet, SBE37_CDM_stream_definition, ctd_stream_definition from prototype.sci_data.constructor_apis import PointSupplementConstructor from interface.services.dm.ipubsub_management_service import PubsubManagementServiceClient ### For new granule and stream interface from pyon.ion.transforma import TransformStreamPublisher from ion.services.dm.utility.granule.record_dictionary import RecordDictionaryTool from ion.services.dm.utility.granule.taxonomy import TaxyTool from ion.services.dm.utility.granule.granule import build_granule ### Taxonomies are defined before hand out of band... somehow. tx = TaxyTool() tx.add_taxonomy_set('temp','long name for temp') tx.add_taxonomy_set('cond','long name for cond') tx.add_taxonomy_set('lat','long name for latitude') tx.add_taxonomy_set('lon','long name for longitude') tx.add_taxonomy_set('pres','long name for pres') tx.add_taxonomy_set('time','long name for time') tx.add_taxonomy_set('height','long name for height') tx.add_taxonomy_set('coordinates','This group contains coordinates...') tx.add_taxonomy_set('data','This group contains data...') class SimpleCtdPublisher(TransformStreamPublisher): def on_start(self): exchange_point = self.CFG.get_safe('process.exchange_point', 'science_data') self.CFG.process.exchange_point = exchange_point
def test_createTransformsThenActivateInstrument(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_module='mi.instrument.seabird.sbe37smb.ooicore.driver', driver_class='SBE37Driver', comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] ) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def) log.debug( 'new Stream Definition id = %s', instDevice_id) log.debug( 'Creating new CDM data product with a stream definition') craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary) log.debug( 'new dp_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) pid = self.create_logger('ctd_parsed', stream_ids[0] ) self.loggerpids.append(pid) print 'test_createTransformsThenActivateInstrument: Data product streams1 = ', stream_ids #------------------------------- # Create CTD Raw as the second data product #------------------------------- log.debug( 'Creating new RAW data product with a stream definition') raw_stream_def = SBE37_RAW_stream_definition() raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary) log.debug( 'new dp_id = %s', str(ctd_raw_data_product)) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) log.debug( 'Data product streams2 = %s', str(stream_ids)) #todo: attaching the taxonomy to the stream is a TEMPORARY measure # Create taxonomies for both parsed and attach to the stream RawTax = TaxyTool() RawTax.add_taxonomy_set('raw_fixed','Fixed length bytes in an array of records') RawTax.add_taxonomy_set('raw_blob','Unlimited length bytes in an array') # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) print 'Data product streams2 = ', stream_ids #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all', process_source='some_source_reference') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) #------------------------------- # L1 Conductivity: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1ConductivityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform', process_source='CTDL1ConductivityTransform source code here...') try: ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex) #------------------------------- # L1 Pressure: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1PressureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform', process_source='CTDL1PressureTransform source code here...') try: ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex) #------------------------------- # L1 Temperature: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1TemperatureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform', process_source='CTDL1TemperatureTransform source code here...') try: ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex) #------------------------------- # L2 Salinity: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform', process_source='SalinityTransform source code here...') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) #------------------------------- # L2 Density: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition DensityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform', process_source='DensityTransform source code here...') try: ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new DensityTransform data process definition: %s" %ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity = L0_conductivity_stream_definition() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_pressure = L0_pressure_stream_definition() outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_temperature = L0_temperature_stream_definition() outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id ) self.output_products={} log.debug("test_createTransformsThenActivateInstrument: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id, parameter_dictionary) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id) log.debug("test_createTransformsThenActivateInstrument: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id, parameter_dictionary) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id) log.debug("test_createTransformsThenActivateInstrument: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id, parameter_dictionary) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id) #------------------------------- # L1 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l1_conductivity = L1_conductivity_stream_definition() outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_conductivity, name='L1_conductivity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id ) outgoing_stream_l1_pressure = L1_pressure_stream_definition() outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_pressure, name='L1_Pressure') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id ) outgoing_stream_l1_temperature = L1_temperature_stream_definition() outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_temperature, name='L1_Temperature') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id ) log.debug("test_createTransformsThenActivateInstrument: create output data product L1 conductivity") ctd_l1_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id, parameter_dictionary) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_conductivity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_conductivity', stream_ids[0] ) self.loggerpids.append(pid) log.debug("test_createTransformsThenActivateInstrument: create output data product L1 pressure") ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id, parameter_dictionary ) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_pressure_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_pressure', stream_ids[0] ) self.loggerpids.append(pid) log.debug("test_createTransformsThenActivateInstrument: create output data product L1 temperature") ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id, parameter_dictionary) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_temperature_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_temperature', stream_ids[0] ) self.loggerpids.append(pid) #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- outgoing_stream_l2_salinity = L2_practical_salinity_stream_definition() outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_salinity, name='L2_salinity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id ) outgoing_stream_l2_density = L2_density_stream_definition() outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_density, name='L2_Density') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id ) log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Salinity") ctd_l2_salinity_output_dp_obj = IonObject(RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id, parameter_dictionary) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_salinity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_salinity', stream_ids[0] ) self.loggerpids.append(pid) log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Density") ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id, parameter_dictionary) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_density_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l2_density_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_density', stream_ids[0] ) self.loggerpids.append(pid) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process return") #------------------------------- # L1 Conductivity: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process start") try: l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'output':ctd_l1_conductivity_output_dp_id}) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process return") #------------------------------- # L1 Pressure: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start") try: l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'output':ctd_l1_pressure_output_dp_id}) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return") #------------------------------- # L1 Temperature: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start") try: l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'output':ctd_l1_temperature_output_dp_id}) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return") #------------------------------- # L2 Salinity: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process start") try: l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_salinity_output_dp_id}) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process return") #------------------------------- # L2 Density: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process start") try: l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_density_output_dp_id}) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process return") #------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) print 'test_createTransformsThenActivateInstrument: Instrument agent instance obj: = ', inst_agent_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) print 'activate_instrument: got ia client %s', self._ia_client log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client)) #------------------------------- # Streaming #------------------------------- cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: initialize %s", str(retval)) log.debug("test_activateInstrumentStream: Sending go_active command (L4-CI-SA-RQ-334)") cmd = AgentCommand(command='go_active') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: return value from go_active %s", str(reply)) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentStream: current state after sending go_active command %s (L4-CI-SA-RQ-334)", str(state)) cmd = AgentCommand(command='run') reply = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentStream: return from run state: %s", str(state)) # Make sure the sampling rate and transmission are sane. params = { SBE37Parameter.NAVG : 1, SBE37Parameter.INTERVAL : 5, SBE37Parameter.TXREALTIME : True } self._ia_client.set_param(params) log.debug("test_activateInstrumentStream: calling go_streaming ") cmd = AgentCommand(command='go_streaming') reply = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentStream: return from go_streaming state: %s", str(state)) time.sleep(7) log.debug("test_activateInstrumentStream: calling go_observatory") cmd = AgentCommand(command='go_observatory') reply = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state)) log.debug("test_activateInstrumentStream: calling reset ") cmd = AgentCommand(command='reset') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: return from reset state:%s", str(reply.result)) time.sleep(2) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid)
class RecordDictionaryToolTestCase(unittest.TestCase): def setUp(self): self._tx = TaxyTool() self._tx.add_taxonomy_set('temp', 'long_temp_name') self._tx.add_taxonomy_set('cond', 'long_cond_name') self._tx.add_taxonomy_set('pres', 'long_pres_name') self._tx.add_taxonomy_set('rdt') self._tx.add_taxonomy_set('rdt2') # map is {<local name>: <granule name or path>} self._rdt = RecordDictionaryTool(taxonomy=self._tx) self._pdict = ParameterDictionary() t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) t_ctxt.reference_frame = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' self._pdict.add_context(t_ctxt) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lat_ctxt.reference_frame = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' self._pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lon_ctxt.reference_frame = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' self._pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' self._pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) cond_ctxt.uom = 'unknown' self._pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext('pres', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) pres_ctxt.uom = 'unknown' self._pdict.add_context(pres_ctxt) self._rdt_pdict = RecordDictionaryTool(param_dictionary=self._pdict) def test_init(self): # initialize with a ParameterDictionary rdt = RecordDictionaryTool(param_dictionary=self._pdict) self.assertIsInstance(rdt._param_dict, ParameterDictionary) # initialize with nonsense self.assertRaises(TypeError, RecordDictionaryTool, ['foo', 'barr']) # initialize with a valid shape rdt = RecordDictionaryTool(param_dictionary=self._pdict, shape=(5,2)) self.assertEquals(rdt._shp, (5,2)) rdt = RecordDictionaryTool(param_dictionary=self._pdict, shape=(5,)) self.assertEquals(rdt._shp, (5,)) rdt = RecordDictionaryTool(param_dictionary=self._tx, shape=5) self.assertEquals(rdt._shp, (5,)) # initialize with no length rdt = RecordDictionaryTool(param_dictionary=self._pdict) self.assertEquals(rdt._shp, None) def test_init_with_taxonomy(self): # initialize with a taxonomy tool rdt = RecordDictionaryTool(taxonomy=self._tx) self.assertIsInstance(rdt._tx, TaxyTool) # initialize with a taxonomy object rdt = RecordDictionaryTool(taxonomy=self._tx._t) self.assertIsInstance(rdt._tx, TaxyTool) # initialize with pooo self.assertRaises(TypeError, RecordDictionaryTool, ['foo', 'barr']) # initialize with a valid shape rdt = RecordDictionaryTool(taxonomy=self._tx, shape=(5,2)) self.assertEquals(rdt._shp, (5,2)) rdt = RecordDictionaryTool(taxonomy=self._tx, shape=(5,)) self.assertEquals(rdt._shp, (5,)) rdt = RecordDictionaryTool(taxonomy=self._tx, shape=5) self.assertEquals(rdt._shp, (5,)) # initialize with no length rdt = RecordDictionaryTool(taxonomy=self._tx) self.assertEquals(rdt._shp, None) def test_set_and_get(self): #make sure you can set and get items in the granule by name in the taxonomy temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self.assertRaises(KeyError, self._rdt.__setitem__, 'long_temp_name', temp_array) self.assertRaises(KeyError, self._rdt.__setitem__, 'nonsense', temp_array) self._rdt_pdict['temp'] = temp_array self._rdt_pdict['conductivity'] = cond_array self._rdt_pdict['pres'] = pres_array self.assertTrue(numpy.allclose(self._rdt_pdict['temp'], temp_array)) self.assertTrue(numpy.allclose(self._rdt_pdict['conductivity'], cond_array)) self.assertTrue(numpy.allclose(self._rdt_pdict['pres'], pres_array)) #want to check to make sure a KeyError is raised when a non-nickname key is used, but it's not working correctly self.assertRaises(KeyError, self._rdt.__getitem__, 'long_temp_name') self.assertRaises(KeyError, self._rdt.__getitem__,'nonsense!') pdict =self._pdict rdt = RecordDictionaryTool(param_dictionary=pdict) rdt['temp'] = temp_array # Now test bad values... list not numpy array... with self.assertRaises(TypeError) as te: rdt['temp'] = [1,2,3] self.assertEquals( te.exception.message, '''Invalid type "<type 'list'>" in Record Dictionary Tool setitem with name "temp". Valid types are numpy.ndarray and RecordDictionaryTool''' ) # Now test numpy scalar array... with self.assertRaises(TypeError) as te: rdt['temp'] = numpy.float32(3.14159) self.assertEquals( te.exception.message, '''Invalid type "<type 'numpy.float32'>" in Record Dictionary Tool setitem with name "temp". Valid types are numpy.ndarray and RecordDictionaryTool''' ) # Now test rank zero array... with self.assertRaises(ValueError) as te: rdt['temp'] = numpy.array(22.5) self.assertEquals( te.exception.message, '''The rank of a value sequence array in a record dictionary must be greater than zero. Got name "temp" with rank "0"''' ) #TODO: Fix record_dictionary shape validation, then put this test back in # # Test set invalid shape # pres_array = numpy.random.standard_normal(90) # with self.assertRaises(ValueError) as te: # rdt['pres'] = pres_array # # self.assertEquals( # te.exception.message, # '''Invalid array shape "(90,)" for name "pres"; Record dictionary defined shape is "(100,)"''' # ) # make a new RDT for testing higher rank objects... rdt = RecordDictionaryTool(param_dictionary=self._pdict) # Now test rank 2 array... rdt['temp'] = numpy.array([[22.5,],]) self.assertTrue((rdt['temp'] == numpy.array([[22.5,],])).all()) # Now test rank 2 array... rdt['conductivity'] = numpy.array([[28.5,],]) self.assertTrue((rdt['conductivity'] == numpy.array([[28.5,],])).all()) def test_set_and_get_with_taxonomy(self): #make sure you can set and get items in the granule by name in the taxonomy temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self.assertRaises(KeyError, self._rdt.__setitem__, 'long_temp_name',temp_array) self.assertRaises(KeyError, self._rdt.__setitem__, 'nonsense',temp_array) self._rdt['temp'] = temp_array self._rdt['cond'] = cond_array self._rdt['pres'] = pres_array self.assertTrue(numpy.allclose(self._rdt['temp'], temp_array)) self.assertTrue(numpy.allclose(self._rdt['cond'], cond_array)) self.assertTrue(numpy.allclose(self._rdt['pres'], pres_array)) #want to check to make sure a KeyError is raised when a non-nickname key is used, but it's not working correctly self.assertRaises(KeyError, self._rdt.__getitem__, 'long_temp_name') self.assertRaises(KeyError, self._rdt.__getitem__,'nonsense!') taxy_tool_obj =self._tx rdt = RecordDictionaryTool(taxonomy=taxy_tool_obj) rdt['temp'] = temp_array self._rdt['rdt'] = rdt # Now test when the Record Dictionary Tool is created with the Taxonomy object rather than the TaxyTool # This can fail if the == method for TaxyTool is implemented incorrectly taxonomy_ion_obj = self._tx._t rdt2 = RecordDictionaryTool(taxonomy=taxonomy_ion_obj) rdt2['temp'] = temp_array self._rdt['rdt2'] = rdt2 # Now test bad values... list not numpy array... with self.assertRaises(TypeError) as te: rdt2['temp'] = [1,2,3] self.assertEquals( te.exception.message, '''Invalid type "<type 'list'>" in Record Dictionary Tool setitem with name "temp". Valid types are numpy.ndarray and RecordDictionaryTool''' ) # Now test numpy scalar array... with self.assertRaises(TypeError) as te: rdt2['temp'] = numpy.float32(3.14159) self.assertEquals( te.exception.message, '''Invalid type "<type 'numpy.float32'>" in Record Dictionary Tool setitem with name "temp". Valid types are numpy.ndarray and RecordDictionaryTool''' ) # Now test rank zero array... with self.assertRaises(ValueError) as te: rdt2['temp'] = numpy.array(22.5) self.assertEquals( te.exception.message, '''The rank of a value sequence array in a record dictionary must be greater than zero. Got name "temp" with rank "0"''' ) #TODO: Fix record_dictionary shape validation, then put this test back in # # Test set invalid shape # pres_array = numpy.random.standard_normal(90) # with self.assertRaises(ValueError) as te: # rdt2['pres'] = pres_array # # self.assertEquals( # te.exception.message, # '''Invalid array shape "(90,)" for name "pres"; Record dictionary defined shape is "(100,)"''' # ) # make a new RDT for testing higher rank objects... taxy_tool_obj =self._tx rdt = RecordDictionaryTool(taxonomy=taxy_tool_obj) # Now test rank 2 array... rdt['temp'] = numpy.array([[22.5,],]) self.assertTrue((rdt['temp'] == numpy.array([[22.5,],])).all()) # Now test rank 2 array... rdt['cond'] = numpy.array([[28.5,],]) self.assertTrue((rdt['cond'] == numpy.array([[28.5,],])).all()) def test_iteration(self): #Test all four iteration methods for items in the granule temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt_pdict['temp'] = temp_array self._rdt_pdict['conductivity'] = cond_array self._rdt_pdict['pres'] = pres_array for k, v in self._rdt_pdict.iteritems(): if k == 'temp': self.assertTrue(numpy.allclose(temp_array, v)) elif k == 'conductivity': self.assertTrue(numpy.allclose(cond_array, v)) elif k == 'pres': self.assertTrue(numpy.allclose(pres_array, v)) else: self.assertTrue(False) for k in self._rdt_pdict.iterkeys(): self.assertTrue(k == 'temp' or k == 'conductivity' or k == 'pres') for v in self._rdt_pdict.itervalues(): self.assertTrue(numpy.allclose(temp_array, v) or numpy.allclose(cond_array, v) or numpy.allclose(pres_array, v)) for k in self._rdt_pdict: self.assertTrue(k == 'temp' or k == 'conductivity' or k == 'pres') def test_iteration_with_taxonomy(self): #Test all four iteration methods for items in the granule temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt['temp'] = temp_array self._rdt['cond'] = cond_array self._rdt['pres'] = pres_array for k, v in self._rdt.iteritems(): if k == 'temp': self.assertTrue(numpy.allclose(temp_array, v)) elif k == 'cond': self.assertTrue(numpy.allclose(cond_array, v)) elif k == 'pres': self.assertTrue(numpy.allclose(pres_array, v)) else: self.assertTrue(False) for k in self._rdt.iterkeys(): self.assertTrue(k == 'temp' or k == 'cond' or k == 'pres') for v in self._rdt.itervalues(): self.assertTrue(numpy.allclose(temp_array, v) or numpy.allclose(cond_array, v) or numpy.allclose(pres_array, v)) for k in self._rdt: self.assertTrue(k == 'temp' or k == 'cond' or k == 'pres') def test_update(self): # Update this granule with the content of another. Assert that the taxonomies are the same... pres_array = numpy.random.standard_normal(100) self._rdt_pdict['pres'] = pres_array rdt2 = RecordDictionaryTool(param_dictionary=self._pdict) temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) rdt2['temp'] = temp_array rdt2['conductivity'] = cond_array self._rdt_pdict.update(rdt2) self.assertIn('pres', self._rdt_pdict) self.assertIn('temp', self._rdt_pdict) self.assertIn('conductivity', self._rdt_pdict) self.assertTrue((self._rdt_pdict['pres'] == pres_array).all()) self.assertTrue((self._rdt_pdict['conductivity'] == cond_array).all()) self.assertTrue((self._rdt_pdict['temp'] == temp_array).all()) self.assertEquals(len(self._rdt_pdict), 3) def test_update_with_taxonomy(self): # Update this granule with the content of another. Assert that the taxonomies are the same... pres_array = numpy.random.standard_normal(100) self._rdt['pres'] = pres_array rdt2 = RecordDictionaryTool(taxonomy=self._tx) temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) rdt2['temp'] = temp_array rdt2['cond'] = cond_array self._rdt.update(rdt2) self.assertIn('pres', self._rdt) self.assertIn('temp', self._rdt) self.assertIn('cond', self._rdt) self.assertTrue((self._rdt['pres'] == pres_array).all()) self.assertTrue((self._rdt['cond'] == cond_array).all()) self.assertTrue((self._rdt['temp'] == temp_array).all()) self.assertEquals(len(self._rdt), 3) def test_len(self): temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt_pdict['temp'] = temp_array self._rdt_pdict['conductivity'] = cond_array self._rdt_pdict['pres'] = pres_array self.assertEquals(len(self._rdt_pdict), 3) def test_len_with_taxonomy(self): temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt['temp'] = temp_array self._rdt['cond'] = cond_array self._rdt['pres'] = pres_array self.assertEquals(len(self._rdt), 3) def test_repr(self): # Come up with a reasonable string representation of the granule for debug purposes only temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt_pdict['temp'] = temp_array self._rdt_pdict['conductivity'] = cond_array self._rdt_pdict['pres'] = pres_array self.assertTrue(len(repr(self._rdt_pdict)) > 0) def test_repr_with_taxonomy(self): # Come up with a reasonable string representation of the granule for debug purposes only temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt['temp'] = temp_array self._rdt['cond'] = cond_array self._rdt['pres'] = pres_array self.assertTrue(len(repr(self._rdt)) > 0) def test_delete(self): temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt_pdict['temp'] = temp_array self._rdt_pdict['conductivity'] = cond_array self._rdt_pdict['pres'] = pres_array self.assertIn('pres', self._rdt_pdict) self.assertIn('temp', self._rdt_pdict) self.assertIn('conductivity', self._rdt_pdict) del self._rdt_pdict['pres'] self.assertNotIn('pres', self._rdt_pdict) self.assertIn('temp', self._rdt_pdict) self.assertIn('conductivity', self._rdt_pdict) def test_delete_with_taxonomy(self): temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt['temp'] = temp_array self._rdt['cond'] = cond_array self._rdt['pres'] = pres_array self.assertIn('pres', self._rdt) self.assertIn('temp', self._rdt) self.assertIn('cond', self._rdt) del self._rdt['pres'] self.assertNotIn('pres', self._rdt) self.assertIn('temp', self._rdt) self.assertIn('cond', self._rdt) def test_contains(self): # foobar isn't even in the taxonomy! self.assertNotIn('foobar', self._rdt_pdict) # Temp is in the ParameterDictionary but not the record dictionary self.assertNotIn('temp', self._rdt_pdict) # Now put in some data and make sure it works... temp_array = numpy.random.standard_normal(100) self._rdt_pdict['temp'] = temp_array self.assertIn('temp', self._rdt_pdict) def test_contains_with_taxonomy(self): # foobar isn't even in the taxonomy! self.assertNotIn('foobar', self._rdt) # Temp is in the taxonomy but not the record dictionary self.assertNotIn('temp', self._rdt) # Now put in some data and make sure it works... temp_array = numpy.random.standard_normal(100) self._rdt['temp'] = temp_array self.assertIn('temp', self._rdt) @unittest.skip('Pretty print not implemented at this time') def test_pretty_print(self): temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt_pdict['temp'] = temp_array self._rdt_pdict['conductivity'] = cond_array self._rdt_pdict['pres'] = pres_array self.assertGreater(len(self._rdt_pdict.pretty_print()), 0) @unittest.skip('Pretty print not implemented at this time') def test_pretty_print_with_taxonomy(self): temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt['temp'] = temp_array self._rdt['cond'] = cond_array self._rdt['pres'] = pres_array rdt = RecordDictionaryTool(taxonomy=self._tx) rdt['rdt'] = temp_array self._rdt['rdt'] = rdt self.assertGreater(len(self._rdt.pretty_print()), 0)