def build_granule(): tt = TaxyTool() tt.add_taxonomy_set('c') tt.add_taxonomy_set('t') tt.add_taxonomy_set('d') rdt = RecordDictionaryTool(taxonomy=tt) rdt['c'] = np.array([0,1]) rdt['t'] = np.array([0,1]) rdt['d'] = np.array([0,1]) granule = bg(data_producer_id='test_identifier', taxonomy=tt, record_dictionary=rdt) return granule
def test_combine_granule(self): tt = TaxyTool() tt.add_taxonomy_set('a') rdt = RecordDictionaryTool(tt) rdt['a'] = np.array([1,2,3]) granule1 = build_granule('test',tt,rdt) rdt = RecordDictionaryTool(tt) rdt['a'] = np.array([4,5,6]) granule2 = build_granule('test',tt,rdt) granule3 = combine_granules(granule1,granule2) rdt = RecordDictionaryTool.load_from_granule(granule3) self.assertTrue(np.allclose(rdt['a'],np.array([1,2,3,4,5,6])))
def test_build_granule_and_load_from_granule_with_taxonomy(self): #Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle tx = TaxyTool() tx.add_taxonomy_set('temp', 'long_temp_name') tx.add_taxonomy_set('cond', 'long_cond_name') tx.add_taxonomy_set('pres', 'long_pres_name') tx.add_taxonomy_set('rdt') # map is {<local name>: <granule name or path>} #Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect rdt = RecordDictionaryTool(taxonomy=tx) #Create some arrays and fill them with random values temp_array = np.random.standard_normal(100) cond_array = np.random.standard_normal(100) pres_array = np.random.standard_normal(100) #Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc. rdt['temp'] = temp_array rdt['cond'] = cond_array rdt['pres'] = pres_array #You can also add in another RecordDictionaryTool, providing the taxonomies are the same. rdt2 = RecordDictionaryTool(taxonomy=tx) rdt2['temp'] = temp_array rdt['rdt'] = rdt2 g = build_granule(data_producer_id='john', taxonomy=tx, record_dictionary=rdt) l_tx = TaxyTool.load_from_granule(g) l_rd = RecordDictionaryTool.load_from_granule(g) # Make sure we got back the same Taxonomy Object self.assertEquals(l_tx._t, tx._t) self.assertEquals(l_tx.get_handles('temp'), tx.get_handles('temp')) self.assertEquals(l_tx.get_handles('testing_2'), tx.get_handles('testing_2')) # Now test the record dictionary object self.assertEquals(l_rd._rd, rdt._rd) self.assertEquals(l_rd._tx._t, rdt._tx._t) for k, v in l_rd.iteritems(): self.assertIn(k, rdt) if isinstance(v, np.ndarray): self.assertTrue( (v == rdt[k]).all()) else: self.assertEquals(v._rd, rdt[k]._rd)
def test_createTransformsThenActivateInstrument(self): self.loggerpids = [] # Create InstrumentModel instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model="SBE37IMModel" ) try: instModel_id = self.imsclient.create_instrument_model(instModel_obj) except BadRequest as ex: self.fail("failed to create new InstrumentModel: %s" %ex) log.debug( 'new InstrumentModel id = %s ', instModel_id) # Create InstrumentAgent instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" ) try: instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj) except BadRequest as ex: self.fail("failed to create new InstrumentAgent: %s" %ex) log.debug( 'new InstrumentAgent id = %s', instAgent_id) self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id) # Create InstrumentDevice log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ') instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" ) try: instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj) self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id) except BadRequest as ex: self.fail("failed to create new InstrumentDevice: %s" %ex) log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id) instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_module='mi.instrument.seabird.sbe37smb.ooicore.driver', driver_class='SBE37Driver', comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] ) instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id) # create a stream definition for the data from the ctd simulator ctd_stream_def = SBE37_CDM_stream_definition() ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def) log.debug( 'new Stream Definition id = %s', instDevice_id) log.debug( 'Creating new CDM data product with a stream definition') craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject(RT.DataProduct, name='the parsed data', description='ctd stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary) log.debug( 'new dp_id = %s', ctd_parsed_data_product) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True) log.debug( 'Data product streams1 = %s', stream_ids) pid = self.create_logger('ctd_parsed', stream_ids[0] ) self.loggerpids.append(pid) print 'test_createTransformsThenActivateInstrument: Data product streams1 = ', stream_ids #------------------------------- # Create CTD Raw as the second data product #------------------------------- log.debug( 'Creating new RAW data product with a stream definition') raw_stream_def = SBE37_RAW_stream_definition() raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def) dp_obj = IonObject(RT.DataProduct, name='the raw data', description='raw stream test', temporal_domain = tdom, spatial_domain = sdom) ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary) log.debug( 'new dp_id = %s', str(ctd_raw_data_product)) self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) log.debug( 'Data product streams2 = %s', str(stream_ids)) #todo: attaching the taxonomy to the stream is a TEMPORARY measure # Create taxonomies for both parsed and attach to the stream RawTax = TaxyTool() RawTax.add_taxonomy_set('raw_fixed','Fixed length bytes in an array of records') RawTax.add_taxonomy_set('raw_blob','Unlimited length bytes in an array') # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True) print 'Data product streams2 = ', stream_ids #------------------------------- # L0 Conductivity - Temperature - Pressure: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition ctd_L0_all") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L0_all', description='transform ctd package into three separate L0 streams', module='ion.processes.data.transforms.ctd.ctd_L0_all', class_name='ctd_L0_all', process_source='some_source_reference') try: ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new ctd_L0_all data process definition: %s" %ex) #------------------------------- # L1 Conductivity: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1ConductivityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_conductivity', description='create the L1 conductivity data product', module='ion.processes.data.transforms.ctd.ctd_L1_conductivity', class_name='CTDL1ConductivityTransform', process_source='CTDL1ConductivityTransform source code here...') try: ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex) #------------------------------- # L1 Pressure: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1PressureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_pressure', description='create the L1 pressure data product', module='ion.processes.data.transforms.ctd.ctd_L1_pressure', class_name='CTDL1PressureTransform', process_source='CTDL1PressureTransform source code here...') try: ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex) #------------------------------- # L1 Temperature: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1TemperatureTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L1_temperature', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L1_temperature', class_name='CTDL1TemperatureTransform', process_source='CTDL1TemperatureTransform source code here...') try: ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex) #------------------------------- # L2 Salinity: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition SalinityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_salinity', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_salinity', class_name='SalinityTransform', process_source='SalinityTransform source code here...') try: ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new SalinityTransform data process definition: %s" %ex) #------------------------------- # L2 Density: Data Process Definition #------------------------------- log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition DensityTransform") dpd_obj = IonObject(RT.DataProcessDefinition, name='ctd_L2_density', description='create the L1 temperature data product', module='ion.processes.data.transforms.ctd.ctd_L2_density', class_name='DensityTransform', process_source='DensityTransform source code here...') try: ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj) except BadRequest as ex: self.fail("failed to create new DensityTransform data process definition: %s" %ex) #------------------------------- # L0 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l0_conductivity = L0_conductivity_stream_definition() outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_pressure = L0_pressure_stream_definition() outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id ) outgoing_stream_l0_temperature = L0_temperature_stream_definition() outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id ) self.output_products={} log.debug("test_createTransformsThenActivateInstrument: create output data product L0 conductivity") ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L0_Conductivity', description='transform output conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id, parameter_dictionary) self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id) log.debug("test_createTransformsThenActivateInstrument: create output data product L0 pressure") ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L0_Pressure', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id, parameter_dictionary) self.output_products['pressure'] = ctd_l0_pressure_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id) log.debug("test_createTransformsThenActivateInstrument: create output data product L0 temperature") ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L0_Temperature', description='transform output temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id, parameter_dictionary) self.output_products['temperature'] = ctd_l0_temperature_output_dp_id self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id) #------------------------------- # L1 Conductivity - Temperature - Pressure: Output Data Products #------------------------------- outgoing_stream_l1_conductivity = L1_conductivity_stream_definition() outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_conductivity, name='L1_conductivity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id ) outgoing_stream_l1_pressure = L1_pressure_stream_definition() outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_pressure, name='L1_Pressure') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id ) outgoing_stream_l1_temperature = L1_temperature_stream_definition() outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_temperature, name='L1_Temperature') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id ) log.debug("test_createTransformsThenActivateInstrument: create output data product L1 conductivity") ctd_l1_conductivity_output_dp_obj = IonObject( RT.DataProduct, name='L1_Conductivity', description='transform output L1 conductivity', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id, parameter_dictionary) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_conductivity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_conductivity', stream_ids[0] ) self.loggerpids.append(pid) log.debug("test_createTransformsThenActivateInstrument: create output data product L1 pressure") ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct, name='L1_Pressure', description='transform output L1 pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id, parameter_dictionary ) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_pressure_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_pressure', stream_ids[0] ) self.loggerpids.append(pid) log.debug("test_createTransformsThenActivateInstrument: create output data product L1 temperature") ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct, name='L1_Temperature', description='transform output L1 temperature', temporal_domain = tdom, spatial_domain = sdom) ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id, parameter_dictionary) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_temperature_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l1_temperature', stream_ids[0] ) self.loggerpids.append(pid) #------------------------------- # L2 Salinity - Density: Output Data Products #------------------------------- outgoing_stream_l2_salinity = L2_practical_salinity_stream_definition() outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_salinity, name='L2_salinity') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id ) outgoing_stream_l2_density = L2_density_stream_definition() outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_density, name='L2_Density') self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id ) log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Salinity") ctd_l2_salinity_output_dp_obj = IonObject(RT.DataProduct, name='L2_Salinity', description='transform output L2 salinity', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id, parameter_dictionary) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_salinity_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_salinity', stream_ids[0] ) self.loggerpids.append(pid) log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Density") ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct, name='L2_Density', description='transform output pressure', temporal_domain = tdom, spatial_domain = sdom) ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id, parameter_dictionary) self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_density_output_dp_id) # Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger stream_ids, _ = self.rrclient.find_objects(ctd_l2_density_output_dp_id, PRED.hasStream, None, True) pid = self.create_logger('ctd_l2_density', stream_ids[0] ) self.loggerpids.append(pid) #------------------------------- # L0 Conductivity - Temperature - Pressure: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process start") try: ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products) self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process return") #------------------------------- # L1 Conductivity: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process start") try: l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'output':ctd_l1_conductivity_output_dp_id}) self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process return") #------------------------------- # L1 Pressure: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start") try: l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'output':ctd_l1_pressure_output_dp_id}) self.dataprocessclient.activate_data_process(l1_pressure_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return") #------------------------------- # L1 Temperature: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start") try: l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'output':ctd_l1_temperature_output_dp_id}) self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return") #------------------------------- # L2 Salinity: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process start") try: l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_salinity_output_dp_id}) self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process return") #------------------------------- # L2 Density: Create the data process #------------------------------- log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process start") try: l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_density_output_dp_id}) self.dataprocessclient.activate_data_process(l2_density_all_data_process_id) except BadRequest as ex: self.fail("failed to create new data process: %s" %ex) log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process return") #------------------------------- # Launch InstrumentAgentInstance, connect to the resource agent client #------------------------------- self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id) print 'test_createTransformsThenActivateInstrument: Instrument agent instance obj: = ', inst_agent_instance_obj # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()) print 'activate_instrument: got ia client %s', self._ia_client log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client)) #------------------------------- # Streaming #------------------------------- cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: initialize %s", str(retval)) log.debug("test_activateInstrumentStream: Sending go_active command (L4-CI-SA-RQ-334)") cmd = AgentCommand(command='go_active') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: return value from go_active %s", str(reply)) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentStream: current state after sending go_active command %s (L4-CI-SA-RQ-334)", str(state)) cmd = AgentCommand(command='run') reply = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentStream: return from run state: %s", str(state)) # Make sure the sampling rate and transmission are sane. params = { SBE37Parameter.NAVG : 1, SBE37Parameter.INTERVAL : 5, SBE37Parameter.TXREALTIME : True } self._ia_client.set_param(params) log.debug("test_activateInstrumentStream: calling go_streaming ") cmd = AgentCommand(command='go_streaming') reply = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentStream: return from go_streaming state: %s", str(state)) time.sleep(7) log.debug("test_activateInstrumentStream: calling go_observatory") cmd = AgentCommand(command='go_observatory') reply = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state)) log.debug("test_activateInstrumentStream: calling reset ") cmd = AgentCommand(command='reset') reply = self._ia_client.execute_agent(cmd) log.debug("test_activateInstrumentStream: return from reset state:%s", str(reply.result)) time.sleep(2) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id) for pid in self.loggerpids: self.processdispatchclient.cancel_process(pid)
def setUp(self): """ Setup test cases. """ parsed_tx = TaxyTool() parsed_tx.add_taxonomy_set('temp', 'long name for temp', 't') parsed_tx.add_taxonomy_set('cond', 'long name for cond', 'c') parsed_tx.add_taxonomy_set('pres', 'long name for pres', 'd') parsed_tx.add_taxonomy_set('lat', 'long name for latitude', 'lt') parsed_tx.add_taxonomy_set('lon', 'long name for longitude', 'ln') parsed_tx.add_taxonomy_set('time', 'long name for time', 'tm') # This is an example of using groups it is not a normative statement about how to use groups parsed_tx.add_taxonomy_set('coordinates', 'This group contains coordinates...') parsed_tx.add_taxonomy_set('data', 'This group contains data...') self._parsed_taxonomy = parsed_tx raw_tx = TaxyTool() raw_tx.add_taxonomy_set('raw_fixed', 'Fixed length bytes in an array of records') raw_tx.add_taxonomy_set('raw_blob', 'Unlimited length bytes in an array') self._raw_taxonomy = raw_tx log.debug("_parsed_taxonomy = %s" % self._parsed_taxonomy.dump()) log.debug("_raw_taxonomy = %s" % self._raw_taxonomy.dump()) self.packet_factory = PacketFactory.get_packet_factory( PacketFactoryType.R2LCAFormat) self.assertIsInstance(self.packet_factory, LCAPacketFactory)
def get_taxonomy(stream_name): """ Gets the taxonomy for a given stream. @param stream_name Stream name. @retval corresponding taxonomy. """ if stream_name in _cache: log.debug("taxonomy for stream_name=%s found in cache" % stream_name) return _cache[stream_name] taxy = None # TODO determine and use appropriate mechanism for definition of # taxonomies. The following hard-coded definitions are taken from # preliminary code in instrument_management_service.py as of 7/15/12. if "parsed" == stream_name: taxy = TaxyTool() taxy.add_taxonomy_set('temp', 't', 'long name for temp') taxy.add_taxonomy_set('cond', 'c', 'long name for cond') taxy.add_taxonomy_set('pres', 'p', 'long name for pres') taxy.add_taxonomy_set('lat', 'long name for latitude') taxy.add_taxonomy_set('lon', 'long name for longitude') taxy.add_taxonomy_set('time', 'long name for time') taxy.add_taxonomy_set('height', 'long name for height') # This is an example of using groups it is not a normative statement about how to use groups taxy.add_taxonomy_set('coordinates', 'This group contains coordinates...') taxy.add_taxonomy_set('data', 'This group contains data...') elif "raw" == stream_name: taxy = TaxyTool() taxy.add_taxonomy_set('blob', 'bytes in an array') taxy.add_taxonomy_set('lat', 'long name for latitude') taxy.add_taxonomy_set('lon', 'long name for longitude') taxy.add_taxonomy_set('time', 'long name for time') taxy.add_taxonomy_set('height', 'long name for height') taxy.add_taxonomy_set('coordinates', 'This group contains coordinates...') taxy.add_taxonomy_set('data', 'This group contains data...') else: log.warn("Undefined taxonomy for stream_name=%s" % stream_name) if taxy: _cache[stream_name] = taxy log.debug("created taxonomy for stream_name=%s " % stream_name) return taxy
def test_activateDatasetAgent(self): # Create ExternalDatasetModel datsetModel_obj = IonObject( RT.ExternalDatasetModel, name="ExampleDatasetModel", description="ExampleDatasetModel", datset_type="FibSeries", ) try: datasetModel_id = self.damsclient.create_external_dataset_model(datsetModel_obj) except BadRequest as ex: self.fail("failed to create new ExternalDatasetModel: %s" % ex) log.debug("TestExternalDatasetAgentMgmt: new ExternalDatasetModel id = %s", str(datasetModel_id)) # Create ExternalDatasetAgent datasetAgent_obj = IonObject( RT.ExternalDatasetAgent, name="datasetagent007", description="datasetagent007", handler_module=EDA_MOD, handler_class=EDA_CLS, ) try: datasetAgent_id = self.damsclient.create_external_dataset_agent(datasetAgent_obj, datasetModel_id) except BadRequest as ex: self.fail("failed to create new ExternalDatasetAgent: %s" % ex) log.debug("TestExternalDatasetAgentMgmt: new ExternalDatasetAgent id = %s", str(datasetAgent_id)) # Create ExternalDataset log.debug("TestExternalDatasetAgentMgmt: Create external dataset resource ") extDataset_obj = IonObject(RT.ExternalDataset, name="ExtDataset", description="ExtDataset") try: extDataset_id = self.damsclient.create_external_dataset(extDataset_obj, datasetModel_id) except BadRequest as ex: self.fail("failed to create new external dataset resource: %s" % ex) log.debug("TestExternalDatasetAgentMgmt: new ExternalDataset id = %s ", str(extDataset_id)) # register the dataset as a data producer dproducer_id = self.damsclient.register_external_data_set(extDataset_id) # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition(name="SBE37_CDM", parameter_dictionary_id=pdict_id) log.debug("TestExternalDatasetAgentMgmt: new Stream Definition id = %s", str(ctd_stream_def_id)) log.debug("TestExternalDatasetAgentMgmt: Creating new data product with a stream definition") dp_obj = IonObject(RT.DataProduct, name="eoi dataset data", description=" stream test") tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() dp_obj = IonObject( RT.DataProduct, name="DP1", description="some new dp", temporal_domain=tdom, spatial_domain=sdom ) data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id) log.debug("TestExternalDatasetAgentMgmt: new dp_id = %s", str(data_product_id1)) self.damsclient.assign_data_product(input_resource_id=extDataset_id, data_product_id=data_product_id1) # todo fix the problem here.... self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug("TestExternalDatasetAgentMgmt: Data product streams1 = %s", str(stream_ids)) stream_id = stream_ids[0] # Build a taxonomy for the dataset tx = TaxyTool() tx.add_taxonomy_set("data", "external_data") # Augment the DVR_CONFIG with the necessary pieces self.DVR_CONFIG["dh_cfg"] = { "TESTING": True, "stream_id": stream_id, # TODO: This should probably be a 'stream_config' dict with stream_name:stream_id members "data_producer_id": dproducer_id, # 'external_dataset_res':extDataset_obj, # Not needed - retrieved by EDA based on resource_id "taxonomy": tx.dump(), # TODO: Currently does not support sets "max_records": 4, } # Create agent config. self._stream_config = {} agent_config = { "driver_config": self.DVR_CONFIG, "stream_config": self._stream_config, "agent": {"resource_id": EDA_RESOURCE_ID}, "test_mode": True, } extDatasetAgentInstance_obj = IonObject( RT.ExternalDatasetAgentInstance, name="DatasetAgentInstance", description="DatasetAgentInstance", dataset_driver_config=self.DVR_CONFIG, dataset_agent_config=agent_config, ) extDatasetAgentInstance_id = self.damsclient.create_external_dataset_agent_instance( external_dataset_agent_instance=extDatasetAgentInstance_obj, external_dataset_agent_id=datasetAgent_id, external_dataset_id=extDataset_id, ) log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(extDatasetAgentInstance_obj)) log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s", str(extDatasetAgentInstance_id)) # Check that the instance is currently not active id, active = self.damsclient.retrieve_external_dataset_agent_instance(extDataset_id) log.debug( "TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 1 = %s ", str(id), str(active) ) self.damsclient.start_external_dataset_agent_instance(extDatasetAgentInstance_id) dataset_agent_instance_obj = self.damsclient.read_external_dataset_agent_instance(extDatasetAgentInstance_id) log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(dataset_agent_instance_obj)) # now the instance process should be active id, active = self.damsclient.retrieve_external_dataset_agent_instance(extDataset_id) log.debug( "TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 2 = %s ", str(id), str(active) ) # Start a resource agent client to talk with the instrument agent. self._dsa_client = ResourceAgentClient(extDataset_id, process=FakeProcess()) print "TestExternalDatasetAgentMgmt: got ia client %s", self._dsa_client log.debug("TestExternalDatasetAgentMgmt: got dataset client %s", str(self._dsa_client)) # cmd=AgentCommand(command='initialize') # _ = self._dsa_client.execute_agent(cmd) # # cmd = AgentCommand(command='go_active') # _ = self._dsa_client.execute_agent(cmd) # # cmd = AgentCommand(command='run') # _ = self._dsa_client.execute_agent(cmd) # # log.info('Send an unconstrained request for data (\'new data\')') # cmd = AgentCommand(command='acquire_data') # self._dsa_client.execute(cmd) # # log.info('Send a second unconstrained request for data (\'new data\'), should be rejected') # cmd = AgentCommand(command='acquire_data') # self._dsa_client.execute(cmd) # # cmd = AgentCommand(command='reset') # _ = self._dsa_client.execute_agent(cmd) # cmd = AgentCommand(command='get_current_state') # retval = self._dsa_client.execute_agent(cmd) # state = retval.result # TODO: Think about what we really should be testing at this point # The following is taken from ion.agents.data.test.test_external_dataset_agent.ExternalDatasetAgentTestBase.test_states() # TODO: Do we also need to show data retrieval? cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) cmd = AgentCommand(command="initialize") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) cmd = AgentCommand(command="go_active") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command="run") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command="pause") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STOPPED) cmd = AgentCommand(command="resume") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command="clear") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command="run") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command="pause") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STOPPED) cmd = AgentCommand(command="clear") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command="run") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command="reset") retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command="get_current_state") retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) # ------------------------------- # Deactivate InstrumentAgentInstance # ------------------------------- self.damsclient.stop_external_dataset_agent_instance(extDatasetAgentInstance_id)
import math from prototype.sci_data.stream_defs import ctd_stream_packet, SBE37_CDM_stream_definition, ctd_stream_definition from prototype.sci_data.constructor_apis import PointSupplementConstructor from interface.services.dm.ipubsub_management_service import PubsubManagementServiceClient from ion.processes.data.ctd_stream_publisher import SimpleCtdPublisher from ion.services.dm.utility.granule.record_dictionary import RecordDictionaryTool from ion.services.dm.utility.granule.taxonomy import TaxyTool from ion.services.dm.utility.granule.granule import build_granule import numpy ### Taxonomies are defined before hand out of band... somehow. tx = TaxyTool() tx.add_taxonomy_set('temp','long name for temp') tx.add_taxonomy_set('cond','long name for cond') tx.add_taxonomy_set('lat','long name for latitude') tx.add_taxonomy_set('lon','long name for longitude') tx.add_taxonomy_set('pres','long name for pres') tx.add_taxonomy_set('time','long name for time') tx.add_taxonomy_set('height','long name for height') class SinusoidalCtdPublisher(SimpleCtdPublisher): def __init__(self, *args, **kwargs): super(SinusoidalCtdPublisher, self).__init__(*args,**kwargs) #@todo Init stuff
import gevent from prototype.sci_data.stream_defs import ctd_stream_packet, SBE37_CDM_stream_definition, ctd_stream_definition from prototype.sci_data.constructor_apis import PointSupplementConstructor from interface.services.dm.ipubsub_management_service import PubsubManagementServiceClient ### For new granule and stream interface from pyon.ion.transforma import TransformStreamPublisher from ion.services.dm.utility.granule.record_dictionary import RecordDictionaryTool from ion.services.dm.utility.granule.taxonomy import TaxyTool from ion.services.dm.utility.granule.granule import build_granule ### Taxonomies are defined before hand out of band... somehow. tx = TaxyTool() tx.add_taxonomy_set('temp','long name for temp') tx.add_taxonomy_set('cond','long name for cond') tx.add_taxonomy_set('lat','long name for latitude') tx.add_taxonomy_set('lon','long name for longitude') tx.add_taxonomy_set('pres','long name for pres') tx.add_taxonomy_set('time','long name for time') tx.add_taxonomy_set('height','long name for height') tx.add_taxonomy_set('coordinates','This group contains coordinates...') tx.add_taxonomy_set('data','This group contains data...') class SimpleCtdPublisher(TransformStreamPublisher): def on_start(self): exchange_point = self.CFG.get_safe('process.exchange_point', 'science_data') self.CFG.process.exchange_point = exchange_point super(SimpleCtdPublisher,self).on_start()
from ion.services.dm.utility.granule.record_dictionary import RecordDictionaryTool from pyon.util.containers import get_safe from prototype.sci_data.stream_defs import SBE37_CDM_stream_definition, SBE37_RAW_stream_definition from prototype.sci_data.stream_parser import PointSupplementStreamParser from prototype.sci_data.constructor_apis import PointSupplementConstructor, RawSupplementConstructor import StringIO from numpy import array, append # Matplotlib related imports from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas from matplotlib.figure import Figure tx = TaxyTool() tx.add_taxonomy_set("matplotlib_graphs", "Matplotlib generated graphs for a particular data product") class VizTransformMatplotlibGraphs(TransformFunction): """ This class is used for instantiating worker processes that have subscriptions to data streams and convert incoming data from CDM format to Matplotlib graphs """ # Need to extract the incoming stream def automatically outgoing_stream_def = SBE37_RAW_stream_definition() incoming_stream_def = SBE37_CDM_stream_definition() def on_start(self):
from pyon.public import IonObject, RT, log from datetime import datetime import numpy from ion.services.dm.utility.granule.granule import build_granule from ion.services.dm.utility.granule.taxonomy import TaxyTool from ion.services.dm.utility.granule.record_dictionary import RecordDictionaryTool from prototype.sci_data.stream_defs import SBE37_CDM_stream_definition, SBE37_RAW_stream_definition from prototype.sci_data.stream_parser import PointSupplementStreamParser from prototype.sci_data.constructor_apis import PointSupplementConstructor, RawSupplementConstructor from pyon.util.containers import get_safe tx = TaxyTool() tx.add_taxonomy_set('google_dt_components','Google datatable') class VizTransformGoogleDT(TransformFunction): """ This class is used for converting incoming data from CDM format to JSON style Google DataTables Note: One behaviour that this class is expected to achieve specifically is to determine if its supposed to work as a realtime transform (exists indefinitely and maintains a sliding window of data) or as a replay transform (one-shot). [2] This transform behaves as an instantaneous forwarder. There is no waiting for the entire stream to create the complete datatable. As the granules come in, they are translated to the datatable 'components'. Components, because we are not creating the actual datatable in this code. That's the job of the viz service to put all the components of a datatable together in JSON format before sending it
def setUp(self): """ Setup test cases. """ parsed_tx = TaxyTool() parsed_tx.add_taxonomy_set('temp','long name for temp', 't') parsed_tx.add_taxonomy_set('cond','long name for cond', 'c') parsed_tx.add_taxonomy_set('pres','long name for pres', 'd') parsed_tx.add_taxonomy_set('lat','long name for latitude', 'lt') parsed_tx.add_taxonomy_set('lon','long name for longitude', 'ln') parsed_tx.add_taxonomy_set('time','long name for time', 'tm') # This is an example of using groups it is not a normative statement about how to use groups parsed_tx.add_taxonomy_set('coordinates','This group contains coordinates...') parsed_tx.add_taxonomy_set('data','This group contains data...') self._parsed_taxonomy = parsed_tx raw_tx = TaxyTool() raw_tx.add_taxonomy_set('raw_fixed','Fixed length bytes in an array of records') raw_tx.add_taxonomy_set('raw_blob','Unlimited length bytes in an array') self._raw_taxonomy = raw_tx log.debug("_parsed_taxonomy = %s" % self._parsed_taxonomy.dump()) log.debug("_raw_taxonomy = %s" % self._raw_taxonomy.dump()) self.packet_factory = PacketFactory.get_packet_factory( PacketFactoryType.R2LCAFormat) self.assertIsInstance(self.packet_factory, LCAPacketFactory)
def test_activateDatasetAgent(self): # Create ExternalDatasetModel datsetModel_obj = IonObject(RT.ExternalDatasetModel, name='ExampleDatasetModel', description="ExampleDatasetModel", datset_type="FibSeries") try: datasetModel_id = self.damsclient.create_external_dataset_model( datsetModel_obj) except BadRequest as ex: self.fail("failed to create new ExternalDatasetModel: %s" % ex) log.debug( "TestExternalDatasetAgentMgmt: new ExternalDatasetModel id = %s", str(datasetModel_id)) # Create ExternalDatasetAgent datasetAgent_obj = IonObject(RT.ExternalDatasetAgent, name='datasetagent007', description="datasetagent007", handler_module=EDA_MOD, handler_class=EDA_CLS) try: datasetAgent_id = self.damsclient.create_external_dataset_agent( datasetAgent_obj, datasetModel_id) except BadRequest as ex: self.fail("failed to create new ExternalDatasetAgent: %s" % ex) log.debug( "TestExternalDatasetAgentMgmt: new ExternalDatasetAgent id = %s", str(datasetAgent_id)) # Create ExternalDataset log.debug( 'TestExternalDatasetAgentMgmt: Create external dataset resource ') extDataset_obj = IonObject(RT.ExternalDataset, name='ExtDataset', description="ExtDataset") try: extDataset_id = self.damsclient.create_external_dataset( extDataset_obj, datasetModel_id) except BadRequest as ex: self.fail("failed to create new external dataset resource: %s" % ex) log.debug( "TestExternalDatasetAgentMgmt: new ExternalDataset id = %s ", str(extDataset_id)) #register the dataset as a data producer dproducer_id = self.damsclient.register_external_data_set( extDataset_id) # create a stream definition for the data from the ctd simulator pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) ctd_stream_def_id = self.pubsubcli.create_stream_definition( name='SBE37_CDM', parameter_dictionary_id=pdict_id) log.debug( "TestExternalDatasetAgentMgmt: new Stream Definition id = %s", str(ctd_stream_def_id)) log.debug( "TestExternalDatasetAgentMgmt: Creating new data product with a stream definition" ) dp_obj = IonObject(RT.DataProduct, name='eoi dataset data', description=' stream test') dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp') data_product_id1 = self.dpclient.create_data_product( dp_obj, ctd_stream_def_id) log.debug("TestExternalDatasetAgentMgmt: new dp_id = %s", str(data_product_id1)) self.damsclient.assign_data_product(input_resource_id=extDataset_id, data_product_id=data_product_id1) #todo fix the problem here.... self.dpclient.activate_data_product_persistence( data_product_id=data_product_id1) # Retrieve the id of the OUTPUT stream from the out Data Product stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True) log.debug("TestExternalDatasetAgentMgmt: Data product streams1 = %s", str(stream_ids)) stream_id = stream_ids[0] # Build a taxonomy for the dataset tx = TaxyTool() tx.add_taxonomy_set('data', 'external_data') # Augment the DVR_CONFIG with the necessary pieces self.DVR_CONFIG['dh_cfg'] = { 'TESTING': True, 'stream_id': stream_id, #TODO: This should probably be a 'stream_config' dict with stream_name:stream_id members 'data_producer_id': dproducer_id, # 'external_dataset_res':extDataset_obj, # Not needed - retrieved by EDA based on resource_id 'taxonomy': tx.dump(), #TODO: Currently does not support sets 'max_records': 4, } # Create agent config. self._stream_config = {} agent_config = { 'driver_config': self.DVR_CONFIG, 'stream_config': self._stream_config, 'agent': { 'resource_id': EDA_RESOURCE_ID }, 'test_mode': True } extDatasetAgentInstance_obj = IonObject( RT.ExternalDatasetAgentInstance, name='DatasetAgentInstance', description="DatasetAgentInstance", dataset_driver_config=self.DVR_CONFIG, dataset_agent_config=agent_config) extDatasetAgentInstance_id = self.damsclient.create_external_dataset_agent_instance( external_dataset_agent_instance=extDatasetAgentInstance_obj, external_dataset_agent_id=datasetAgent_id, external_dataset_id=extDataset_id) log.debug( "TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(extDatasetAgentInstance_obj)) log.debug( "TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s", str(extDatasetAgentInstance_id)) #Check that the instance is currently not active id, active = self.damsclient.retrieve_external_dataset_agent_instance( extDataset_id) log.debug( "TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 1 = %s ", str(id), str(active)) self.damsclient.start_external_dataset_agent_instance( extDatasetAgentInstance_id) dataset_agent_instance_obj = self.damsclient.read_external_dataset_agent_instance( extDatasetAgentInstance_id) log.debug( "TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(dataset_agent_instance_obj)) # now the instance process should be active id, active = self.damsclient.retrieve_external_dataset_agent_instance( extDataset_id) log.debug( "TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 2 = %s ", str(id), str(active)) # Start a resource agent client to talk with the instrument agent. self._dsa_client = ResourceAgentClient(extDataset_id, process=FakeProcess()) print 'TestExternalDatasetAgentMgmt: got ia client %s', self._dsa_client log.debug("TestExternalDatasetAgentMgmt: got dataset client %s", str(self._dsa_client)) # cmd=AgentCommand(command='initialize') # _ = self._dsa_client.execute_agent(cmd) # # cmd = AgentCommand(command='go_active') # _ = self._dsa_client.execute_agent(cmd) # # cmd = AgentCommand(command='run') # _ = self._dsa_client.execute_agent(cmd) # # log.info('Send an unconstrained request for data (\'new data\')') # cmd = AgentCommand(command='acquire_data') # self._dsa_client.execute(cmd) # # log.info('Send a second unconstrained request for data (\'new data\'), should be rejected') # cmd = AgentCommand(command='acquire_data') # self._dsa_client.execute(cmd) # # cmd = AgentCommand(command='reset') # _ = self._dsa_client.execute_agent(cmd) # cmd = AgentCommand(command='get_current_state') # retval = self._dsa_client.execute_agent(cmd) # state = retval.result # TODO: Think about what we really should be testing at this point # The following is taken from ion.agents.data.test.test_external_dataset_agent.ExternalDatasetAgentTestBase.test_states() # TODO: Do we also need to show data retrieval? cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) cmd = AgentCommand(command='initialize') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) cmd = AgentCommand(command='go_active') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command='pause') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STOPPED) cmd = AgentCommand(command='resume') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command='clear') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command='pause') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STOPPED) cmd = AgentCommand(command='clear') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command='reset') retval = self._dsa_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._dsa_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) #------------------------------- # Deactivate InstrumentAgentInstance #------------------------------- self.damsclient.stop_external_dataset_agent_instance( extDatasetAgentInstance_id)
def setUp(self): """ Setup test cases. """ parsed_tx = TaxyTool() parsed_tx.add_taxonomy_set("temp", "long name for temp", "t") parsed_tx.add_taxonomy_set("cond", "long name for cond", "c") parsed_tx.add_taxonomy_set("pres", "long name for pres", "d") parsed_tx.add_taxonomy_set("lat", "long name for latitude", "lt") parsed_tx.add_taxonomy_set("lon", "long name for longitude", "ln") parsed_tx.add_taxonomy_set("time", "long name for time", "tm") # This is an example of using groups it is not a normative statement about how to use groups parsed_tx.add_taxonomy_set("coordinates", "This group contains coordinates...") parsed_tx.add_taxonomy_set("data", "This group contains data...") self._parsed_taxonomy = parsed_tx raw_tx = TaxyTool() raw_tx.add_taxonomy_set("raw_fixed", "Fixed length bytes in an array of records") raw_tx.add_taxonomy_set("raw_blob", "Unlimited length bytes in an array") self._raw_taxonomy = raw_tx log.debug("_parsed_taxonomy = %s" % self._parsed_taxonomy.dump()) log.debug("_raw_taxonomy = %s" % self._raw_taxonomy.dump()) self.packet_factory = PacketFactory.get_packet_factory(PacketFactoryType.R2LCAFormat) self.assertIsInstance(self.packet_factory, LCAPacketFactory)
def get_taxonomy(stream_name): """ Gets the taxonomy for a given stream. @param stream_name Stream name. @retval corresponding taxonomy. """ if stream_name in _cache: log.debug("taxonomy for stream_name=%s found in cache" % stream_name) return _cache[stream_name] taxy = None # TODO determine and use appropriate mechanism for definition of # taxonomies. The following hard-coded definitions are taken from # preliminary code in instrument_management_service.py as of 7/15/12. if "parsed" == stream_name: taxy = TaxyTool() taxy.add_taxonomy_set('temp', 't', 'long name for temp') taxy.add_taxonomy_set('cond', 'c', 'long name for cond') taxy.add_taxonomy_set('pres', 'p', 'long name for pres') taxy.add_taxonomy_set('lat','long name for latitude') taxy.add_taxonomy_set('lon','long name for longitude') taxy.add_taxonomy_set('time','long name for time') taxy.add_taxonomy_set('height','long name for height') # This is an example of using groups it is not a normative statement about how to use groups taxy.add_taxonomy_set('coordinates','This group contains coordinates...') taxy.add_taxonomy_set('data','This group contains data...') elif "raw" == stream_name: taxy = TaxyTool() taxy.add_taxonomy_set('blob','bytes in an array') taxy.add_taxonomy_set('lat','long name for latitude') taxy.add_taxonomy_set('lon','long name for longitude') taxy.add_taxonomy_set('time','long name for time') taxy.add_taxonomy_set('height','long name for height') taxy.add_taxonomy_set('coordinates','This group contains coordinates...') taxy.add_taxonomy_set('data','This group contains data...') else: log.warn("Undefined taxonomy for stream_name=%s" % stream_name) if taxy: _cache[stream_name] = taxy log.debug("created taxonomy for stream_name=%s " % stream_name) return taxy
class RecordDictionaryToolTestCase(unittest.TestCase): def setUp(self): self._tx = TaxyTool() self._tx.add_taxonomy_set('temp', 'long_temp_name') self._tx.add_taxonomy_set('cond', 'long_cond_name') self._tx.add_taxonomy_set('pres', 'long_pres_name') self._tx.add_taxonomy_set('rdt') self._tx.add_taxonomy_set('rdt2') # map is {<local name>: <granule name or path>} self._rdt = RecordDictionaryTool(taxonomy=self._tx) self._pdict = ParameterDictionary() t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) t_ctxt.reference_frame = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' self._pdict.add_context(t_ctxt) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lat_ctxt.reference_frame = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' self._pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lon_ctxt.reference_frame = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' self._pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' self._pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) cond_ctxt.uom = 'unknown' self._pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext('pres', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) pres_ctxt.uom = 'unknown' self._pdict.add_context(pres_ctxt) self._rdt_pdict = RecordDictionaryTool(param_dictionary=self._pdict) def test_init(self): # initialize with a ParameterDictionary rdt = RecordDictionaryTool(param_dictionary=self._pdict) self.assertIsInstance(rdt._param_dict, ParameterDictionary) # initialize with nonsense self.assertRaises(TypeError, RecordDictionaryTool, ['foo', 'barr']) # initialize with a valid shape rdt = RecordDictionaryTool(param_dictionary=self._pdict, shape=(5,2)) self.assertEquals(rdt._shp, (5,2)) rdt = RecordDictionaryTool(param_dictionary=self._pdict, shape=(5,)) self.assertEquals(rdt._shp, (5,)) rdt = RecordDictionaryTool(param_dictionary=self._tx, shape=5) self.assertEquals(rdt._shp, (5,)) # initialize with no length rdt = RecordDictionaryTool(param_dictionary=self._pdict) self.assertEquals(rdt._shp, None) def test_init_with_taxonomy(self): # initialize with a taxonomy tool rdt = RecordDictionaryTool(taxonomy=self._tx) self.assertIsInstance(rdt._tx, TaxyTool) # initialize with a taxonomy object rdt = RecordDictionaryTool(taxonomy=self._tx._t) self.assertIsInstance(rdt._tx, TaxyTool) # initialize with pooo self.assertRaises(TypeError, RecordDictionaryTool, ['foo', 'barr']) # initialize with a valid shape rdt = RecordDictionaryTool(taxonomy=self._tx, shape=(5,2)) self.assertEquals(rdt._shp, (5,2)) rdt = RecordDictionaryTool(taxonomy=self._tx, shape=(5,)) self.assertEquals(rdt._shp, (5,)) rdt = RecordDictionaryTool(taxonomy=self._tx, shape=5) self.assertEquals(rdt._shp, (5,)) # initialize with no length rdt = RecordDictionaryTool(taxonomy=self._tx) self.assertEquals(rdt._shp, None) def test_set_and_get(self): #make sure you can set and get items in the granule by name in the taxonomy temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self.assertRaises(KeyError, self._rdt.__setitem__, 'long_temp_name', temp_array) self.assertRaises(KeyError, self._rdt.__setitem__, 'nonsense', temp_array) self._rdt_pdict['temp'] = temp_array self._rdt_pdict['conductivity'] = cond_array self._rdt_pdict['pres'] = pres_array self.assertTrue(numpy.allclose(self._rdt_pdict['temp'], temp_array)) self.assertTrue(numpy.allclose(self._rdt_pdict['conductivity'], cond_array)) self.assertTrue(numpy.allclose(self._rdt_pdict['pres'], pres_array)) #want to check to make sure a KeyError is raised when a non-nickname key is used, but it's not working correctly self.assertRaises(KeyError, self._rdt.__getitem__, 'long_temp_name') self.assertRaises(KeyError, self._rdt.__getitem__,'nonsense!') pdict =self._pdict rdt = RecordDictionaryTool(param_dictionary=pdict) rdt['temp'] = temp_array # Now test bad values... list not numpy array... with self.assertRaises(TypeError) as te: rdt['temp'] = [1,2,3] self.assertEquals( te.exception.message, '''Invalid type "<type 'list'>" in Record Dictionary Tool setitem with name "temp". Valid types are numpy.ndarray and RecordDictionaryTool''' ) # Now test numpy scalar array... with self.assertRaises(TypeError) as te: rdt['temp'] = numpy.float32(3.14159) self.assertEquals( te.exception.message, '''Invalid type "<type 'numpy.float32'>" in Record Dictionary Tool setitem with name "temp". Valid types are numpy.ndarray and RecordDictionaryTool''' ) # Now test rank zero array... with self.assertRaises(ValueError) as te: rdt['temp'] = numpy.array(22.5) self.assertEquals( te.exception.message, '''The rank of a value sequence array in a record dictionary must be greater than zero. Got name "temp" with rank "0"''' ) #TODO: Fix record_dictionary shape validation, then put this test back in # # Test set invalid shape # pres_array = numpy.random.standard_normal(90) # with self.assertRaises(ValueError) as te: # rdt['pres'] = pres_array # # self.assertEquals( # te.exception.message, # '''Invalid array shape "(90,)" for name "pres"; Record dictionary defined shape is "(100,)"''' # ) # make a new RDT for testing higher rank objects... rdt = RecordDictionaryTool(param_dictionary=self._pdict) # Now test rank 2 array... rdt['temp'] = numpy.array([[22.5,],]) self.assertTrue((rdt['temp'] == numpy.array([[22.5,],])).all()) # Now test rank 2 array... rdt['conductivity'] = numpy.array([[28.5,],]) self.assertTrue((rdt['conductivity'] == numpy.array([[28.5,],])).all()) def test_set_and_get_with_taxonomy(self): #make sure you can set and get items in the granule by name in the taxonomy temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self.assertRaises(KeyError, self._rdt.__setitem__, 'long_temp_name',temp_array) self.assertRaises(KeyError, self._rdt.__setitem__, 'nonsense',temp_array) self._rdt['temp'] = temp_array self._rdt['cond'] = cond_array self._rdt['pres'] = pres_array self.assertTrue(numpy.allclose(self._rdt['temp'], temp_array)) self.assertTrue(numpy.allclose(self._rdt['cond'], cond_array)) self.assertTrue(numpy.allclose(self._rdt['pres'], pres_array)) #want to check to make sure a KeyError is raised when a non-nickname key is used, but it's not working correctly self.assertRaises(KeyError, self._rdt.__getitem__, 'long_temp_name') self.assertRaises(KeyError, self._rdt.__getitem__,'nonsense!') taxy_tool_obj =self._tx rdt = RecordDictionaryTool(taxonomy=taxy_tool_obj) rdt['temp'] = temp_array self._rdt['rdt'] = rdt # Now test when the Record Dictionary Tool is created with the Taxonomy object rather than the TaxyTool # This can fail if the == method for TaxyTool is implemented incorrectly taxonomy_ion_obj = self._tx._t rdt2 = RecordDictionaryTool(taxonomy=taxonomy_ion_obj) rdt2['temp'] = temp_array self._rdt['rdt2'] = rdt2 # Now test bad values... list not numpy array... with self.assertRaises(TypeError) as te: rdt2['temp'] = [1,2,3] self.assertEquals( te.exception.message, '''Invalid type "<type 'list'>" in Record Dictionary Tool setitem with name "temp". Valid types are numpy.ndarray and RecordDictionaryTool''' ) # Now test numpy scalar array... with self.assertRaises(TypeError) as te: rdt2['temp'] = numpy.float32(3.14159) self.assertEquals( te.exception.message, '''Invalid type "<type 'numpy.float32'>" in Record Dictionary Tool setitem with name "temp". Valid types are numpy.ndarray and RecordDictionaryTool''' ) # Now test rank zero array... with self.assertRaises(ValueError) as te: rdt2['temp'] = numpy.array(22.5) self.assertEquals( te.exception.message, '''The rank of a value sequence array in a record dictionary must be greater than zero. Got name "temp" with rank "0"''' ) #TODO: Fix record_dictionary shape validation, then put this test back in # # Test set invalid shape # pres_array = numpy.random.standard_normal(90) # with self.assertRaises(ValueError) as te: # rdt2['pres'] = pres_array # # self.assertEquals( # te.exception.message, # '''Invalid array shape "(90,)" for name "pres"; Record dictionary defined shape is "(100,)"''' # ) # make a new RDT for testing higher rank objects... taxy_tool_obj =self._tx rdt = RecordDictionaryTool(taxonomy=taxy_tool_obj) # Now test rank 2 array... rdt['temp'] = numpy.array([[22.5,],]) self.assertTrue((rdt['temp'] == numpy.array([[22.5,],])).all()) # Now test rank 2 array... rdt['cond'] = numpy.array([[28.5,],]) self.assertTrue((rdt['cond'] == numpy.array([[28.5,],])).all()) def test_iteration(self): #Test all four iteration methods for items in the granule temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt_pdict['temp'] = temp_array self._rdt_pdict['conductivity'] = cond_array self._rdt_pdict['pres'] = pres_array for k, v in self._rdt_pdict.iteritems(): if k == 'temp': self.assertTrue(numpy.allclose(temp_array, v)) elif k == 'conductivity': self.assertTrue(numpy.allclose(cond_array, v)) elif k == 'pres': self.assertTrue(numpy.allclose(pres_array, v)) else: self.assertTrue(False) for k in self._rdt_pdict.iterkeys(): self.assertTrue(k == 'temp' or k == 'conductivity' or k == 'pres') for v in self._rdt_pdict.itervalues(): self.assertTrue(numpy.allclose(temp_array, v) or numpy.allclose(cond_array, v) or numpy.allclose(pres_array, v)) for k in self._rdt_pdict: self.assertTrue(k == 'temp' or k == 'conductivity' or k == 'pres') def test_iteration_with_taxonomy(self): #Test all four iteration methods for items in the granule temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt['temp'] = temp_array self._rdt['cond'] = cond_array self._rdt['pres'] = pres_array for k, v in self._rdt.iteritems(): if k == 'temp': self.assertTrue(numpy.allclose(temp_array, v)) elif k == 'cond': self.assertTrue(numpy.allclose(cond_array, v)) elif k == 'pres': self.assertTrue(numpy.allclose(pres_array, v)) else: self.assertTrue(False) for k in self._rdt.iterkeys(): self.assertTrue(k == 'temp' or k == 'cond' or k == 'pres') for v in self._rdt.itervalues(): self.assertTrue(numpy.allclose(temp_array, v) or numpy.allclose(cond_array, v) or numpy.allclose(pres_array, v)) for k in self._rdt: self.assertTrue(k == 'temp' or k == 'cond' or k == 'pres') def test_update(self): # Update this granule with the content of another. Assert that the taxonomies are the same... pres_array = numpy.random.standard_normal(100) self._rdt_pdict['pres'] = pres_array rdt2 = RecordDictionaryTool(param_dictionary=self._pdict) temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) rdt2['temp'] = temp_array rdt2['conductivity'] = cond_array self._rdt_pdict.update(rdt2) self.assertIn('pres', self._rdt_pdict) self.assertIn('temp', self._rdt_pdict) self.assertIn('conductivity', self._rdt_pdict) self.assertTrue((self._rdt_pdict['pres'] == pres_array).all()) self.assertTrue((self._rdt_pdict['conductivity'] == cond_array).all()) self.assertTrue((self._rdt_pdict['temp'] == temp_array).all()) self.assertEquals(len(self._rdt_pdict), 3) def test_update_with_taxonomy(self): # Update this granule with the content of another. Assert that the taxonomies are the same... pres_array = numpy.random.standard_normal(100) self._rdt['pres'] = pres_array rdt2 = RecordDictionaryTool(taxonomy=self._tx) temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) rdt2['temp'] = temp_array rdt2['cond'] = cond_array self._rdt.update(rdt2) self.assertIn('pres', self._rdt) self.assertIn('temp', self._rdt) self.assertIn('cond', self._rdt) self.assertTrue((self._rdt['pres'] == pres_array).all()) self.assertTrue((self._rdt['cond'] == cond_array).all()) self.assertTrue((self._rdt['temp'] == temp_array).all()) self.assertEquals(len(self._rdt), 3) def test_len(self): temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt_pdict['temp'] = temp_array self._rdt_pdict['conductivity'] = cond_array self._rdt_pdict['pres'] = pres_array self.assertEquals(len(self._rdt_pdict), 3) def test_len_with_taxonomy(self): temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt['temp'] = temp_array self._rdt['cond'] = cond_array self._rdt['pres'] = pres_array self.assertEquals(len(self._rdt), 3) def test_repr(self): # Come up with a reasonable string representation of the granule for debug purposes only temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt_pdict['temp'] = temp_array self._rdt_pdict['conductivity'] = cond_array self._rdt_pdict['pres'] = pres_array self.assertTrue(len(repr(self._rdt_pdict)) > 0) def test_repr_with_taxonomy(self): # Come up with a reasonable string representation of the granule for debug purposes only temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt['temp'] = temp_array self._rdt['cond'] = cond_array self._rdt['pres'] = pres_array self.assertTrue(len(repr(self._rdt)) > 0) def test_delete(self): temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt_pdict['temp'] = temp_array self._rdt_pdict['conductivity'] = cond_array self._rdt_pdict['pres'] = pres_array self.assertIn('pres', self._rdt_pdict) self.assertIn('temp', self._rdt_pdict) self.assertIn('conductivity', self._rdt_pdict) del self._rdt_pdict['pres'] self.assertNotIn('pres', self._rdt_pdict) self.assertIn('temp', self._rdt_pdict) self.assertIn('conductivity', self._rdt_pdict) def test_delete_with_taxonomy(self): temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt['temp'] = temp_array self._rdt['cond'] = cond_array self._rdt['pres'] = pres_array self.assertIn('pres', self._rdt) self.assertIn('temp', self._rdt) self.assertIn('cond', self._rdt) del self._rdt['pres'] self.assertNotIn('pres', self._rdt) self.assertIn('temp', self._rdt) self.assertIn('cond', self._rdt) def test_contains(self): # foobar isn't even in the taxonomy! self.assertNotIn('foobar', self._rdt_pdict) # Temp is in the ParameterDictionary but not the record dictionary self.assertNotIn('temp', self._rdt_pdict) # Now put in some data and make sure it works... temp_array = numpy.random.standard_normal(100) self._rdt_pdict['temp'] = temp_array self.assertIn('temp', self._rdt_pdict) def test_contains_with_taxonomy(self): # foobar isn't even in the taxonomy! self.assertNotIn('foobar', self._rdt) # Temp is in the taxonomy but not the record dictionary self.assertNotIn('temp', self._rdt) # Now put in some data and make sure it works... temp_array = numpy.random.standard_normal(100) self._rdt['temp'] = temp_array self.assertIn('temp', self._rdt) @unittest.skip('Pretty print not implemented at this time') def test_pretty_print(self): temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt_pdict['temp'] = temp_array self._rdt_pdict['conductivity'] = cond_array self._rdt_pdict['pres'] = pres_array self.assertGreater(len(self._rdt_pdict.pretty_print()), 0) @unittest.skip('Pretty print not implemented at this time') def test_pretty_print_with_taxonomy(self): temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) self._rdt['temp'] = temp_array self._rdt['cond'] = cond_array self._rdt['pres'] = pres_array rdt = RecordDictionaryTool(taxonomy=self._tx) rdt['rdt'] = temp_array self._rdt['rdt'] = rdt self.assertGreater(len(self._rdt.pretty_print()), 0)