def check_density_algorithm_execution(self, publish_granule, granule_from_transform): #------------------------------------------------------------------ # Calculate the correct density from the input granule data #------------------------------------------------------------------ input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule) output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform) conductivity = input_rdt_to_transform['conductivity'] pressure = input_rdt_to_transform['pressure'] temperature = input_rdt_to_transform['temp'] longitude = input_rdt_to_transform['lon'] latitude = input_rdt_to_transform['lat'] sp = SP_from_cndr(r=conductivity/cte.C3515, t=temperature, p=pressure) sa = SA_from_SP(sp, pressure, longitude, latitude) dens_value = rho(sa, temperature, pressure) out_density = output_rdt_transform['density'] #----------------------------------------------------------------------------- # Check that the output data from the transform has the correct density values #----------------------------------------------------------------------------- self.assertTrue(dens_value.all() == out_density.all())
def check_density_algorithm_execution(self, publish_granule, granule_from_transform): #------------------------------------------------------------------ # Calculate the correct density from the input granule data #------------------------------------------------------------------ input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule) output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform) conductivity = input_rdt_to_transform['conductivity'] pressure = input_rdt_to_transform['pressure'] temperature = input_rdt_to_transform['temp'] longitude = input_rdt_to_transform['lon'] latitude = input_rdt_to_transform['lat'] sp = SP_from_cndr(r=conductivity/cte.C3515, t=temperature, p=pressure) sa = SA_from_SP(sp, pressure, longitude, latitude) dens_value = rho(sa, temperature, pressure) out_density = output_rdt_transform['density'] log.debug("density output from the transform: %s", out_density) log.debug("values of density expected from the transform: %s", dens_value) numpy.testing.assert_array_almost_equal(out_density, dens_value, decimal=3)
def check_density_algorithm_execution(self, publish_granule, granule_from_transform): #------------------------------------------------------------------ # Calculate the correct density from the input granule data #------------------------------------------------------------------ input_rdt_to_transform = RecordDictionaryTool.load_from_granule( publish_granule) output_rdt_transform = RecordDictionaryTool.load_from_granule( granule_from_transform) conductivity = input_rdt_to_transform['conductivity'] pressure = input_rdt_to_transform['pressure'] temperature = input_rdt_to_transform['temp'] longitude = input_rdt_to_transform['lon'] latitude = input_rdt_to_transform['lat'] sp = SP_from_cndr(r=conductivity / cte.C3515, t=temperature, p=pressure) sa = SA_from_SP(sp, pressure, longitude, latitude) dens_value = rho(sa, temperature, pressure) out_density = output_rdt_transform['density'] log.debug("density output from the transform: %s", out_density) log.debug("values of density expected from the transform: %s", dens_value) numpy.testing.assert_array_almost_equal(out_density, dens_value, decimal=3)
def check_salinity_algorithm_execution(self, publish_granule, granule_from_transform): #------------------------------------------------------------------ # Calculate the correct density from the input granule data #------------------------------------------------------------------ input_rdt_to_transform = RecordDictionaryTool.load_from_granule( publish_granule) output_rdt_transform = RecordDictionaryTool.load_from_granule( granule_from_transform) conductivity = input_rdt_to_transform['conductivity'] pressure = input_rdt_to_transform['pressure'] temperature = input_rdt_to_transform['temp'] sal_value = SP_from_cndr(r=conductivity / cte.C3515, t=temperature, p=pressure) out_salinity = output_rdt_transform['salinity'] #----------------------------------------------------------------------------- # Check that the output data from the transform has the correct density values #----------------------------------------------------------------------------- self.assertTrue(numpy.array_equal(sal_value, out_salinity))
def validate_multiple_vis_queue_messages(self, msg1, msg2): assertions = self.assertTrue # verify that the salinity in msg2 is a result of content from msg1 rdt1 = RecordDictionaryTool.load_from_granule(msg1) rdt2 = RecordDictionaryTool.load_from_granule(msg2) # msg1 should not have salinity # assertions(rdt1['salinity'] == None) conductivity = rdt1['conductivity'] pressure = rdt1['pressure'] temperature = rdt1['temp'] msg1_sal_value = SP_from_cndr(r=conductivity / cte.C3515, t=temperature, p=pressure) msg2_sal_value = rdt2['salinity'] b = msg1_sal_value == msg2_sal_value if isinstance(b, bool): assertions(b) else: assertions(b.all()) return
def validate_multiple_vis_queue_messages(self, msg1, msg2): assertions = self.assertTrue # verify that the salinity in msg2 is a result of content from msg1 rdt1 = RecordDictionaryTool.load_from_granule(msg1) rdt2 = RecordDictionaryTool.load_from_granule(msg2) # msg1 should not have salinity # assertions(rdt1['salinity'] == None) conductivity = rdt1['conductivity'] pressure = rdt1['pressure'] temperature = rdt1['temp'] msg1_sal_value = SP_from_cndr(r=conductivity/cte.C3515, t=temperature, p=pressure) msg2_sal_value = rdt2['salinity'] b = msg1_sal_value == msg2_sal_value if isinstance(b,bool): assertions(b) else: assertions(b.all()) return
def check_pres_algorithm_execution(self, publish_granule, granule_from_transform): input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule) output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform) output_data = output_rdt_transform['pressure'] input_data = input_rdt_to_transform['pressure'] self.assertTrue(input_data.all() == output_data.all())
def check_cond_algorithm_execution(self, publish_granule, granule_from_transform): input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule) output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform) output_data = output_rdt_transform["conductivity"] input_data = input_rdt_to_transform["conductivity"] self.assertTrue(((input_data / 100000.0) - 0.5).all() == output_data.all())
def check_temp_algorithm_execution(self, publish_granule, granule_from_transform): input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule) output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform) output_data = output_rdt_transform['temp'] input_data = input_rdt_to_transform['temp'] self.assertTrue(((input_data / 10000.0) - 10).all() == output_data.all())
def check_pres_algorithm_execution(self, publish_granule, granule_from_transform): input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule) output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform) output_data = output_rdt_transform['pressure'] input_data = input_rdt_to_transform['pressure'] self.assertTrue(numpy.array_equal((input_data/ 100.0) + 0.5,output_data))
def test_stream_ingestion_worker(self): self.start_ingestion_worker() context_ids, time_ctxt = self._create_param_contexts() pdict_id = self.dataset_management_client.create_parameter_dictionary( name='stream_ingestion_pdict', parameter_context_ids=context_ids, temporal_context='ingestion_timestamp') self.addCleanup( self.dataset_management_client.delete_parameter_dictionary, pdict_id) dataset_id = self.dataset_management_client.create_dataset( name='fake_dataset', description='fake_dataset', stream_id=self.stream_id, spatial_domain=self.spatial_dom.dump(), temporal_domain=self.time_dom.dump(), parameter_dictionary_id=pdict_id) self.addCleanup(self.dataset_management_client.delete_dataset, dataset_id) self.cov = self._create_coverage(dataset_id=dataset_id, parameter_dict_id=pdict_id, time_dom=self.time_dom, spatial_dom=self.spatial_dom) self.addCleanup(self.cov.close) rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id) rdt['conductivity'] = 1 rdt['pressure'] = 2 rdt['salinity'] = 3 self.start_listener(dataset_id) self.publisher.publish(rdt.to_granule()) self.data_modified = Event() self.data_modified.wait(30) cov = self.get_coverage(dataset_id) self.assertIsNotNone(cov.get_parameter_values('raw')) deserializer = IonObjectDeserializer(obj_registry=get_obj_registry()) granule = retrieve_stream(dataset_id) rdt_complex = RecordDictionaryTool.load_from_granule(granule) rdt_complex['raw'] = [ deserializer.deserialize(i) for i in rdt_complex['raw'] ] for gran in rdt_complex['raw']: rdt_new = RecordDictionaryTool.load_from_granule(gran) self.assertIn(1, rdt_new['conductivity']) self.assertIn(2, rdt_new['pressure']) self.assertIn(3, rdt_new['salinity']) cov.close()
def recv_packet(self, msg, stream_route, stream_id): ''' receive packet for ingestion ''' log.debug('received granule for stream %s', stream_id) if msg == {}: log.error('Received empty message from stream: %s', stream_id) return # Message validation if not isinstance(msg, Granule): log.error('Ingestion received a message that is not a granule: %s', msg) return rdt = RecordDictionaryTool.load_from_granule(msg) if rdt is None: log.error('Invalid granule (no RDT) for stream %s', stream_id) return if not len(rdt): log.debug('Empty granule for stream %s', stream_id) return dp_id_list = self.retrieve_dataprocess_for_stream(stream_id) for dp_id in dp_id_list: function, argument_list = self.retrieve_function_and_define_args(dp_id) args = [] rdt = RecordDictionaryTool.load_from_granule(msg) #create the input arguments list #todo: this logic is tied to the example funcation, generalize for func_param, record_param in argument_list.iteritems(): args.append(rdt[record_param]) try: #run the calc #todo: nothing in the data process resource to specify multi-out map result = function(*args) out_stream_definition, output_parameter = self.retrieve_dp_output_params(dp_id) rdt = RecordDictionaryTool(stream_definition_id=out_stream_definition) publisher = self._publisher_map.get(dp_id,'') rdt[ output_parameter ] = result if publisher: publisher.publish(rdt.to_granule()) else: log.error('Publisher not found for data process %s', dp_id) self.update_dp_metrics( dp_id ) except ImportError: log.error('Error running transform')
def check_temp_algorithm_execution(self, publish_granule, granule_from_transform): input_rdt_to_transform = RecordDictionaryTool.load_from_granule( publish_granule) output_rdt_transform = RecordDictionaryTool.load_from_granule( granule_from_transform) output_data = output_rdt_transform['temp'] input_data = input_rdt_to_transform['temp'] self.assertTrue( numpy.array_equal(((input_data / 10000.0) - 10), output_data))
def check_cond_algorithm_execution(self, publish_granule, granule_from_transform): input_rdt_to_transform = RecordDictionaryTool.load_from_granule( publish_granule) output_rdt_transform = RecordDictionaryTool.load_from_granule( granule_from_transform) output_data = output_rdt_transform['conductivity'] input_data = input_rdt_to_transform['conductivity'] self.assertTrue(((input_data / 100000.0) - 0.5).all() == output_data.all())
def execute(input=None, context=None, config=None, params=None, state=None): rdt = RecordDictionaryTool.load_from_granule(input) out_rdt = RecordDictionaryTool(stream_definition_id=params) conductivity = rdt['conductivity'] pressure = rdt['pressure'] temperature = rdt['temp'] longitude = rdt['lon'] if rdt['lon'] is not None else 0 latitude = rdt['lat'] if rdt['lat'] is not None else 0 sp = SP_from_cndr(r=conductivity/cte.C3515, t=temperature, p=pressure) log.debug("Density algorithm calculated the sp (practical salinity) values: %s", sp) sa = SA_from_SP(sp, pressure, longitude, latitude) log.debug("Density algorithm calculated the sa (actual salinity) values: %s", sa) dens_value = rho(sa, temperature, pressure) for key, value in rdt.iteritems(): if key in out_rdt: if key=='conductivity' or key=='temp' or key=='pressure': continue out_rdt[key] = value[:] out_rdt['density'] = dens_value log.debug("Density algorithm returning density values: %s", out_rdt['density']) return out_rdt.to_granule()
def validate_mpl_graphs_transform_results(self, results): cc = self.container assertions = self.assertTrue # if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases if isinstance(results,Granule): results =[results] found_data = False for g in results: if isinstance(g,Granule): rdt = RecordDictionaryTool.load_from_granule(g) graphs = get_safe(rdt, 'matplotlib_graphs') if graphs == None: continue for graph in graphs[0]: # At this point only dictionaries containing image data should be passed # For some reason non dictionary values are filtering through. if not isinstance(graph, dict): continue assertions(graph['viz_product_type'] == 'matplotlib_graphs' ) # check to see if the list (numpy array) contains actual images assertions(imghdr.what(graph['image_name'], h = graph['image_obj']) == 'png') found_data = True return found_data
def _execute_transform(self, msg, streams): stream_in_id,stream_out_id = streams stream_def_in = self.read_stream_def(stream_in_id) stream_def_out = self.read_stream_def(stream_out_id) rdt_temp = self._merge_rdt(stream_def_in, stream_def_out) rdt_in = RecordDictionaryTool.load_from_granule(msg) for field in rdt_temp.fields: if not isinstance(rdt_temp._pdict.get_context(field).param_type, ParameterFunctionType): try: rdt_temp[field] = rdt_in[field] except KeyError: pass rdt_temp.fetch_lookup_values() for lookup_field in rdt_temp.lookup_values(): s = lookup_field stored_value = self._get_lookup_value(rdt_temp.context(s).lookup_value) if stored_value is not None: rdt_temp[s] = stored_value for field in rdt_temp.fields: if isinstance(rdt_temp._pdict.get_context(field).param_type, ParameterFunctionType): rdt_temp[field] = rdt_temp[field] rdt_out = RecordDictionaryTool(stream_definition_id=stream_def_out._id) for field in rdt_out.fields: rdt_out[field] = rdt_temp[field] return rdt_out
def recv_packet(self, msg, stream_route, stream_id): ''' The consumer callback to parse and manage the granule. The message is ACK'd once the function returns ''' log.trace('received granule for stream %s', stream_id) if msg == {}: log.error('Received empty message from stream: %s', stream_id) return # Message validation if not isinstance(msg, Granule): log.error('Ingestion received a message that is not a granule: %s', msg) return rdt = RecordDictionaryTool.load_from_granule(msg) if rdt is None: log.error('Invalid granule (no RDT) for stream %s', stream_id) return if not len(rdt): log.debug('Empty granule for stream %s', stream_id) return self.persist_or_timeout(stream_id, rdt)
def validate_highcharts_transform_results(self, results): assertions = self.assertTrue # if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases if isinstance(results,Granule): results =[results] for g in results: if isinstance(g,Granule): rdt = RecordDictionaryTool.load_from_granule(g) hc_data_arr = get_safe(rdt, 'hc_data') if hc_data_arr == None: log.debug("hc_data in granule is None") continue assertions(len(hc_data_arr) >= 0) # Need to come up with a better check hc_data = hc_data_arr[0] assertions(len(hc_data) >= 0) assertions(len(hc_data[0]["name"]) >= 0) assertions(len(hc_data[0]["data"]) >= 0)
def execute(input=None, context=None, config=None, params=None, state=None): """ @param input granule @retval result_list list of dictionaries containing granules as values """ result_list = [] for x in input: rdt = RecordDictionaryTool.load_from_granule(x) conductivity = rdt['conductivity'] pressure = rdt['pressure'] temperature = rdt['temperature'] time = rdt['time'] # build the granule for conductivity, temperature and pressure granule = ctdbp_L0_algorithm._build_granule( stream_definition_id=params['L0_stream'], field_names=[ 'conductivity', 'pressure', 'temperature', 'time' ], # these are the field names for the output record dictionary values=[conductivity, pressure, temperature, time]) result_list.append(granule) return result_list
def validate_output_granule(self, msg, route, stream_id): self.assertIn(stream_id, self._output_stream_ids) rdt = RecordDictionaryTool.load_from_granule(msg) log.debug('validate_output_granule rdt: %s', rdt) sal_val = rdt['salinity'] np.testing.assert_array_equal(sal_val, np.array([3]))
def validate_google_dt_transform_results(self, results): cc = self.container assertions = self.assertTrue # if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases if isinstance(results,Granule): results =[results] for g in results: if isinstance(g,Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) gdt_data = get_safe(rdt, 'google_dt_components') # IF this granule does not contains google dt, skip if gdt_data == None: continue gdt = gdt_data[0] assertions(gdt['viz_product_type'] == 'google_dt' ) assertions(len(gdt['data_description']) >= 0) # Need to come up with a better check assertions(len(gdt['data_content']) >= 0)
def execute(input=None, context=None, config=None, params=None, state=None): rdt = RecordDictionaryTool.load_from_granule(input) out_rdt = RecordDictionaryTool(stream_definition_id=params) conductivity = rdt["conductivity"] pressure = rdt["pressure"] temperature = rdt["temp"] longitude = rdt["lon"] if rdt["lon"] is not None else 0 latitude = rdt["lat"] if rdt["lat"] is not None else 0 sp = SP_from_cndr(r=conductivity / cte.C3515, t=temperature, p=pressure) log.debug("Density algorithm calculated the sp (practical salinity) values: %s", sp) sa = SA_from_SP(sp, pressure, longitude, latitude) log.debug("Density algorithm calculated the sa (actual salinity) values: %s", sa) dens_value = rho(sa, temperature, pressure) for key, value in rdt.iteritems(): if key in out_rdt: if key == "conductivity" or key == "temp" or key == "pressure": continue out_rdt[key] = value[:] out_rdt["density"] = dens_value log.debug("Density algorithm returning density values: %s", out_rdt["density"]) return out_rdt.to_granule()
def validate_output_granule(self, msg, route, stream_id): self.assertIn( stream_id, self._output_stream_ids) rdt = RecordDictionaryTool.load_from_granule(msg) log.debug('validate_output_granule rdt: %s', rdt) sal_val = rdt['salinity'] np.testing.assert_array_equal(sal_val, np.array([3]))
def execute(input=None, context=None, config=None, params=None, state=None): rdt = RecordDictionaryTool.load_from_granule(input) out_rdt = RecordDictionaryTool(stream_definition_id=params) conductivity = rdt['conductivity'] pressure = rdt['pressure'] temperature = rdt['temp'] sal_value = SP_from_cndr(r=conductivity / cte.C3515, t=temperature, p=pressure) log.debug( "Salinity algorithm calculated the sp (practical salinity) values: %s", sal_value) for key, value in rdt.iteritems(): if key in out_rdt: if key == 'conductivity' or key == 'temp' or key == 'pressure': continue out_rdt[key] = value[:] out_rdt['salinity'] = sal_value return out_rdt.to_granule()
def validate_mpl_graphs_transform_results(self, results): cc = self.container assertions = self.assertTrue # if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases if isinstance(results, Granule): results = [results] found_data = False for g in results: if isinstance(g, Granule): rdt = RecordDictionaryTool.load_from_granule(g) graphs = get_safe(rdt, 'matplotlib_graphs') if graphs == None: continue for graph in graphs[0]: # At this point only dictionaries containing image data should be passed # For some reason non dictionary values are filtering through. if not isinstance(graph, dict): continue assertions( graph['viz_product_type'] == 'matplotlib_graphs') # check to see if the list (numpy array) contains actual images assertions( imghdr.what(graph['image_name'], h=graph['image_obj']) == 'png') found_data = True return found_data
def execute(input=None, context=None, config=None, params=None, state=None): """ Find if the input data has values, which are out of range @param input granule @param context parameter context @param config DotDict @param params list @param state @return bad_values, bad_value_times tuple of lists """ rdt = RecordDictionaryTool.load_from_granule(input) # Retrieve the name used for the variable_name, the name used for timestamps and the range of valid values from the config valid_values = config.get_safe('valid_values', [-100,100]) variable_name = config.get_safe('variable_name', 'input_voltage') time_field_name = config.get_safe('time_field_name', 'preferred_timestamp') # These variable_names will store the bad values and the timestamps of those values bad_values = [] bad_value_times = [] # retrieve the values and the times from the record dictionary values = rdt[variable_name][:] times = rdt[time_field_name][:] for val, t in zip(values, times): if val < valid_values[0] or val > valid_values[1]: bad_values.append(val) bad_value_times.append(t) # return the list of bad values and their timestamps return bad_values, bad_value_times
def recv_packet(self, msg, stream_route, stream_id): ''' The consumer callback to parse and manage the granule. The message is ACK'd once the function returns ''' log.trace('received granule for stream %s', stream_id) if msg == {}: log.error('Received empty message from stream: %s', stream_id) return # Message validation if not isinstance(msg, Granule): log.error('Ingestion received a message that is not a granule: %s', msg) return rdt = RecordDictionaryTool.load_from_granule(msg) if rdt is None: log.error('Invalid granule (no RDT) for stream %s', stream_id) return if not len(rdt): log.debug('Empty granule for stream %s', stream_id) return self.persist_or_timeout(stream_id, rdt)
def add_granule(self,stream_id, granule): ''' Appends the granule's data to the coverage and persists it. ''' #-------------------------------------------------------------------------------- # Coverage determiniation and appending #-------------------------------------------------------------------------------- dataset_id = self.get_dataset(stream_id) if not dataset_id: log.error('No dataset could be determined on this stream: %s', stream_id) return coverage = self.get_coverage(stream_id) if not coverage: log.error('Could not persist coverage from granule, coverage is None') return #-------------------------------------------------------------------------------- # Actual persistence #-------------------------------------------------------------------------------- rdt = RecordDictionaryTool.load_from_granule(granule) elements = len(rdt) if not elements: return coverage.insert_timesteps(elements) start_index = coverage.num_timesteps - elements for k,v in rdt.iteritems(): if k == 'image_obj': log.trace( '%s:', k) else: log.trace( '%s: %s', k, v) slice_ = slice(start_index, None) coverage.set_parameter_values(param_name=k, tdoa=slice_, value=v) coverage.flush()
def execute(input=None, context=None, config=None, params=None, state=None): rdt = RecordDictionaryTool.load_from_granule(input) out_rdt = RecordDictionaryTool(stream_definition_id=params['stream_def_id']) # Fill the time values out_rdt['time'] = rdt['time'] # The calibration coefficients temp_calibration_coeffs= params['calibration_coeffs']['temp_calibration_coeffs'] pres_calibration_coeffs= params['calibration_coeffs']['pres_calibration_coeffs'] cond_calibration_coeffs = params['calibration_coeffs']['cond_calibration_coeffs'] log.debug("params['calibration_coeffs']: %s", params['calibration_coeffs']) # Set the temperature values for the output granule out_rdt = CTDBP_L1_TransformAlgorithm.calculate_temperature( input_rdt = rdt, out_rdt = out_rdt, temp_calibration_coeffs= temp_calibration_coeffs ) # Set the pressure values for the output granule out_rdt = CTDBP_L1_TransformAlgorithm.calculate_pressure( input_rdt= rdt, out_rdt = out_rdt, pres_calibration_coeffs= pres_calibration_coeffs) # Set the conductivity values for the output granule # Note that since the conductivity caculation depends on whether TEMPWAT_L1, PRESWAT_L1 have been calculated, we need to do this last out_rdt = CTDBP_L1_TransformAlgorithm.calculate_conductivity( input_rdt = rdt, out_rdt = out_rdt, cond_calibration_coeffs = cond_calibration_coeffs ) # build the granule for the L1 stream return out_rdt.to_granule()
def execute(input=None, context=None, config=None, params=None, state=None): ''' @param input Granule @retval result Granule ''' rdt = RecordDictionaryTool.load_from_granule(input) out_rdt = RecordDictionaryTool(stream_definition_id=params) conductivity = rdt['conductivity'] cond_value = (conductivity / 100000.0) - 0.5 for key, value in rdt.iteritems(): if key in out_rdt: out_rdt[key] = value[:] # Update the conductivity values out_rdt['conductivity'] = cond_value # build the granule for conductivity return out_rdt.to_granule()
def recv_packet(self, packet, stream_route, stream_id): if packet == {}: return l0_values = RecordDictionaryTool.load_from_granule(packet) l1_values = RecordDictionaryTool( stream_definition_id=self.stream_definition_id) log.debug( "CTDBP L1 transform using L0 values: tempurature %s, pressure %s, conductivity %s", l0_values['temperature'], l0_values['pressure'], l0_values['conductivity']) #for key, value in 'lat', 'lon', 'time', ...: <-- do we want to be a little more specific here? for key, value in l0_values.iteritems(): if key in l1_values: l1_values[key] = value[:] l1_values['temp'] = self.calculate_temperature(l0=l0_values) l1_values['pressure'] = self.calculate_pressure(l0=l0_values) l1_values['conductivity'] = self.calculate_conductivity(l0=l0_values, l1=l1_values) log.debug( 'calculated L1 values: temp %s, pressure %s, conductivity %s', l1_values['temp'], l1_values['pressure'], l1_values['conductivity']) self.publisher.publish(msg=l1_values.to_granule())
def execute(self, granule): """Processes incoming data!!!! """ rdt = RecordDictionaryTool.load_from_granule(granule) #todo: use only flat dicts for now, may change later... # rdt0 = rdt['coordinates'] # rdt1 = rdt['data'] pressure = get_safe(rdt, 'pres') #psd.get_values('conductivity') longitude = get_safe(rdt, 'lon') # psd.get_values('longitude') latitude = get_safe(rdt, 'lat') #psd.get_values('latitude') time = get_safe(rdt, 'time') # psd.get_values('time') height = get_safe(rdt, 'height') # psd.get_values('time') log.warn('Got pressure: %s' % str(pressure)) # L1 # 1) The algorithm input is the L0 pressure data product (p_hex) and, in the case of the SBE 37IM, the pressure range (P_rng) from metadata. # 2) Convert the hexadecimal string to a decimal string # 3) For the SBE 37IM only, convert the pressure range (P_rng) from psia to dbar SBE 37IM # Convert P_rng (input from metadata) from psia to dbar # 4) Perform scaling operation # SBE 37IM # L1 pressure data product (in dbar): # Use the constructor to put data into a granule psc = PointSupplementConstructor(point_definition=self.outgoing_stream_def, stream_id=self.streams['output']) ### Assumes the config argument for output streams is known and there is only one 'output'. ### the stream id is part of the metadata which much go in each stream granule - this is awkward to do at the ### application level like this! scaled_pressure = pressure for i in xrange(len(pressure)): #todo: get pressure range from metadata (if present) and include in calc scaled_pressure[i] = ( pressure[i]) root_rdt = RecordDictionaryTool(taxonomy=self.tx) #todo: use only flat dicts for now, may change later... # data_rdt = RecordDictionaryTool(taxonomy=self.tx) # coord_rdt = RecordDictionaryTool(taxonomy=self.tx) root_rdt['pres'] = scaled_pressure root_rdt['time'] = time root_rdt['lat'] = latitude root_rdt['lon'] = longitude root_rdt['height'] = height # root_rdt['coordinates'] = coord_rdt # root_rdt['data'] = data_rdt return build_granule(data_producer_id='ctd_L1_pressure', taxonomy=self.tx, record_dictionary=root_rdt) return psc.close_stream_granule()
def execute(input=None, context=None, config=None, params=None, state=None, fileName = None): stream_definition_id = params mpl_allowed_numerical_types = ['int32', 'int64', 'uint32', 'uint64', 'float32', 'float64'] if stream_definition_id == None: log.error("Matplotlib transform: Need a output stream definition to process graphs") return None # parse the incoming data rdt = RecordDictionaryTool.load_from_granule(input) # build a list of fields/variables that need to be plotted. Use the list provided by the UI # since the retrieved granule might have extra fields. fields = rdt.fields resolution = "640x480" if config: if 'parameters' in config: fields = config['parameters'] if 'resolution' in config: resolution = config['resolution'] vardict = {} vardict['time'] = get_safe(rdt, 'time') if vardict['time'] == None: print "Matplotlib transform: Did not receive a time field to work with" log.error("Matplotlib transform: Did not receive a time field to work with") return None for field in fields: if field == 'time': continue # only consider fields which are supposed to be numbers. if (rdt[field] != None) and (rdt[field].dtype not in mpl_allowed_numerical_types): continue vardict[field] = get_safe(rdt, field) print arrLen = len(vardict['time']) # init the graph_data structure for storing values graph_data = {} for varname in vardict.keys(): graph_data[varname] = [] # If code reached here, the graph data storage has been initialized. Just add values # to the list for varname in vardict.keys(): # psd.list_field_names(): if vardict[varname] == None: # create an array of zeros to compensate for missing values graph_data[varname].extend([0.0]*arrLen) else: graph_data[varname].extend(vardict[varname]) out_granule = VizTransformMatplotlibGraphsAlgorithm.render_graphs(graph_data, stream_definition_id, fileName, resolution=resolution) return out_granule
def _check_application_of_L1_algorithm(self, granule = None): """ Check the algorithm applied by the L1 transform """ rdt = RecordDictionaryTool.load_from_granule(granule) list_of_expected_keys = [ 'time', 'pressure', 'conductivity', 'temp'] for key in list_of_expected_keys: self.assertIn(key, rdt)
def validate_data_ingest_retrieve(self, dataset_id): assertions = self.assertTrue #validate that data was ingested replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id, 10) rdt = RecordDictionaryTool.load_from_granule(replay_granule) salinity = get_safe(rdt, 'salinity') assertions(salinity != None) #retrieve all the granules from the database and check the values replay_granule_all = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule_all) for k, v in rdt.iteritems(): if k == 'salinity': for val in numpy.nditer(v): assertions(val > 0)
def _check_application_of_L2_salinity_algorithm(self, granule = None): """ Check the algorithm applied by the L2 transform """ rdt = RecordDictionaryTool.load_from_granule(granule) list_of_expected_keys = ['time', 'salinity'] for key in list_of_expected_keys: self.assertIn(key, rdt)
def validate_data_ingest_retrieve(self, dataset_id): assertions = self.assertTrue #validate that data was ingested replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id, 10) rdt = RecordDictionaryTool.load_from_granule(replay_granule) salinity = get_safe(rdt, 'salinity') assertions(salinity != None) #retrieve all the granules from the database and check the values replay_granule_all = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule_all) for k, v in rdt.iteritems(): if k == 'salinity': for val in numpy.nditer(v): assertions(val > 0)
def _check_application_of_L2_salinity_algorithm(self, granule = None): """ Check the algorithm applied by the L2 transform """ rdt = RecordDictionaryTool.load_from_granule(granule) for key, value in rdt.iteritems(): list_of_expected_keys = ['time', 'salinity'] if key not in list_of_expected_keys: self.fail("The L2 salinity transform is sending out data for more parameters than it should")
def execute(input=None, context=None, config=None, params=None, state=None): """ Find if the input data has values, which are out of range @param input granule @param context parameter context @param config DotDict @param params list @param state @return bad_values, bad_value_times tuple of lists """ # Retrieve the name used for the variable_name, the name used for timestamps and the range of valid values from the config valid_values = config.get_safe('valid_values', [-100,100]) variable_name = config.get_safe('variable_name', 'input_voltage') preferred_time = config.get_safe('time_field_name', 'preferred_timestamp') # Get the source of the granules which will be used to set the origin of the DeviceStatusEvent and DeviceCommsEvent events origin = input.data_producer_id if not origin: raise NotFound("The DemoStreamAlertTransform could not figure out the origin for DeviceStatusEvent. The data_producer_id attribute should be filled for the granules that are sent to it so that it can figure out the origin to use.") log.debug("The origin the demo transform is listening to is: %s" % origin) rdt = RecordDictionaryTool.load_from_granule(input) # These variable_names will store the bad values and the timestamps of those values bad_values = [] bad_value_times = [] # retrieve the values and the times from the record dictionary values = rdt[variable_name][:] time_names = rdt[preferred_time][:] log.debug("Values unravelled: %s" % values) log.debug("Time names unravelled: %s" % time_names) log.debug("Transform got the rdt here: %s", rdt) indexes = [l for l in xrange(len(time_names))] for val, index in zip(values, indexes): if val < valid_values[0] or val > valid_values[1]: bad_values.append(val) arr = rdt[time_names[index]] log.debug("In the stream alert transform, got the arr here: %s", arr) if isinstance(arr, numpy.ndarray): bad_value_times.append(arr[index]) else: # Since we do not want the transform to crash because a granule came in that did not have the values of the preferred time filled as expected bad_value_times.append(None) log.debug("Returning bad_values: %s, bad_value_times: %s and the origin: %s" % (bad_values, bad_value_times, origin)) # return the list of bad values and their timestamps return bad_values, bad_value_times, origin
def execute(input=None, context=None, config=None, params=None, state=None): """ Find if the input data has values, which are out of range @param input granule @param context parameter context @param config DotDict @param params list @param state @return bad_values, bad_value_times tuple of lists """ # Retrieve the name used for the variable_name, the name used for timestamps and the range of valid values from the config valid_values = config.get_safe('valid_values', [-100,100]) variable_name = config.get_safe('variable_name', 'input_voltage') preferred_time = config.get_safe('time_field_name', 'preferred_timestamp') # Get the source of the granules which will be used to set the origin of the DeviceStatusEvent and DeviceCommsEvent events origin = input.data_producer_id if not origin: raise NotFound("The DemoStreamAlertTransform could not figure out the origin for DeviceStatusEvent. The data_producer_id attribute should be filled for the granules that are sent to it so that it can figure out the origin to use.") log.debug("The origin the demo transform is listening to is: %s" % origin) rdt = RecordDictionaryTool.load_from_granule(input) # These variable_names will store the bad values and the timestamps of those values bad_values = [] bad_value_times = [] # retrieve the values and the times from the record dictionary values = rdt[variable_name][:] time_names = rdt[preferred_time][:] log.debug("Values unravelled: %s" % values) log.debug("Time names unravelled: %s" % time_names) log.debug("Transform got the rdt here: %s", rdt) indexes = [l for l in xrange(len(time_names))] for val, index in zip(values, indexes): if val < valid_values[0] or val > valid_values[1]: bad_values.append(val) arr = rdt[time_names[index]] log.debug("In the stream alert transform, got the arr here: %s", arr) if isinstance(arr, numpy.ndarray): bad_value_times.append(arr[index]) else: # Since we do not want the transform to crash because a granule came in that did not have the values of the preferred time filled as expected bad_value_times.append(None) log.debug("Returning bad_values: %s, bad_value_times: %s and the origin: %s" % (bad_values, bad_value_times, origin)) # return the list of bad values and their timestamps return bad_values, bad_value_times, origin
def _check_application_of_L2_salinity_algorithm(self, granule=None): """ Check the algorithm applied by the L2 transform """ rdt = RecordDictionaryTool.load_from_granule(granule) for key, value in rdt.iteritems(): list_of_expected_keys = ['time', 'salinity'] if key not in list_of_expected_keys: self.fail( "The L2 salinity transform is sending out data for more parameters than it should" )
def _check_granule_from_transform(self, granule): """ An internal method to check if a granule has the right properties """ rdt = RecordDictionaryTool.load_from_granule(granule) self.assertIn("pressure", rdt) self.assertIn("temp", rdt) self.assertIn("conductivity", rdt) self.assertIn("time", rdt)
def _check_granule_from_transform(self, granule): """ An internal method to check if a granule has the right properties """ rdt = RecordDictionaryTool.load_from_granule(granule) self.assertIn('pressure', rdt) self.assertIn('temp', rdt) self.assertIn('conductivity', rdt) self.assertIn('time', rdt)
def validate_messages(self, msgs): msg = msgs rdt = RecordDictionaryTool.load_from_granule(msg.body) vardict = {} vardict['temp'] = get_safe(rdt, 'temp') vardict['time'] = get_safe(rdt, 'time') print vardict['time'] print vardict['temp']
def recv_packet(self, msg, stream_route, stream_id): from ion.services.dm.utility.granule.record_dictionary import RecordDictionaryTool rdt = RecordDictionaryTool.load_from_granule(msg) #print 'Message Received: {0}'.format(rdt.pretty_print()) flds = '' for field in rdt.fields: flds += field + ' ' print '\n' + flds for i in xrange(len(rdt)): var_tuple = [ float(rdt[field][i]) if rdt[field] is not None else 0.0 for field in rdt.fields] print var_tuple
def check_salinity_algorithm_execution(self, publish_granule, granule_from_transform): #------------------------------------------------------------------ # Calculate the correct density from the input granule data #------------------------------------------------------------------ input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule) output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform) conductivity = input_rdt_to_transform['conductivity'] pressure = input_rdt_to_transform['pressure'] temperature = input_rdt_to_transform['temp'] sal_value = SP_from_cndr(r=conductivity/cte.C3515, t=temperature, p=pressure) out_salinity = output_rdt_transform['salinity'] #----------------------------------------------------------------------------- # Check that the output data from the transform has the correct density values #----------------------------------------------------------------------------- self.assertTrue(sal_value.all() == out_salinity.all())
def recv_packet(self, msg, stream_route, stream_id): if not isinstance(msg,Granule): log.warn('Received non-compatible granule in stream %s.', stream_id) return rdt = RecordDictionaryTool.load_from_granule(msg) if self.log: log.warn('Received record dictionary:\n%s', rdt.pretty_print()) else: import sys print>>sys.stderr, 'Received record dictionary:\n%s' % rdt.pretty_print()
def test_build_granule_and_load_from_granule_with_taxonomy(self): #Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle tx = TaxyTool() tx.add_taxonomy_set('temp', 'long_temp_name') tx.add_taxonomy_set('cond', 'long_cond_name') tx.add_taxonomy_set('pres', 'long_pres_name') tx.add_taxonomy_set('rdt') # map is {<local name>: <granule name or path>} #Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect rdt = RecordDictionaryTool(taxonomy=tx) #Create some arrays and fill them with random values temp_array = np.random.standard_normal(100) cond_array = np.random.standard_normal(100) pres_array = np.random.standard_normal(100) #Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc. rdt['temp'] = temp_array rdt['cond'] = cond_array rdt['pres'] = pres_array #You can also add in another RecordDictionaryTool, providing the taxonomies are the same. rdt2 = RecordDictionaryTool(taxonomy=tx) rdt2['temp'] = temp_array rdt['rdt'] = rdt2 g = build_granule(data_producer_id='john', taxonomy=tx, record_dictionary=rdt) l_tx = TaxyTool.load_from_granule(g) l_rd = RecordDictionaryTool.load_from_granule(g) # Make sure we got back the same Taxonomy Object self.assertEquals(l_tx._t, tx._t) self.assertEquals(l_tx.get_handles('temp'), tx.get_handles('temp')) self.assertEquals(l_tx.get_handles('testing_2'), tx.get_handles('testing_2')) # Now test the record dictionary object self.assertEquals(l_rd._rd, rdt._rd) self.assertEquals(l_rd._tx._t, rdt._tx._t) for k, v in l_rd.iteritems(): self.assertIn(k, rdt) if isinstance(v, np.ndarray): self.assertTrue( (v == rdt[k]).all()) else: self.assertEquals(v._rd, rdt[k]._rd)
def validate_messages(self, msgs): msg = msgs rdt = RecordDictionaryTool.load_from_granule(msg.body) vardict = {} vardict['temp'] = get_safe(rdt, 'temp') vardict['time'] = get_safe(rdt, 'time') print vardict['time'] print vardict['temp']
def validate_messages(self, results): bin1 = numpy.array([]) bin2 = numpy.array([]) for message in results: rdt = RecordDictionaryTool.load_from_granule(message) if 'salinity' in message.data_producer_id: if 'double' in message.data_producer_id: bin2 = numpy.append(bin2, rdt['salinity']) else: bin1 = numpy.append(bin1, rdt['salinity']) assert_array_almost_equal(bin2, bin1 * 2.0)
def recv_packet(self, msg, stream_route, stream_id): if not isinstance(msg, Granule): log.warn('Received non-compatible granule in stream %s.', stream_id) return rdt = RecordDictionaryTool.load_from_granule(msg) if self.log: log.warn('Received record dictionary:\n%s', rdt.pretty_print()) else: import sys print >> sys.stderr, 'Received record dictionary:\n%s' % rdt.pretty_print( )
def assert_data_received(self): #let it go for up to 120 seconds, then stop the agent and reset it if not self.dataset_modified.is_set(): self.dataset_modified.wait(30) self.assertTrue(self.granule_count > 2, msg='granule count = %d' % self.granule_count) rdt = RecordDictionaryTool.load_from_granule(self.granule_capture[0]) self.assertAlmostEqual(0, rdt['oxygen'][0], delta=0.01) self.assertAlmostEqual(309.77, rdt['pressure'][0], delta=0.01) self.assertAlmostEqual(37.9848, rdt['conductivity'][0], delta=0.01) self.assertAlmostEqual(9.5163, rdt['temp'][0], delta=0.01) self.assertAlmostEqual(3527207897.0, rdt['time'][0], delta=1)
def execute(input=None, context=None, config=None, params=None, state=None): rdt = RecordDictionaryTool.load_from_granule(input) out_rdt = RecordDictionaryTool(stream_definition_id=params) pressure = rdt['pressure'] pres_value = (pressure / 100.0) + 0.5 for key, value in rdt.iteritems(): if key in out_rdt: out_rdt[key] = value[:] out_rdt['pressure'] = pres_value return out_rdt.to_granule()
def execute(input=None, context=None, config=None, params=None, state=None): """ Dependencies ------------ CONDWAT_L1, TEMPWAT_L1, PRESWAT_L1 Algorithms used --------------- PRACSAL = gsw_SP_from_C((CONDWAT_L1 * 10),TEMPWAT_L1,PRESWAT_L1) Reference --------- The calculations below are based on the following spreadsheet document: https://docs.google.com/spreadsheet/ccc?key=0Au7PUzWoCKU4dDRMeVI0RU9yY180Z0Y5U0hyMUZERmc#gid=0 """ rdt = RecordDictionaryTool.load_from_granule(input) out_rdt = RecordDictionaryTool(stream_definition_id=params) out_rdt['time'] = rdt['time'] conductivity = rdt['conductivity'] pressure = rdt['pressure'] temperature = rdt['temp'] sal_value = gsw.sp_from_c(conductivity * 10, temperature, pressure) log.debug( "CTDBP Salinity algorithm calculated the sp (practical salinity) values: %s", sal_value) for key, value in rdt.iteritems(): if key in out_rdt: if key == 'conductivity' or key == 'temp' or key == 'pressure': continue out_rdt[key] = value[:] out_rdt['salinity'] = sal_value return out_rdt.to_granule()
def validate_output_granule(self, msg, route, stream_id): self.assertTrue( stream_id in [self._output_stream_one_id, self._output_stream_two_id]) rdt = RecordDictionaryTool.load_from_granule(msg) log.debug('validate_output_granule stream_id: %s', stream_id) if stream_id == self._output_stream_one_id: sal_val = rdt['salinity'] log.debug('validate_output_granule sal_val: %s', sal_val) np.testing.assert_array_equal(sal_val, np.array([3])) self.event1_verified.set() else: cond_val = rdt['conductivity'] log.debug('validate_output_granule cond_val: %s', cond_val) np.testing.assert_array_equal(cond_val, np.array([5])) self.event2_verified.set()