def check_density_algorithm_execution(self, publish_granule, granule_from_transform):
#------------------------------------------------------------------
# Calculate the correct density from the input granule data
#------------------------------------------------------------------
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform)
conductivity = input_rdt_to_transform['conductivity']
pressure = input_rdt_to_transform['pressure']
temperature = input_rdt_to_transform['temp']
longitude = input_rdt_to_transform['lon']
latitude = input_rdt_to_transform['lat']
sp = SP_from_cndr(r=conductivity/cte.C3515, t=temperature, p=pressure)
sa = SA_from_SP(sp, pressure, longitude, latitude)
dens_value = rho(sa, temperature, pressure)
out_density = output_rdt_transform['density']
#-----------------------------------------------------------------------------
# Check that the output data from the transform has the correct density values
#-----------------------------------------------------------------------------
self.assertTrue(dens_value.all() == out_density.all())
def check_density_algorithm_execution(self, publish_granule, granule_from_transform):
#------------------------------------------------------------------
# Calculate the correct density from the input granule data
#------------------------------------------------------------------
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform)
conductivity = input_rdt_to_transform['conductivity']
pressure = input_rdt_to_transform['pressure']
temperature = input_rdt_to_transform['temp']
longitude = input_rdt_to_transform['lon']
latitude = input_rdt_to_transform['lat']
sp = SP_from_cndr(r=conductivity/cte.C3515, t=temperature, p=pressure)
sa = SA_from_SP(sp, pressure, longitude, latitude)
dens_value = rho(sa, temperature, pressure)
out_density = output_rdt_transform['density']
log.debug("density output from the transform: %s", out_density)
log.debug("values of density expected from the transform: %s", dens_value)
numpy.testing.assert_array_almost_equal(out_density, dens_value, decimal=3)
def check_density_algorithm_execution(self, publish_granule,
granule_from_transform):
#------------------------------------------------------------------
# Calculate the correct density from the input granule data
#------------------------------------------------------------------
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(
publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(
granule_from_transform)
conductivity = input_rdt_to_transform['conductivity']
pressure = input_rdt_to_transform['pressure']
temperature = input_rdt_to_transform['temp']
longitude = input_rdt_to_transform['lon']
latitude = input_rdt_to_transform['lat']
sp = SP_from_cndr(r=conductivity / cte.C3515,
t=temperature,
p=pressure)
sa = SA_from_SP(sp, pressure, longitude, latitude)
dens_value = rho(sa, temperature, pressure)
out_density = output_rdt_transform['density']
log.debug("density output from the transform: %s", out_density)
log.debug("values of density expected from the transform: %s",
dens_value)
numpy.testing.assert_array_almost_equal(out_density,
dens_value,
decimal=3)
def check_salinity_algorithm_execution(self, publish_granule,
granule_from_transform):
#------------------------------------------------------------------
# Calculate the correct density from the input granule data
#------------------------------------------------------------------
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(
publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(
granule_from_transform)
conductivity = input_rdt_to_transform['conductivity']
pressure = input_rdt_to_transform['pressure']
temperature = input_rdt_to_transform['temp']
sal_value = SP_from_cndr(r=conductivity / cte.C3515,
t=temperature,
p=pressure)
out_salinity = output_rdt_transform['salinity']
#-----------------------------------------------------------------------------
# Check that the output data from the transform has the correct density values
#-----------------------------------------------------------------------------
self.assertTrue(numpy.array_equal(sal_value, out_salinity))
def validate_multiple_vis_queue_messages(self, msg1, msg2):
assertions = self.assertTrue
# verify that the salinity in msg2 is a result of content from msg1
rdt1 = RecordDictionaryTool.load_from_granule(msg1)
rdt2 = RecordDictionaryTool.load_from_granule(msg2)
# msg1 should not have salinity
# assertions(rdt1['salinity'] == None)
conductivity = rdt1['conductivity']
pressure = rdt1['pressure']
temperature = rdt1['temp']
msg1_sal_value = SP_from_cndr(r=conductivity / cte.C3515,
t=temperature,
p=pressure)
msg2_sal_value = rdt2['salinity']
b = msg1_sal_value == msg2_sal_value
if isinstance(b, bool):
assertions(b)
else:
assertions(b.all())
return
def validate_multiple_vis_queue_messages(self, msg1, msg2):
assertions = self.assertTrue
# verify that the salinity in msg2 is a result of content from msg1
rdt1 = RecordDictionaryTool.load_from_granule(msg1)
rdt2 = RecordDictionaryTool.load_from_granule(msg2)
# msg1 should not have salinity
# assertions(rdt1['salinity'] == None)
conductivity = rdt1['conductivity']
pressure = rdt1['pressure']
temperature = rdt1['temp']
msg1_sal_value = SP_from_cndr(r=conductivity/cte.C3515, t=temperature, p=pressure)
msg2_sal_value = rdt2['salinity']
b = msg1_sal_value == msg2_sal_value
if isinstance(b,bool):
assertions(b)
else:
assertions(b.all())
return
def check_pres_algorithm_execution(self, publish_granule, granule_from_transform):
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform)
output_data = output_rdt_transform['pressure']
input_data = input_rdt_to_transform['pressure']
self.assertTrue(input_data.all() == output_data.all())
def check_cond_algorithm_execution(self, publish_granule, granule_from_transform):
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform)
output_data = output_rdt_transform["conductivity"]
input_data = input_rdt_to_transform["conductivity"]
self.assertTrue(((input_data / 100000.0) - 0.5).all() == output_data.all())
def check_temp_algorithm_execution(self, publish_granule, granule_from_transform):
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform)
output_data = output_rdt_transform['temp']
input_data = input_rdt_to_transform['temp']
self.assertTrue(((input_data / 10000.0) - 10).all() == output_data.all())
def check_pres_algorithm_execution(self, publish_granule, granule_from_transform):
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform)
output_data = output_rdt_transform['pressure']
input_data = input_rdt_to_transform['pressure']
self.assertTrue(numpy.array_equal((input_data/ 100.0) + 0.5,output_data))
def test_stream_ingestion_worker(self):
self.start_ingestion_worker()
context_ids, time_ctxt = self._create_param_contexts()
pdict_id = self.dataset_management_client.create_parameter_dictionary(
name='stream_ingestion_pdict',
parameter_context_ids=context_ids,
temporal_context='ingestion_timestamp')
self.addCleanup(
self.dataset_management_client.delete_parameter_dictionary,
pdict_id)
dataset_id = self.dataset_management_client.create_dataset(
name='fake_dataset',
description='fake_dataset',
stream_id=self.stream_id,
spatial_domain=self.spatial_dom.dump(),
temporal_domain=self.time_dom.dump(),
parameter_dictionary_id=pdict_id)
self.addCleanup(self.dataset_management_client.delete_dataset,
dataset_id)
self.cov = self._create_coverage(dataset_id=dataset_id,
parameter_dict_id=pdict_id,
time_dom=self.time_dom,
spatial_dom=self.spatial_dom)
self.addCleanup(self.cov.close)
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['conductivity'] = 1
rdt['pressure'] = 2
rdt['salinity'] = 3
self.start_listener(dataset_id)
self.publisher.publish(rdt.to_granule())
self.data_modified = Event()
self.data_modified.wait(30)
cov = self.get_coverage(dataset_id)
self.assertIsNotNone(cov.get_parameter_values('raw'))
deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
granule = retrieve_stream(dataset_id)
rdt_complex = RecordDictionaryTool.load_from_granule(granule)
rdt_complex['raw'] = [
deserializer.deserialize(i) for i in rdt_complex['raw']
]
for gran in rdt_complex['raw']:
rdt_new = RecordDictionaryTool.load_from_granule(gran)
self.assertIn(1, rdt_new['conductivity'])
self.assertIn(2, rdt_new['pressure'])
self.assertIn(3, rdt_new['salinity'])
cov.close()
def recv_packet(self, msg, stream_route, stream_id):
''' receive packet for ingestion '''
log.debug('received granule for stream %s', stream_id)
if msg == {}:
log.error('Received empty message from stream: %s', stream_id)
return
# Message validation
if not isinstance(msg, Granule):
log.error('Ingestion received a message that is not a granule: %s', msg)
return
rdt = RecordDictionaryTool.load_from_granule(msg)
if rdt is None:
log.error('Invalid granule (no RDT) for stream %s', stream_id)
return
if not len(rdt):
log.debug('Empty granule for stream %s', stream_id)
return
dp_id_list = self.retrieve_dataprocess_for_stream(stream_id)
for dp_id in dp_id_list:
function, argument_list = self.retrieve_function_and_define_args(dp_id)
args = []
rdt = RecordDictionaryTool.load_from_granule(msg)
#create the input arguments list
#todo: this logic is tied to the example funcation, generalize
for func_param, record_param in argument_list.iteritems():
args.append(rdt[record_param])
try:
#run the calc
#todo: nothing in the data process resource to specify multi-out map
result = function(*args)
out_stream_definition, output_parameter = self.retrieve_dp_output_params(dp_id)
rdt = RecordDictionaryTool(stream_definition_id=out_stream_definition)
publisher = self._publisher_map.get(dp_id,'')
rdt[ output_parameter ] = result
if publisher:
publisher.publish(rdt.to_granule())
else:
log.error('Publisher not found for data process %s', dp_id)
self.update_dp_metrics( dp_id )
except ImportError:
log.error('Error running transform')
def check_temp_algorithm_execution(self, publish_granule,
granule_from_transform):
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(
publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(
granule_from_transform)
output_data = output_rdt_transform['temp']
input_data = input_rdt_to_transform['temp']
self.assertTrue(
numpy.array_equal(((input_data / 10000.0) - 10), output_data))
def check_cond_algorithm_execution(self, publish_granule,
granule_from_transform):
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(
publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(
granule_from_transform)
output_data = output_rdt_transform['conductivity']
input_data = input_rdt_to_transform['conductivity']
self.assertTrue(((input_data / 100000.0) -
0.5).all() == output_data.all())
def execute(input=None, context=None, config=None, params=None, state=None):
rdt = RecordDictionaryTool.load_from_granule(input)
out_rdt = RecordDictionaryTool(stream_definition_id=params)
conductivity = rdt['conductivity']
pressure = rdt['pressure']
temperature = rdt['temp']
longitude = rdt['lon'] if rdt['lon'] is not None else 0
latitude = rdt['lat'] if rdt['lat'] is not None else 0
sp = SP_from_cndr(r=conductivity/cte.C3515, t=temperature, p=pressure)
log.debug("Density algorithm calculated the sp (practical salinity) values: %s", sp)
sa = SA_from_SP(sp, pressure, longitude, latitude)
log.debug("Density algorithm calculated the sa (actual salinity) values: %s", sa)
dens_value = rho(sa, temperature, pressure)
for key, value in rdt.iteritems():
if key in out_rdt:
if key=='conductivity' or key=='temp' or key=='pressure':
continue
out_rdt[key] = value[:]
out_rdt['density'] = dens_value
log.debug("Density algorithm returning density values: %s", out_rdt['density'])
return out_rdt.to_granule()
def validate_mpl_graphs_transform_results(self, results):
cc = self.container
assertions = self.assertTrue
# if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases
if isinstance(results,Granule):
results =[results]
found_data = False
for g in results:
if isinstance(g,Granule):
rdt = RecordDictionaryTool.load_from_granule(g)
graphs = get_safe(rdt, 'matplotlib_graphs')
if graphs == None:
continue
for graph in graphs[0]:
# At this point only dictionaries containing image data should be passed
# For some reason non dictionary values are filtering through.
if not isinstance(graph, dict):
continue
assertions(graph['viz_product_type'] == 'matplotlib_graphs' )
# check to see if the list (numpy array) contains actual images
assertions(imghdr.what(graph['image_name'], h = graph['image_obj']) == 'png')
found_data = True
return found_data
def _execute_transform(self, msg, streams):
stream_in_id,stream_out_id = streams
stream_def_in = self.read_stream_def(stream_in_id)
stream_def_out = self.read_stream_def(stream_out_id)
rdt_temp = self._merge_rdt(stream_def_in, stream_def_out)
rdt_in = RecordDictionaryTool.load_from_granule(msg)
for field in rdt_temp.fields:
if not isinstance(rdt_temp._pdict.get_context(field).param_type, ParameterFunctionType):
try:
rdt_temp[field] = rdt_in[field]
except KeyError:
pass
rdt_temp.fetch_lookup_values()
for lookup_field in rdt_temp.lookup_values():
s = lookup_field
stored_value = self._get_lookup_value(rdt_temp.context(s).lookup_value)
if stored_value is not None:
rdt_temp[s] = stored_value
for field in rdt_temp.fields:
if isinstance(rdt_temp._pdict.get_context(field).param_type, ParameterFunctionType):
rdt_temp[field] = rdt_temp[field]
rdt_out = RecordDictionaryTool(stream_definition_id=stream_def_out._id)
for field in rdt_out.fields:
rdt_out[field] = rdt_temp[field]
return rdt_out
def recv_packet(self, msg, stream_route, stream_id):
'''
The consumer callback to parse and manage the granule.
The message is ACK'd once the function returns
'''
log.trace('received granule for stream %s', stream_id)
if msg == {}:
log.error('Received empty message from stream: %s', stream_id)
return
# Message validation
if not isinstance(msg, Granule):
log.error('Ingestion received a message that is not a granule: %s', msg)
return
rdt = RecordDictionaryTool.load_from_granule(msg)
if rdt is None:
log.error('Invalid granule (no RDT) for stream %s', stream_id)
return
if not len(rdt):
log.debug('Empty granule for stream %s', stream_id)
return
self.persist_or_timeout(stream_id, rdt)
def validate_highcharts_transform_results(self, results):
assertions = self.assertTrue
# if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases
if isinstance(results,Granule):
results =[results]
for g in results:
if isinstance(g,Granule):
rdt = RecordDictionaryTool.load_from_granule(g)
hc_data_arr = get_safe(rdt, 'hc_data')
if hc_data_arr == None:
log.debug("hc_data in granule is None")
continue
assertions(len(hc_data_arr) >= 0) # Need to come up with a better check
hc_data = hc_data_arr[0]
assertions(len(hc_data) >= 0)
assertions(len(hc_data[0]["name"]) >= 0)
assertions(len(hc_data[0]["data"]) >= 0)
def execute(input=None,
context=None,
config=None,
params=None,
state=None):
"""
@param input granule
@retval result_list list of dictionaries containing granules as values
"""
result_list = []
for x in input:
rdt = RecordDictionaryTool.load_from_granule(x)
conductivity = rdt['conductivity']
pressure = rdt['pressure']
temperature = rdt['temperature']
time = rdt['time']
# build the granule for conductivity, temperature and pressure
granule = ctdbp_L0_algorithm._build_granule(
stream_definition_id=params['L0_stream'],
field_names=[
'conductivity', 'pressure', 'temperature', 'time'
], # these are the field names for the output record dictionary
values=[conductivity, pressure, temperature, time])
result_list.append(granule)
return result_list
def validate_output_granule(self, msg, route, stream_id):
self.assertIn(stream_id, self._output_stream_ids)
rdt = RecordDictionaryTool.load_from_granule(msg)
log.debug('validate_output_granule rdt: %s', rdt)
sal_val = rdt['salinity']
np.testing.assert_array_equal(sal_val, np.array([3]))
def validate_google_dt_transform_results(self, results):
cc = self.container
assertions = self.assertTrue
# if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases
if isinstance(results,Granule):
results =[results]
for g in results:
if isinstance(g,Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
gdt_data = get_safe(rdt, 'google_dt_components')
# IF this granule does not contains google dt, skip
if gdt_data == None:
continue
gdt = gdt_data[0]
assertions(gdt['viz_product_type'] == 'google_dt' )
assertions(len(gdt['data_description']) >= 0) # Need to come up with a better check
assertions(len(gdt['data_content']) >= 0)
def execute(input=None, context=None, config=None, params=None, state=None):
rdt = RecordDictionaryTool.load_from_granule(input)
out_rdt = RecordDictionaryTool(stream_definition_id=params)
conductivity = rdt["conductivity"]
pressure = rdt["pressure"]
temperature = rdt["temp"]
longitude = rdt["lon"] if rdt["lon"] is not None else 0
latitude = rdt["lat"] if rdt["lat"] is not None else 0
sp = SP_from_cndr(r=conductivity / cte.C3515, t=temperature, p=pressure)
log.debug("Density algorithm calculated the sp (practical salinity) values: %s", sp)
sa = SA_from_SP(sp, pressure, longitude, latitude)
log.debug("Density algorithm calculated the sa (actual salinity) values: %s", sa)
dens_value = rho(sa, temperature, pressure)
for key, value in rdt.iteritems():
if key in out_rdt:
if key == "conductivity" or key == "temp" or key == "pressure":
continue
out_rdt[key] = value[:]
out_rdt["density"] = dens_value
log.debug("Density algorithm returning density values: %s", out_rdt["density"])
return out_rdt.to_granule()
def validate_output_granule(self, msg, route, stream_id):
self.assertIn( stream_id, self._output_stream_ids)
rdt = RecordDictionaryTool.load_from_granule(msg)
log.debug('validate_output_granule rdt: %s', rdt)
sal_val = rdt['salinity']
np.testing.assert_array_equal(sal_val, np.array([3]))
def execute(input=None,
context=None,
config=None,
params=None,
state=None):
rdt = RecordDictionaryTool.load_from_granule(input)
out_rdt = RecordDictionaryTool(stream_definition_id=params)
conductivity = rdt['conductivity']
pressure = rdt['pressure']
temperature = rdt['temp']
sal_value = SP_from_cndr(r=conductivity / cte.C3515,
t=temperature,
p=pressure)
log.debug(
"Salinity algorithm calculated the sp (practical salinity) values: %s",
sal_value)
for key, value in rdt.iteritems():
if key in out_rdt:
if key == 'conductivity' or key == 'temp' or key == 'pressure':
continue
out_rdt[key] = value[:]
out_rdt['salinity'] = sal_value
return out_rdt.to_granule()
def validate_mpl_graphs_transform_results(self, results):
cc = self.container
assertions = self.assertTrue
# if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases
if isinstance(results, Granule):
results = [results]
found_data = False
for g in results:
if isinstance(g, Granule):
rdt = RecordDictionaryTool.load_from_granule(g)
graphs = get_safe(rdt, 'matplotlib_graphs')
if graphs == None:
continue
for graph in graphs[0]:
# At this point only dictionaries containing image data should be passed
# For some reason non dictionary values are filtering through.
if not isinstance(graph, dict):
continue
assertions(
graph['viz_product_type'] == 'matplotlib_graphs')
# check to see if the list (numpy array) contains actual images
assertions(
imghdr.what(graph['image_name'], h=graph['image_obj'])
== 'png')
found_data = True
return found_data
def execute(input=None, context=None, config=None, params=None, state=None):
"""
Find if the input data has values, which are out of range
@param input granule
@param context parameter context
@param config DotDict
@param params list
@param state
@return bad_values, bad_value_times tuple of lists
"""
rdt = RecordDictionaryTool.load_from_granule(input)
# Retrieve the name used for the variable_name, the name used for timestamps and the range of valid values from the config
valid_values = config.get_safe('valid_values', [-100,100])
variable_name = config.get_safe('variable_name', 'input_voltage')
time_field_name = config.get_safe('time_field_name', 'preferred_timestamp')
# These variable_names will store the bad values and the timestamps of those values
bad_values = []
bad_value_times = []
# retrieve the values and the times from the record dictionary
values = rdt[variable_name][:]
times = rdt[time_field_name][:]
for val, t in zip(values, times):
if val < valid_values[0] or val > valid_values[1]:
bad_values.append(val)
bad_value_times.append(t)
# return the list of bad values and their timestamps
return bad_values, bad_value_times
def recv_packet(self, msg, stream_route, stream_id):
'''
The consumer callback to parse and manage the granule.
The message is ACK'd once the function returns
'''
log.trace('received granule for stream %s', stream_id)
if msg == {}:
log.error('Received empty message from stream: %s', stream_id)
return
# Message validation
if not isinstance(msg, Granule):
log.error('Ingestion received a message that is not a granule: %s',
msg)
return
rdt = RecordDictionaryTool.load_from_granule(msg)
if rdt is None:
log.error('Invalid granule (no RDT) for stream %s', stream_id)
return
if not len(rdt):
log.debug('Empty granule for stream %s', stream_id)
return
self.persist_or_timeout(stream_id, rdt)
def add_granule(self,stream_id, granule):
'''
Appends the granule's data to the coverage and persists it.
'''
#--------------------------------------------------------------------------------
# Coverage determiniation and appending
#--------------------------------------------------------------------------------
dataset_id = self.get_dataset(stream_id)
if not dataset_id:
log.error('No dataset could be determined on this stream: %s', stream_id)
return
coverage = self.get_coverage(stream_id)
if not coverage:
log.error('Could not persist coverage from granule, coverage is None')
return
#--------------------------------------------------------------------------------
# Actual persistence
#--------------------------------------------------------------------------------
rdt = RecordDictionaryTool.load_from_granule(granule)
elements = len(rdt)
if not elements:
return
coverage.insert_timesteps(elements)
start_index = coverage.num_timesteps - elements
for k,v in rdt.iteritems():
if k == 'image_obj':
log.trace( '%s:', k)
else:
log.trace( '%s: %s', k, v)
slice_ = slice(start_index, None)
coverage.set_parameter_values(param_name=k, tdoa=slice_, value=v)
coverage.flush()
def execute(input=None, context=None, config=None, params=None, state=None):
rdt = RecordDictionaryTool.load_from_granule(input)
out_rdt = RecordDictionaryTool(stream_definition_id=params['stream_def_id'])
# Fill the time values
out_rdt['time'] = rdt['time']
# The calibration coefficients
temp_calibration_coeffs= params['calibration_coeffs']['temp_calibration_coeffs']
pres_calibration_coeffs= params['calibration_coeffs']['pres_calibration_coeffs']
cond_calibration_coeffs = params['calibration_coeffs']['cond_calibration_coeffs']
log.debug("params['calibration_coeffs']: %s", params['calibration_coeffs'])
# Set the temperature values for the output granule
out_rdt = CTDBP_L1_TransformAlgorithm.calculate_temperature( input_rdt = rdt,
out_rdt = out_rdt,
temp_calibration_coeffs= temp_calibration_coeffs )
# Set the pressure values for the output granule
out_rdt = CTDBP_L1_TransformAlgorithm.calculate_pressure( input_rdt= rdt,
out_rdt = out_rdt,
pres_calibration_coeffs= pres_calibration_coeffs)
# Set the conductivity values for the output granule
# Note that since the conductivity caculation depends on whether TEMPWAT_L1, PRESWAT_L1 have been calculated, we need to do this last
out_rdt = CTDBP_L1_TransformAlgorithm.calculate_conductivity( input_rdt = rdt,
out_rdt = out_rdt,
cond_calibration_coeffs = cond_calibration_coeffs
)
# build the granule for the L1 stream
return out_rdt.to_granule()
def execute(input=None,
context=None,
config=None,
params=None,
state=None):
'''
@param input Granule
@retval result Granule
'''
rdt = RecordDictionaryTool.load_from_granule(input)
out_rdt = RecordDictionaryTool(stream_definition_id=params)
conductivity = rdt['conductivity']
cond_value = (conductivity / 100000.0) - 0.5
for key, value in rdt.iteritems():
if key in out_rdt:
out_rdt[key] = value[:]
# Update the conductivity values
out_rdt['conductivity'] = cond_value
# build the granule for conductivity
return out_rdt.to_granule()
def recv_packet(self, packet, stream_route, stream_id):
if packet == {}:
return
l0_values = RecordDictionaryTool.load_from_granule(packet)
l1_values = RecordDictionaryTool(
stream_definition_id=self.stream_definition_id)
log.debug(
"CTDBP L1 transform using L0 values: tempurature %s, pressure %s, conductivity %s",
l0_values['temperature'], l0_values['pressure'],
l0_values['conductivity'])
#for key, value in 'lat', 'lon', 'time', ...: <-- do we want to be a little more specific here?
for key, value in l0_values.iteritems():
if key in l1_values:
l1_values[key] = value[:]
l1_values['temp'] = self.calculate_temperature(l0=l0_values)
l1_values['pressure'] = self.calculate_pressure(l0=l0_values)
l1_values['conductivity'] = self.calculate_conductivity(l0=l0_values,
l1=l1_values)
log.debug(
'calculated L1 values: temp %s, pressure %s, conductivity %s',
l1_values['temp'], l1_values['pressure'],
l1_values['conductivity'])
self.publisher.publish(msg=l1_values.to_granule())
def execute(self, granule):
"""Processes incoming data!!!!
"""
rdt = RecordDictionaryTool.load_from_granule(granule)
#todo: use only flat dicts for now, may change later...
# rdt0 = rdt['coordinates']
# rdt1 = rdt['data']
pressure = get_safe(rdt, 'pres') #psd.get_values('conductivity')
longitude = get_safe(rdt, 'lon') # psd.get_values('longitude')
latitude = get_safe(rdt, 'lat') #psd.get_values('latitude')
time = get_safe(rdt, 'time') # psd.get_values('time')
height = get_safe(rdt, 'height') # psd.get_values('time')
log.warn('Got pressure: %s' % str(pressure))
# L1
# 1) The algorithm input is the L0 pressure data product (p_hex) and, in the case of the SBE 37IM, the pressure range (P_rng) from metadata.
# 2) Convert the hexadecimal string to a decimal string
# 3) For the SBE 37IM only, convert the pressure range (P_rng) from psia to dbar SBE 37IM
# Convert P_rng (input from metadata) from psia to dbar
# 4) Perform scaling operation
# SBE 37IM
# L1 pressure data product (in dbar):
# Use the constructor to put data into a granule
psc = PointSupplementConstructor(point_definition=self.outgoing_stream_def, stream_id=self.streams['output'])
### Assumes the config argument for output streams is known and there is only one 'output'.
### the stream id is part of the metadata which much go in each stream granule - this is awkward to do at the
### application level like this!
scaled_pressure = pressure
for i in xrange(len(pressure)):
#todo: get pressure range from metadata (if present) and include in calc
scaled_pressure[i] = ( pressure[i])
root_rdt = RecordDictionaryTool(taxonomy=self.tx)
#todo: use only flat dicts for now, may change later...
# data_rdt = RecordDictionaryTool(taxonomy=self.tx)
# coord_rdt = RecordDictionaryTool(taxonomy=self.tx)
root_rdt['pres'] = scaled_pressure
root_rdt['time'] = time
root_rdt['lat'] = latitude
root_rdt['lon'] = longitude
root_rdt['height'] = height
# root_rdt['coordinates'] = coord_rdt
# root_rdt['data'] = data_rdt
return build_granule(data_producer_id='ctd_L1_pressure', taxonomy=self.tx, record_dictionary=root_rdt)
return psc.close_stream_granule()
def execute(input=None, context=None, config=None, params=None, state=None, fileName = None):
stream_definition_id = params
mpl_allowed_numerical_types = ['int32', 'int64', 'uint32', 'uint64', 'float32', 'float64']
if stream_definition_id == None:
log.error("Matplotlib transform: Need a output stream definition to process graphs")
return None
# parse the incoming data
rdt = RecordDictionaryTool.load_from_granule(input)
# build a list of fields/variables that need to be plotted. Use the list provided by the UI
# since the retrieved granule might have extra fields.
fields = rdt.fields
resolution = "640x480"
if config:
if 'parameters' in config:
fields = config['parameters']
if 'resolution' in config:
resolution = config['resolution']
vardict = {}
vardict['time'] = get_safe(rdt, 'time')
if vardict['time'] == None:
print "Matplotlib transform: Did not receive a time field to work with"
log.error("Matplotlib transform: Did not receive a time field to work with")
return None
for field in fields:
if field == 'time':
continue
# only consider fields which are supposed to be numbers.
if (rdt[field] != None) and (rdt[field].dtype not in mpl_allowed_numerical_types):
continue
vardict[field] = get_safe(rdt, field)
print
arrLen = len(vardict['time'])
# init the graph_data structure for storing values
graph_data = {}
for varname in vardict.keys():
graph_data[varname] = []
# If code reached here, the graph data storage has been initialized. Just add values
# to the list
for varname in vardict.keys(): # psd.list_field_names():
if vardict[varname] == None:
# create an array of zeros to compensate for missing values
graph_data[varname].extend([0.0]*arrLen)
else:
graph_data[varname].extend(vardict[varname])
out_granule = VizTransformMatplotlibGraphsAlgorithm.render_graphs(graph_data, stream_definition_id, fileName, resolution=resolution)
return out_granule
def _check_application_of_L1_algorithm(self, granule = None):
""" Check the algorithm applied by the L1 transform """
rdt = RecordDictionaryTool.load_from_granule(granule)
list_of_expected_keys = [ 'time', 'pressure', 'conductivity', 'temp']
for key in list_of_expected_keys:
self.assertIn(key, rdt)
def validate_data_ingest_retrieve(self, dataset_id):
assertions = self.assertTrue
#validate that data was ingested
replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id, 10)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
salinity = get_safe(rdt, 'salinity')
assertions(salinity != None)
#retrieve all the granules from the database and check the values
replay_granule_all = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(replay_granule_all)
for k, v in rdt.iteritems():
if k == 'salinity':
for val in numpy.nditer(v):
assertions(val > 0)
def _check_application_of_L2_salinity_algorithm(self, granule = None):
""" Check the algorithm applied by the L2 transform """
rdt = RecordDictionaryTool.load_from_granule(granule)
list_of_expected_keys = ['time', 'salinity']
for key in list_of_expected_keys:
self.assertIn(key, rdt)
def validate_data_ingest_retrieve(self, dataset_id):
assertions = self.assertTrue
#validate that data was ingested
replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id, 10)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
salinity = get_safe(rdt, 'salinity')
assertions(salinity != None)
#retrieve all the granules from the database and check the values
replay_granule_all = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(replay_granule_all)
for k, v in rdt.iteritems():
if k == 'salinity':
for val in numpy.nditer(v):
assertions(val > 0)
def _check_application_of_L2_salinity_algorithm(self, granule = None):
""" Check the algorithm applied by the L2 transform """
rdt = RecordDictionaryTool.load_from_granule(granule)
for key, value in rdt.iteritems():
list_of_expected_keys = ['time', 'salinity']
if key not in list_of_expected_keys:
self.fail("The L2 salinity transform is sending out data for more parameters than it should")
def execute(input=None, context=None, config=None, params=None, state=None):
"""
Find if the input data has values, which are out of range
@param input granule
@param context parameter context
@param config DotDict
@param params list
@param state
@return bad_values, bad_value_times tuple of lists
"""
# Retrieve the name used for the variable_name, the name used for timestamps and the range of valid values from the config
valid_values = config.get_safe('valid_values', [-100,100])
variable_name = config.get_safe('variable_name', 'input_voltage')
preferred_time = config.get_safe('time_field_name', 'preferred_timestamp')
# Get the source of the granules which will be used to set the origin of the DeviceStatusEvent and DeviceCommsEvent events
origin = input.data_producer_id
if not origin:
raise NotFound("The DemoStreamAlertTransform could not figure out the origin for DeviceStatusEvent. The data_producer_id attribute should be filled for the granules that are sent to it so that it can figure out the origin to use.")
log.debug("The origin the demo transform is listening to is: %s" % origin)
rdt = RecordDictionaryTool.load_from_granule(input)
# These variable_names will store the bad values and the timestamps of those values
bad_values = []
bad_value_times = []
# retrieve the values and the times from the record dictionary
values = rdt[variable_name][:]
time_names = rdt[preferred_time][:]
log.debug("Values unravelled: %s" % values)
log.debug("Time names unravelled: %s" % time_names)
log.debug("Transform got the rdt here: %s", rdt)
indexes = [l for l in xrange(len(time_names))]
for val, index in zip(values, indexes):
if val < valid_values[0] or val > valid_values[1]:
bad_values.append(val)
arr = rdt[time_names[index]]
log.debug("In the stream alert transform, got the arr here: %s", arr)
if isinstance(arr, numpy.ndarray):
bad_value_times.append(arr[index])
else:
# Since we do not want the transform to crash because a granule came in that did not have the values of the preferred time filled as expected
bad_value_times.append(None)
log.debug("Returning bad_values: %s, bad_value_times: %s and the origin: %s" % (bad_values, bad_value_times, origin))
# return the list of bad values and their timestamps
return bad_values, bad_value_times, origin
def execute(input=None, context=None, config=None, params=None, state=None):
"""
Find if the input data has values, which are out of range
@param input granule
@param context parameter context
@param config DotDict
@param params list
@param state
@return bad_values, bad_value_times tuple of lists
"""
# Retrieve the name used for the variable_name, the name used for timestamps and the range of valid values from the config
valid_values = config.get_safe('valid_values', [-100,100])
variable_name = config.get_safe('variable_name', 'input_voltage')
preferred_time = config.get_safe('time_field_name', 'preferred_timestamp')
# Get the source of the granules which will be used to set the origin of the DeviceStatusEvent and DeviceCommsEvent events
origin = input.data_producer_id
if not origin:
raise NotFound("The DemoStreamAlertTransform could not figure out the origin for DeviceStatusEvent. The data_producer_id attribute should be filled for the granules that are sent to it so that it can figure out the origin to use.")
log.debug("The origin the demo transform is listening to is: %s" % origin)
rdt = RecordDictionaryTool.load_from_granule(input)
# These variable_names will store the bad values and the timestamps of those values
bad_values = []
bad_value_times = []
# retrieve the values and the times from the record dictionary
values = rdt[variable_name][:]
time_names = rdt[preferred_time][:]
log.debug("Values unravelled: %s" % values)
log.debug("Time names unravelled: %s" % time_names)
log.debug("Transform got the rdt here: %s", rdt)
indexes = [l for l in xrange(len(time_names))]
for val, index in zip(values, indexes):
if val < valid_values[0] or val > valid_values[1]:
bad_values.append(val)
arr = rdt[time_names[index]]
log.debug("In the stream alert transform, got the arr here: %s", arr)
if isinstance(arr, numpy.ndarray):
bad_value_times.append(arr[index])
else:
# Since we do not want the transform to crash because a granule came in that did not have the values of the preferred time filled as expected
bad_value_times.append(None)
log.debug("Returning bad_values: %s, bad_value_times: %s and the origin: %s" % (bad_values, bad_value_times, origin))
# return the list of bad values and their timestamps
return bad_values, bad_value_times, origin
def _check_application_of_L2_salinity_algorithm(self, granule=None):
""" Check the algorithm applied by the L2 transform """
rdt = RecordDictionaryTool.load_from_granule(granule)
for key, value in rdt.iteritems():
list_of_expected_keys = ['time', 'salinity']
if key not in list_of_expected_keys:
self.fail(
"The L2 salinity transform is sending out data for more parameters than it should"
)
def _check_granule_from_transform(self, granule):
"""
An internal method to check if a granule has the right properties
"""
rdt = RecordDictionaryTool.load_from_granule(granule)
self.assertIn("pressure", rdt)
self.assertIn("temp", rdt)
self.assertIn("conductivity", rdt)
self.assertIn("time", rdt)
def _check_granule_from_transform(self, granule):
"""
An internal method to check if a granule has the right properties
"""
rdt = RecordDictionaryTool.load_from_granule(granule)
self.assertIn('pressure', rdt)
self.assertIn('temp', rdt)
self.assertIn('conductivity', rdt)
self.assertIn('time', rdt)
def validate_messages(self, msgs):
msg = msgs
rdt = RecordDictionaryTool.load_from_granule(msg.body)
vardict = {}
vardict['temp'] = get_safe(rdt, 'temp')
vardict['time'] = get_safe(rdt, 'time')
print vardict['time']
print vardict['temp']
def recv_packet(self, msg, stream_route, stream_id):
from ion.services.dm.utility.granule.record_dictionary import RecordDictionaryTool
rdt = RecordDictionaryTool.load_from_granule(msg)
#print 'Message Received: {0}'.format(rdt.pretty_print())
flds = ''
for field in rdt.fields:
flds += field + ' '
print '\n' + flds
for i in xrange(len(rdt)):
var_tuple = [ float(rdt[field][i]) if rdt[field] is not None else 0.0 for field in rdt.fields]
print var_tuple
def check_salinity_algorithm_execution(self, publish_granule, granule_from_transform):
#------------------------------------------------------------------
# Calculate the correct density from the input granule data
#------------------------------------------------------------------
input_rdt_to_transform = RecordDictionaryTool.load_from_granule(publish_granule)
output_rdt_transform = RecordDictionaryTool.load_from_granule(granule_from_transform)
conductivity = input_rdt_to_transform['conductivity']
pressure = input_rdt_to_transform['pressure']
temperature = input_rdt_to_transform['temp']
sal_value = SP_from_cndr(r=conductivity/cte.C3515, t=temperature, p=pressure)
out_salinity = output_rdt_transform['salinity']
#-----------------------------------------------------------------------------
# Check that the output data from the transform has the correct density values
#-----------------------------------------------------------------------------
self.assertTrue(sal_value.all() == out_salinity.all())
def recv_packet(self, msg, stream_route, stream_id):
if not isinstance(msg,Granule):
log.warn('Received non-compatible granule in stream %s.', stream_id)
return
rdt = RecordDictionaryTool.load_from_granule(msg)
if self.log:
log.warn('Received record dictionary:\n%s', rdt.pretty_print())
else:
import sys
print>>sys.stderr, 'Received record dictionary:\n%s' % rdt.pretty_print()
def test_build_granule_and_load_from_granule_with_taxonomy(self):
#Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle
tx = TaxyTool()
tx.add_taxonomy_set('temp', 'long_temp_name')
tx.add_taxonomy_set('cond', 'long_cond_name')
tx.add_taxonomy_set('pres', 'long_pres_name')
tx.add_taxonomy_set('rdt')
# map is {<local name>: <granule name or path>}
#Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect
rdt = RecordDictionaryTool(taxonomy=tx)
#Create some arrays and fill them with random values
temp_array = np.random.standard_normal(100)
cond_array = np.random.standard_normal(100)
pres_array = np.random.standard_normal(100)
#Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc.
rdt['temp'] = temp_array
rdt['cond'] = cond_array
rdt['pres'] = pres_array
#You can also add in another RecordDictionaryTool, providing the taxonomies are the same.
rdt2 = RecordDictionaryTool(taxonomy=tx)
rdt2['temp'] = temp_array
rdt['rdt'] = rdt2
g = build_granule(data_producer_id='john', taxonomy=tx, record_dictionary=rdt)
l_tx = TaxyTool.load_from_granule(g)
l_rd = RecordDictionaryTool.load_from_granule(g)
# Make sure we got back the same Taxonomy Object
self.assertEquals(l_tx._t, tx._t)
self.assertEquals(l_tx.get_handles('temp'), tx.get_handles('temp'))
self.assertEquals(l_tx.get_handles('testing_2'), tx.get_handles('testing_2'))
# Now test the record dictionary object
self.assertEquals(l_rd._rd, rdt._rd)
self.assertEquals(l_rd._tx._t, rdt._tx._t)
for k, v in l_rd.iteritems():
self.assertIn(k, rdt)
if isinstance(v, np.ndarray):
self.assertTrue( (v == rdt[k]).all())
else:
self.assertEquals(v._rd, rdt[k]._rd)
def validate_messages(self, msgs):
msg = msgs
rdt = RecordDictionaryTool.load_from_granule(msg.body)
vardict = {}
vardict['temp'] = get_safe(rdt, 'temp')
vardict['time'] = get_safe(rdt, 'time')
print vardict['time']
print vardict['temp']
def validate_messages(self, results):
bin1 = numpy.array([])
bin2 = numpy.array([])
for message in results:
rdt = RecordDictionaryTool.load_from_granule(message)
if 'salinity' in message.data_producer_id:
if 'double' in message.data_producer_id:
bin2 = numpy.append(bin2, rdt['salinity'])
else:
bin1 = numpy.append(bin1, rdt['salinity'])
assert_array_almost_equal(bin2, bin1 * 2.0)
def recv_packet(self, msg, stream_route, stream_id):
if not isinstance(msg, Granule):
log.warn('Received non-compatible granule in stream %s.',
stream_id)
return
rdt = RecordDictionaryTool.load_from_granule(msg)
if self.log:
log.warn('Received record dictionary:\n%s', rdt.pretty_print())
else:
import sys
print >> sys.stderr, 'Received record dictionary:\n%s' % rdt.pretty_print(
)
def assert_data_received(self):
#let it go for up to 120 seconds, then stop the agent and reset it
if not self.dataset_modified.is_set():
self.dataset_modified.wait(30)
self.assertTrue(self.granule_count > 2,
msg='granule count = %d' % self.granule_count)
rdt = RecordDictionaryTool.load_from_granule(self.granule_capture[0])
self.assertAlmostEqual(0, rdt['oxygen'][0], delta=0.01)
self.assertAlmostEqual(309.77, rdt['pressure'][0], delta=0.01)
self.assertAlmostEqual(37.9848, rdt['conductivity'][0], delta=0.01)
self.assertAlmostEqual(9.5163, rdt['temp'][0], delta=0.01)
self.assertAlmostEqual(3527207897.0, rdt['time'][0], delta=1)
def execute(input=None, context=None, config=None, params=None, state=None):
rdt = RecordDictionaryTool.load_from_granule(input)
out_rdt = RecordDictionaryTool(stream_definition_id=params)
pressure = rdt['pressure']
pres_value = (pressure / 100.0) + 0.5
for key, value in rdt.iteritems():
if key in out_rdt:
out_rdt[key] = value[:]
out_rdt['pressure'] = pres_value
return out_rdt.to_granule()
def execute(input=None,
context=None,
config=None,
params=None,
state=None):
"""
Dependencies
------------
CONDWAT_L1, TEMPWAT_L1, PRESWAT_L1
Algorithms used
---------------
PRACSAL = gsw_SP_from_C((CONDWAT_L1 * 10),TEMPWAT_L1,PRESWAT_L1)
Reference
---------
The calculations below are based on the following spreadsheet document:
https://docs.google.com/spreadsheet/ccc?key=0Au7PUzWoCKU4dDRMeVI0RU9yY180Z0Y5U0hyMUZERmc#gid=0
"""
rdt = RecordDictionaryTool.load_from_granule(input)
out_rdt = RecordDictionaryTool(stream_definition_id=params)
out_rdt['time'] = rdt['time']
conductivity = rdt['conductivity']
pressure = rdt['pressure']
temperature = rdt['temp']
sal_value = gsw.sp_from_c(conductivity * 10, temperature, pressure)
log.debug(
"CTDBP Salinity algorithm calculated the sp (practical salinity) values: %s",
sal_value)
for key, value in rdt.iteritems():
if key in out_rdt:
if key == 'conductivity' or key == 'temp' or key == 'pressure':
continue
out_rdt[key] = value[:]
out_rdt['salinity'] = sal_value
return out_rdt.to_granule()
def validate_output_granule(self, msg, route, stream_id):
self.assertTrue(
stream_id in
[self._output_stream_one_id, self._output_stream_two_id])
rdt = RecordDictionaryTool.load_from_granule(msg)
log.debug('validate_output_granule stream_id: %s', stream_id)
if stream_id == self._output_stream_one_id:
sal_val = rdt['salinity']
log.debug('validate_output_granule sal_val: %s', sal_val)
np.testing.assert_array_equal(sal_val, np.array([3]))
self.event1_verified.set()
else:
cond_val = rdt['conductivity']
log.debug('validate_output_granule cond_val: %s', cond_val)
np.testing.assert_array_equal(cond_val, np.array([5]))
self.event2_verified.set()
