def test_array_visualization(self):
data_product_id, stream_def_id = self.make_array_data_product()
# Make a granule with an array type, give it a few values
# Send it to google_dt transform, verify output
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(2208988800, 2208988810)
rdt['temp_sample'] = np.arange(10*4).reshape(10,4)
rdt['cond_sample'] = np.arange(10*4).reshape(10,4)
granule = rdt.to_granule()
dataset_monitor = DatasetMonitor(self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id))
self.addCleanup(dataset_monitor.stop)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
dataset_monitor.event.wait(10)
gdt_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('google_dt',id_only=True)
gdt_stream_def = self.create_stream_definition('gdt', parameter_dictionary_id=gdt_pdict_id)
gdt_data_granule = VizTransformGoogleDTAlgorithm.execute(granule, params=gdt_stream_def)
rdt = RecordDictionaryTool.load_from_granule(gdt_data_granule)
testval = {'data_content': [
[0.0 , 0.0 , 1.0 , 2.0 , 3.0 , 0.0 , 2.0 , 4.0 , 6.0 , 0.0 , 1.0 , 2.0 , 3.0] ,
[1.0 , 4.0 , 5.0 , 6.0 , 7.0 , 8.0 , 10.0 , 12.0 , 14.0 , 4.0 , 5.0 , 6.0 , 7.0] ,
[2.0 , 8.0 , 9.0 , 10.0 , 11.0 , 16.0 , 18.0 , 20.0 , 22.0 , 8.0 , 9.0 , 10.0 , 11.0] ,
[3.0 , 12.0 , 13.0 , 14.0 , 15.0 , 24.0 , 26.0 , 28.0 , 30.0 , 12.0 , 13.0 , 14.0 , 15.0] ,
[4.0 , 16.0 , 17.0 , 18.0 , 19.0 , 32.0 , 34.0 , 36.0 , 38.0 , 16.0 , 17.0 , 18.0 , 19.0] ,
[5.0 , 20.0 , 21.0 , 22.0 , 23.0 , 40.0 , 42.0 , 44.0 , 46.0 , 20.0 , 21.0 , 22.0 , 23.0] ,
[6.0 , 24.0 , 25.0 , 26.0 , 27.0 , 48.0 , 50.0 , 52.0 , 54.0 , 24.0 , 25.0 , 26.0 , 27.0] ,
[7.0 , 28.0 , 29.0 , 30.0 , 31.0 , 56.0 , 58.0 , 60.0 , 62.0 , 28.0 , 29.0 , 30.0 , 31.0] ,
[8.0 , 32.0 , 33.0 , 34.0 , 35.0 , 64.0 , 66.0 , 68.0 , 70.0 , 32.0 , 33.0 , 34.0 , 35.0] ,
[9.0 , 36.0 , 37.0 , 38.0 , 39.0 , 72.0 , 74.0 , 76.0 , 78.0 , 36.0 , 37.0 , 38.0 , 39.0]] ,
'data_description': [('time', 'number', 'time'),
('temp_sample[0]', 'number', 'temp_sample[0]', {'precision': '5'}),
('temp_sample[1]', 'number', 'temp_sample[1]', {'precision': '5'}),
('temp_sample[2]', 'number', 'temp_sample[2]', {'precision': '5'}),
('temp_sample[3]', 'number', 'temp_sample[3]', {'precision': '5'}),
('temp_offset[0]', 'number', 'temp_offset[0]', {'precision': '5'}),
('temp_offset[1]', 'number', 'temp_offset[1]', {'precision': '5'}),
('temp_offset[2]', 'number', 'temp_offset[2]', {'precision': '5'}),
('temp_offset[3]', 'number', 'temp_offset[3]', {'precision': '5'}),
('cond_sample[0]', 'number', 'cond_sample[0]', {'precision': '5'}),
('cond_sample[1]', 'number', 'cond_sample[1]', {'precision': '5'}),
('cond_sample[2]', 'number', 'cond_sample[2]', {'precision': '5'}),
('cond_sample[3]', 'number', 'cond_sample[3]', {'precision': '5'})],
'viz_product_type': 'google_dt'}
self.assertEquals(rdt['google_dt_components'][0], testval)
def test_array_visualization(self):
data_product_id, stream_def_id = self.make_array_data_product()
# Make a granule with an array type, give it a few values
# Send it to google_dt transform, verify output
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(2208988800, 2208988810)
rdt['temp_sample'] = np.arange(10*4).reshape(10,4)
rdt['cond_sample'] = np.arange(10*4).reshape(10,4)
granule = rdt.to_granule()
dataset_monitor = DatasetMonitor(self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id))
self.addCleanup(dataset_monitor.stop)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
dataset_monitor.event.wait(10)
gdt_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('google_dt',id_only=True)
gdt_stream_def = self.create_stream_definition('gdt', parameter_dictionary_id=gdt_pdict_id)
gdt_data_granule = VizTransformGoogleDTAlgorithm.execute(granule, params=gdt_stream_def)
rdt = RecordDictionaryTool.load_from_granule(gdt_data_granule)
testval = {'data_content': [
[0.0 , 0.0 , 1.0 , 2.0 , 3.0 , 0.0 , 2.0 , 4.0 , 6.0 , 0.0 , 1.0 , 2.0 , 3.0] ,
[1.0 , 4.0 , 5.0 , 6.0 , 7.0 , 8.0 , 10.0 , 12.0 , 14.0 , 4.0 , 5.0 , 6.0 , 7.0] ,
[2.0 , 8.0 , 9.0 , 10.0 , 11.0 , 16.0 , 18.0 , 20.0 , 22.0 , 8.0 , 9.0 , 10.0 , 11.0] ,
[3.0 , 12.0 , 13.0 , 14.0 , 15.0 , 24.0 , 26.0 , 28.0 , 30.0 , 12.0 , 13.0 , 14.0 , 15.0] ,
[4.0 , 16.0 , 17.0 , 18.0 , 19.0 , 32.0 , 34.0 , 36.0 , 38.0 , 16.0 , 17.0 , 18.0 , 19.0] ,
[5.0 , 20.0 , 21.0 , 22.0 , 23.0 , 40.0 , 42.0 , 44.0 , 46.0 , 20.0 , 21.0 , 22.0 , 23.0] ,
[6.0 , 24.0 , 25.0 , 26.0 , 27.0 , 48.0 , 50.0 , 52.0 , 54.0 , 24.0 , 25.0 , 26.0 , 27.0] ,
[7.0 , 28.0 , 29.0 , 30.0 , 31.0 , 56.0 , 58.0 , 60.0 , 62.0 , 28.0 , 29.0 , 30.0 , 31.0] ,
[8.0 , 32.0 , 33.0 , 34.0 , 35.0 , 64.0 , 66.0 , 68.0 , 70.0 , 32.0 , 33.0 , 34.0 , 35.0] ,
[9.0 , 36.0 , 37.0 , 38.0 , 39.0 , 72.0 , 74.0 , 76.0 , 78.0 , 36.0 , 37.0 , 38.0 , 39.0]] ,
'data_description': [('time', 'number', 'time'),
('temp_sample[0]', 'number', 'temp_sample[0]', {'precision': '5'}),
('temp_sample[1]', 'number', 'temp_sample[1]', {'precision': '5'}),
('temp_sample[2]', 'number', 'temp_sample[2]', {'precision': '5'}),
('temp_sample[3]', 'number', 'temp_sample[3]', {'precision': '5'}),
('temp_offset[0]', 'number', 'temp_offset[0]', {'precision': '5'}),
('temp_offset[1]', 'number', 'temp_offset[1]', {'precision': '5'}),
('temp_offset[2]', 'number', 'temp_offset[2]', {'precision': '5'}),
('temp_offset[3]', 'number', 'temp_offset[3]', {'precision': '5'}),
('cond_sample[0]', 'number', 'cond_sample[0]', {'precision': '5'}),
('cond_sample[1]', 'number', 'cond_sample[1]', {'precision': '5'}),
('cond_sample[2]', 'number', 'cond_sample[2]', {'precision': '5'}),
('cond_sample[3]', 'number', 'cond_sample[3]', {'precision': '5'})],
'viz_product_type': 'google_dt'}
self.assertEquals(rdt['google_dt_components'][0], testval)
def get_visualization_data(self, data_product_id='', visualization_parameters=None, callback=''):
"""Retrieves the data for the specified DP and sends a token back which can be checked in
a non-blocking fashion till data is ready
@param data_product_id str
@param visualization_parameters str
@param callback str
@retval jsonp_visualization_data str
@throws NotFound object with specified id, query does not exist
"""
# error check
if not data_product_id:
raise BadRequest("The data_product_id parameter is missing")
if visualization_parameters == {}:
visualization_parameters = None
# Extract the retrieval related parameters. Definitely init all parameters first
query = None
if visualization_parameters:
query = {'parameters':[]}
# Error check and damage control. Definitely need time
if 'parameters' in visualization_parameters:
if not 'time' in visualization_parameters['parameters']:
visualization_parameters['parameters'].append('time')
print ">>>>>>>>>>>>>>>>>>>> VIS PARAMS :", visualization_parameters['parameters']
query['parameters'] = visualization_parameters['parameters']
if 'stride_time' in visualization_parameters:
query['stride_time'] = visualization_parameters['stride_time']
if 'start_time' in visualization_parameters:
query['start_time'] = visualization_parameters['start_time']
if 'end_time' in visualization_parameters:
query['end_time'] = visualization_parameters['end_time']
# get the dataset_id associated with the data_product. Need it to do the data retrieval
ds_ids,_ = self.clients.resource_registry.find_objects(data_product_id, PRED.hasDataset, RT.DataSet, True)
if ds_ids is None or not ds_ids:
raise NotFound("Could not find dataset associated with data product")
# Ideally just need the latest granule to figure out the list of images
#replay_granule = self.clients.data_retriever.retrieve(ds_ids[0],{'start_time':0,'end_time':2})
retrieved_granule = self.clients.data_retriever.retrieve(ds_ids[0], query=query)
if retrieved_granule is None:
return None
temp_rdt = RecordDictionaryTool.load_from_granule(retrieved_granule)
# send the granule through the transform to get the google datatable
gdt_pdict_id = self.clients.dataset_management.read_parameter_dictionary_by_name('google_dt',id_only=True)
gdt_stream_def = self.clients.pubsub_management.create_stream_definition('gdt', parameter_dictionary_id=gdt_pdict_id)
gdt_data_granule = VizTransformGoogleDTAlgorithm.execute(retrieved_granule, params=gdt_stream_def)
if gdt_data_granule == None:
return None
gdt_rdt = RecordDictionaryTool.load_from_granule(gdt_data_granule)
gdt_components = get_safe(gdt_rdt, 'google_dt_components')
gdt_component = gdt_components[0]
temp_gdt_description = gdt_component["data_description"]
temp_gdt_content = gdt_component["data_content"]
# adjust the 'float' time to datetime in the content
gdt_description = [('time', 'datetime', 'time')]
gdt_content = []
for idx in range(1,len(temp_gdt_description)):
temp_arr = temp_gdt_description[idx]
if temp_arr != None and temp_arr[0] != 'time':
gdt_description.append((temp_arr[0], temp_arr[1], temp_arr[2]))
for tempTuple in temp_gdt_content:
# sometimes there are inexplicable empty tuples in the content. Drop them
if tempTuple == [] or len(tempTuple) == 0:
continue
varTuple = []
varTuple.append(datetime.fromtimestamp(tempTuple[0]))
for idx in range(1,len(tempTuple)):
# some silly numpy format won't go away so need to cast numbers to floats
if(gdt_description[idx][1] == 'number'):
if tempTuple[idx] == None:
varTuple.append(0.0)
else:
varTuple.append(float(tempTuple[idx]))
else:
varTuple.append(tempTuple[idx])
gdt_content.append(varTuple)
# now generate the Google datatable out of the description and content
gdt = gviz_api.DataTable(gdt_description)
gdt.LoadData(gdt_content)
# return the json version of the table
if callback == '':
return gdt.ToJSonResponse()
else:
return callback + "(\"" + gdt.ToJSonResponse() + "\")"
def _get_google_dt(self, data_product_id='', visualization_parameters=None):
"""Retrieves the data for the specified DP
@param data_product_id str
@param visualization_parameters str
@retval jsonp_visualization_data str
@throws NotFound object with specified id, query does not exist
"""
# An empty dict is returned in case there is no data in coverage
empty_gdt = gviz_api.DataTable([('time', 'datetime', 'time')])
# error check
if not data_product_id:
raise BadRequest("The data_product_id parameter is missing")
use_direct_access = False
if visualization_parameters == {}:
visualization_parameters = None
# Extract the parameters. Definitely init first
query = None
if visualization_parameters:
#query = {'parameters':[]}
query = {}
# Error check and damage control. Definitely need time
if 'parameters' in visualization_parameters and len(visualization_parameters['parameters']) > 0:
if not 'time' in visualization_parameters['parameters']:
visualization_parameters['parameters'].append('time')
query['parameters'] = visualization_parameters['parameters']
# The times passed from UI are system times so convert them to NTP
if 'start_time' in visualization_parameters:
query['start_time'] = int(visualization_parameters['start_time'])
if 'end_time' in visualization_parameters:
query['end_time'] = int((visualization_parameters['end_time']))
# stride time
if 'stride_time' in visualization_parameters:
try:
query['stride_time'] = int(visualization_parameters['stride_time'])
except TypeError:
# There are some (rare) situations where the AJAX request has 'null' in the request
# Example:
# {"query_type":"google_dt","parameters":[],"start_time":-2208988800,"end_time":-2208988800,"stride_time":null,"use_direct_access":0}
query['stride_time'] = 1
else:
query['stride_time'] = 1
# direct access parameter
if 'use_direct_access' in visualization_parameters:
if (int(visualization_parameters['use_direct_access']) == 1):
use_direct_access = True
else:
use_direct_access = False
# get the dataset_id and objs associated with the data_product. Need it to do the data retrieval
ds_ids,_ = self.clients.resource_registry.find_objects(data_product_id, PRED.hasDataset, RT.Dataset, True)
if ds_ids is None or not ds_ids:
raise NotFound("Could not find dataset associated with data product")
stream_def_ids, _ = self.clients.resource_registry.find_objects(data_product_id, PRED.hasStreamDefinition, id_only=True)
if not stream_def_ids:
raise NotFound('Could not find stream definition associated with data product')
stream_def_id = stream_def_ids[0]
if use_direct_access:
retrieved_granule = DataRetrieverService.retrieve_oob(ds_ids[0], query=query, delivery_format=stream_def_id)
else:
#replay_granule = self.clients.data_retriever.retrieve(ds_ids[0],{'start_time':0,'end_time':2})
retrieved_granule = self.clients.data_retriever.retrieve(ds_ids[0], query=query, delivery_format=stream_def_id)
# If thereis no data, return an empty dict
if retrieved_granule is None:
return empty_gdt.ToJSon()
# send the granule through the transform to get the google datatable
gdt_pdict_id = self.clients.dataset_management.read_parameter_dictionary_by_name('google_dt',id_only=True)
gdt_stream_def = self.clients.pubsub_management.create_stream_definition('gdt', parameter_dictionary_id=gdt_pdict_id)
gdt_data_granule = VizTransformGoogleDTAlgorithm.execute(retrieved_granule, params=gdt_stream_def, config=visualization_parameters)
if gdt_data_granule == None:
return empty_gdt.ToJSon()
gdt_rdt = RecordDictionaryTool.load_from_granule(gdt_data_granule)
gdt_components = get_safe(gdt_rdt, 'google_dt_components')
gdt_component = gdt_components[0]
temp_gdt_description = gdt_component["data_description"]
temp_gdt_content = gdt_component["data_content"]
# adjust the 'float' time to datetime in the content
gdt_description = [('time', 'datetime', 'time')]
gdt_content = []
for idx in range(1,len(temp_gdt_description)):
temp_arr = temp_gdt_description[idx]
if temp_arr != None and temp_arr[0] != 'time':
if len(temp_arr) == 3:
gdt_description.append((temp_arr[0], temp_arr[1], temp_arr[2]))
if len(temp_arr) == 4:
gdt_description.append((temp_arr[0], temp_arr[1], temp_arr[2], temp_arr[3]))
for tempTuple in temp_gdt_content:
# sometimes there are inexplicable empty tuples in the content. Drop them
if tempTuple == [] or len(tempTuple) == 0:
continue
varTuple = []
varTuple.append(datetime.fromtimestamp(tempTuple[0]))
for idx in range(1,len(tempTuple)):
varTuple.append(tempTuple[idx])
gdt_content.append(varTuple)
# now generate the Google datatable out of the description and content
gdt = gviz_api.DataTable(gdt_description)
gdt.LoadData(gdt_content)
# return the json version of the table
return gdt.ToJSon()
def get_visualization_data(self, data_product_id='', visualization_parameters=None, callback='', tqx=""):
"""Retrieves the data for the specified DP
@param data_product_id str
@param visualization_parameters str
@param callback str
@retval jsonp_visualization_data str
@throws NotFound object with specified id, query does not exist
"""
# error check
if not data_product_id:
raise BadRequest("The data_product_id parameter is missing")
use_direct_access = False
if visualization_parameters == {}:
visualization_parameters = None
reqId = 0
# If a reqId was passed in tqx, extract it
if tqx:
tqx_param_list = tqx.split(";")
for param in tqx_param_list:
key, value = param.split(":")
if key == 'reqId':
reqId = value
# Extract the parameters. Definitely init first
query = None
if visualization_parameters:
#query = {'parameters':[]}
query = {}
# Error check and damage control. Definitely need time
if 'parameters' in visualization_parameters and len(visualization_parameters['parameters']) > 0:
if not 'time' in visualization_parameters['parameters']:
visualization_parameters['parameters'].append('time')
query['parameters'] = visualization_parameters['parameters']
# The times passed from UI are system times so convert them to NTP
if 'start_time' in visualization_parameters:
#query['start_time'] = int(ntplib.system_to_ntp_time(float(visualization_parameters['start_time'])))
query['start_time'] = int(visualization_parameters['start_time'])
if 'end_time' in visualization_parameters:
#query['end_time'] = int(ntplib.system_to_ntp_time(float(visualization_parameters['end_time'])))
query['end_time'] = int((visualization_parameters['end_time']))
# stride time
if 'stride_time' in visualization_parameters:
query['stride_time'] = int(visualization_parameters['stride_time'])
else:
query['stride_time'] == 1
# direct access parameter
if 'use_direct_access' in visualization_parameters:
if (int(visualization_parameters['use_direct_access']) == 1):
use_direct_access = True
else:
use_direct_access = False
# get the dataset_id associated with the data_product. Need it to do the data retrieval
ds_ids,_ = self.clients.resource_registry.find_objects(data_product_id, PRED.hasDataset, RT.Dataset, True)
if ds_ids is None or not ds_ids:
raise NotFound("Could not find dataset associated with data product")
if use_direct_access:
retrieved_granule = DataRetrieverService.retrieve_oob(ds_ids[0], query=query)
else:
#replay_granule = self.clients.data_retriever.retrieve(ds_ids[0],{'start_time':0,'end_time':2})
retrieved_granule = self.clients.data_retriever.retrieve(ds_ids[0], query=query)
if retrieved_granule is None:
return None
#temp_rdt = RecordDictionaryTool.load_from_granule(retrieved_granule)
# send the granule through the transform to get the google datatable
gdt_pdict_id = self.clients.dataset_management.read_parameter_dictionary_by_name('google_dt',id_only=True)
gdt_stream_def = self.clients.pubsub_management.create_stream_definition('gdt', parameter_dictionary_id=gdt_pdict_id)
gdt_data_granule = VizTransformGoogleDTAlgorithm.execute(retrieved_granule, params=gdt_stream_def, config=visualization_parameters)
if gdt_data_granule == None:
return None
gdt_rdt = RecordDictionaryTool.load_from_granule(gdt_data_granule)
gdt_components = get_safe(gdt_rdt, 'google_dt_components')
gdt_component = gdt_components[0]
temp_gdt_description = gdt_component["data_description"]
temp_gdt_content = gdt_component["data_content"]
# adjust the 'float' time to datetime in the content
gdt_description = [('time', 'datetime', 'time')]
gdt_content = []
for idx in range(1,len(temp_gdt_description)):
temp_arr = temp_gdt_description[idx]
if temp_arr != None and temp_arr[0] != 'time':
gdt_description.append((temp_arr[0], temp_arr[1], temp_arr[2]))
for tempTuple in temp_gdt_content:
# sometimes there are inexplicable empty tuples in the content. Drop them
if tempTuple == [] or len(tempTuple) == 0:
continue
varTuple = []
varTuple.append(datetime.fromtimestamp(tempTuple[0]))
for idx in range(1,len(tempTuple)):
# some silly numpy format won't go away so need to cast numbers to floats
if(gdt_description[idx][1] == 'number'):
if tempTuple[idx] == None:
varTuple.append(0.0)
else:
# Precision hardcoded for now. Needs to be on a per parameter basis
varTuple.append(round(float(tempTuple[idx]),5))
else:
varTuple.append(tempTuple[idx])
gdt_content.append(varTuple)
# now generate the Google datatable out of the description and content
gdt = gviz_api.DataTable(gdt_description)
gdt.LoadData(gdt_content)
# return the json version of the table
if callback == '':
return gdt.ToJSonResponse(req_id = reqId)
else:
return callback + "(\"" + gdt.ToJSonResponse(req_id = reqId) + "\")"
