def test_array_visualization(self): data_product_id, stream_def_id = self.make_array_data_product() # Make a granule with an array type, give it a few values # Send it to google_dt transform, verify output rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(2208988800, 2208988810) rdt['temp_sample'] = np.arange(10*4).reshape(10,4) rdt['cond_sample'] = np.arange(10*4).reshape(10,4) granule = rdt.to_granule() dataset_monitor = DatasetMonitor(self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id)) self.addCleanup(dataset_monitor.stop) self.ph.publish_rdt_to_data_product(data_product_id, rdt) dataset_monitor.event.wait(10) gdt_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('google_dt',id_only=True) gdt_stream_def = self.create_stream_definition('gdt', parameter_dictionary_id=gdt_pdict_id) gdt_data_granule = VizTransformGoogleDTAlgorithm.execute(granule, params=gdt_stream_def) rdt = RecordDictionaryTool.load_from_granule(gdt_data_granule) testval = {'data_content': [ [0.0 , 0.0 , 1.0 , 2.0 , 3.0 , 0.0 , 2.0 , 4.0 , 6.0 , 0.0 , 1.0 , 2.0 , 3.0] , [1.0 , 4.0 , 5.0 , 6.0 , 7.0 , 8.0 , 10.0 , 12.0 , 14.0 , 4.0 , 5.0 , 6.0 , 7.0] , [2.0 , 8.0 , 9.0 , 10.0 , 11.0 , 16.0 , 18.0 , 20.0 , 22.0 , 8.0 , 9.0 , 10.0 , 11.0] , [3.0 , 12.0 , 13.0 , 14.0 , 15.0 , 24.0 , 26.0 , 28.0 , 30.0 , 12.0 , 13.0 , 14.0 , 15.0] , [4.0 , 16.0 , 17.0 , 18.0 , 19.0 , 32.0 , 34.0 , 36.0 , 38.0 , 16.0 , 17.0 , 18.0 , 19.0] , [5.0 , 20.0 , 21.0 , 22.0 , 23.0 , 40.0 , 42.0 , 44.0 , 46.0 , 20.0 , 21.0 , 22.0 , 23.0] , [6.0 , 24.0 , 25.0 , 26.0 , 27.0 , 48.0 , 50.0 , 52.0 , 54.0 , 24.0 , 25.0 , 26.0 , 27.0] , [7.0 , 28.0 , 29.0 , 30.0 , 31.0 , 56.0 , 58.0 , 60.0 , 62.0 , 28.0 , 29.0 , 30.0 , 31.0] , [8.0 , 32.0 , 33.0 , 34.0 , 35.0 , 64.0 , 66.0 , 68.0 , 70.0 , 32.0 , 33.0 , 34.0 , 35.0] , [9.0 , 36.0 , 37.0 , 38.0 , 39.0 , 72.0 , 74.0 , 76.0 , 78.0 , 36.0 , 37.0 , 38.0 , 39.0]] , 'data_description': [('time', 'number', 'time'), ('temp_sample[0]', 'number', 'temp_sample[0]', {'precision': '5'}), ('temp_sample[1]', 'number', 'temp_sample[1]', {'precision': '5'}), ('temp_sample[2]', 'number', 'temp_sample[2]', {'precision': '5'}), ('temp_sample[3]', 'number', 'temp_sample[3]', {'precision': '5'}), ('temp_offset[0]', 'number', 'temp_offset[0]', {'precision': '5'}), ('temp_offset[1]', 'number', 'temp_offset[1]', {'precision': '5'}), ('temp_offset[2]', 'number', 'temp_offset[2]', {'precision': '5'}), ('temp_offset[3]', 'number', 'temp_offset[3]', {'precision': '5'}), ('cond_sample[0]', 'number', 'cond_sample[0]', {'precision': '5'}), ('cond_sample[1]', 'number', 'cond_sample[1]', {'precision': '5'}), ('cond_sample[2]', 'number', 'cond_sample[2]', {'precision': '5'}), ('cond_sample[3]', 'number', 'cond_sample[3]', {'precision': '5'})], 'viz_product_type': 'google_dt'} self.assertEquals(rdt['google_dt_components'][0], testval)
def get_visualization_data(self, data_product_id='', visualization_parameters=None, callback=''): """Retrieves the data for the specified DP and sends a token back which can be checked in a non-blocking fashion till data is ready @param data_product_id str @param visualization_parameters str @param callback str @retval jsonp_visualization_data str @throws NotFound object with specified id, query does not exist """ # error check if not data_product_id: raise BadRequest("The data_product_id parameter is missing") if visualization_parameters == {}: visualization_parameters = None # Extract the retrieval related parameters. Definitely init all parameters first query = None if visualization_parameters: query = {'parameters':[]} # Error check and damage control. Definitely need time if 'parameters' in visualization_parameters: if not 'time' in visualization_parameters['parameters']: visualization_parameters['parameters'].append('time') print ">>>>>>>>>>>>>>>>>>>> VIS PARAMS :", visualization_parameters['parameters'] query['parameters'] = visualization_parameters['parameters'] if 'stride_time' in visualization_parameters: query['stride_time'] = visualization_parameters['stride_time'] if 'start_time' in visualization_parameters: query['start_time'] = visualization_parameters['start_time'] if 'end_time' in visualization_parameters: query['end_time'] = visualization_parameters['end_time'] # get the dataset_id associated with the data_product. Need it to do the data retrieval ds_ids,_ = self.clients.resource_registry.find_objects(data_product_id, PRED.hasDataset, RT.DataSet, True) if ds_ids is None or not ds_ids: raise NotFound("Could not find dataset associated with data product") # Ideally just need the latest granule to figure out the list of images #replay_granule = self.clients.data_retriever.retrieve(ds_ids[0],{'start_time':0,'end_time':2}) retrieved_granule = self.clients.data_retriever.retrieve(ds_ids[0], query=query) if retrieved_granule is None: return None temp_rdt = RecordDictionaryTool.load_from_granule(retrieved_granule) # send the granule through the transform to get the google datatable gdt_pdict_id = self.clients.dataset_management.read_parameter_dictionary_by_name('google_dt',id_only=True) gdt_stream_def = self.clients.pubsub_management.create_stream_definition('gdt', parameter_dictionary_id=gdt_pdict_id) gdt_data_granule = VizTransformGoogleDTAlgorithm.execute(retrieved_granule, params=gdt_stream_def) if gdt_data_granule == None: return None gdt_rdt = RecordDictionaryTool.load_from_granule(gdt_data_granule) gdt_components = get_safe(gdt_rdt, 'google_dt_components') gdt_component = gdt_components[0] temp_gdt_description = gdt_component["data_description"] temp_gdt_content = gdt_component["data_content"] # adjust the 'float' time to datetime in the content gdt_description = [('time', 'datetime', 'time')] gdt_content = [] for idx in range(1,len(temp_gdt_description)): temp_arr = temp_gdt_description[idx] if temp_arr != None and temp_arr[0] != 'time': gdt_description.append((temp_arr[0], temp_arr[1], temp_arr[2])) for tempTuple in temp_gdt_content: # sometimes there are inexplicable empty tuples in the content. Drop them if tempTuple == [] or len(tempTuple) == 0: continue varTuple = [] varTuple.append(datetime.fromtimestamp(tempTuple[0])) for idx in range(1,len(tempTuple)): # some silly numpy format won't go away so need to cast numbers to floats if(gdt_description[idx][1] == 'number'): if tempTuple[idx] == None: varTuple.append(0.0) else: varTuple.append(float(tempTuple[idx])) else: varTuple.append(tempTuple[idx]) gdt_content.append(varTuple) # now generate the Google datatable out of the description and content gdt = gviz_api.DataTable(gdt_description) gdt.LoadData(gdt_content) # return the json version of the table if callback == '': return gdt.ToJSonResponse() else: return callback + "(\"" + gdt.ToJSonResponse() + "\")"
def _get_google_dt(self, data_product_id='', visualization_parameters=None): """Retrieves the data for the specified DP @param data_product_id str @param visualization_parameters str @retval jsonp_visualization_data str @throws NotFound object with specified id, query does not exist """ # An empty dict is returned in case there is no data in coverage empty_gdt = gviz_api.DataTable([('time', 'datetime', 'time')]) # error check if not data_product_id: raise BadRequest("The data_product_id parameter is missing") use_direct_access = False if visualization_parameters == {}: visualization_parameters = None # Extract the parameters. Definitely init first query = None if visualization_parameters: #query = {'parameters':[]} query = {} # Error check and damage control. Definitely need time if 'parameters' in visualization_parameters and len(visualization_parameters['parameters']) > 0: if not 'time' in visualization_parameters['parameters']: visualization_parameters['parameters'].append('time') query['parameters'] = visualization_parameters['parameters'] # The times passed from UI are system times so convert them to NTP if 'start_time' in visualization_parameters: query['start_time'] = int(visualization_parameters['start_time']) if 'end_time' in visualization_parameters: query['end_time'] = int((visualization_parameters['end_time'])) # stride time if 'stride_time' in visualization_parameters: try: query['stride_time'] = int(visualization_parameters['stride_time']) except TypeError: # There are some (rare) situations where the AJAX request has 'null' in the request # Example: # {"query_type":"google_dt","parameters":[],"start_time":-2208988800,"end_time":-2208988800,"stride_time":null,"use_direct_access":0} query['stride_time'] = 1 else: query['stride_time'] = 1 # direct access parameter if 'use_direct_access' in visualization_parameters: if (int(visualization_parameters['use_direct_access']) == 1): use_direct_access = True else: use_direct_access = False # get the dataset_id and objs associated with the data_product. Need it to do the data retrieval ds_ids,_ = self.clients.resource_registry.find_objects(data_product_id, PRED.hasDataset, RT.Dataset, True) if ds_ids is None or not ds_ids: raise NotFound("Could not find dataset associated with data product") stream_def_ids, _ = self.clients.resource_registry.find_objects(data_product_id, PRED.hasStreamDefinition, id_only=True) if not stream_def_ids: raise NotFound('Could not find stream definition associated with data product') stream_def_id = stream_def_ids[0] if use_direct_access: retrieved_granule = DataRetrieverService.retrieve_oob(ds_ids[0], query=query, delivery_format=stream_def_id) else: #replay_granule = self.clients.data_retriever.retrieve(ds_ids[0],{'start_time':0,'end_time':2}) retrieved_granule = self.clients.data_retriever.retrieve(ds_ids[0], query=query, delivery_format=stream_def_id) # If thereis no data, return an empty dict if retrieved_granule is None: return empty_gdt.ToJSon() # send the granule through the transform to get the google datatable gdt_pdict_id = self.clients.dataset_management.read_parameter_dictionary_by_name('google_dt',id_only=True) gdt_stream_def = self.clients.pubsub_management.create_stream_definition('gdt', parameter_dictionary_id=gdt_pdict_id) gdt_data_granule = VizTransformGoogleDTAlgorithm.execute(retrieved_granule, params=gdt_stream_def, config=visualization_parameters) if gdt_data_granule == None: return empty_gdt.ToJSon() gdt_rdt = RecordDictionaryTool.load_from_granule(gdt_data_granule) gdt_components = get_safe(gdt_rdt, 'google_dt_components') gdt_component = gdt_components[0] temp_gdt_description = gdt_component["data_description"] temp_gdt_content = gdt_component["data_content"] # adjust the 'float' time to datetime in the content gdt_description = [('time', 'datetime', 'time')] gdt_content = [] for idx in range(1,len(temp_gdt_description)): temp_arr = temp_gdt_description[idx] if temp_arr != None and temp_arr[0] != 'time': if len(temp_arr) == 3: gdt_description.append((temp_arr[0], temp_arr[1], temp_arr[2])) if len(temp_arr) == 4: gdt_description.append((temp_arr[0], temp_arr[1], temp_arr[2], temp_arr[3])) for tempTuple in temp_gdt_content: # sometimes there are inexplicable empty tuples in the content. Drop them if tempTuple == [] or len(tempTuple) == 0: continue varTuple = [] varTuple.append(datetime.fromtimestamp(tempTuple[0])) for idx in range(1,len(tempTuple)): varTuple.append(tempTuple[idx]) gdt_content.append(varTuple) # now generate the Google datatable out of the description and content gdt = gviz_api.DataTable(gdt_description) gdt.LoadData(gdt_content) # return the json version of the table return gdt.ToJSon()
def get_visualization_data(self, data_product_id='', visualization_parameters=None, callback='', tqx=""): """Retrieves the data for the specified DP @param data_product_id str @param visualization_parameters str @param callback str @retval jsonp_visualization_data str @throws NotFound object with specified id, query does not exist """ # error check if not data_product_id: raise BadRequest("The data_product_id parameter is missing") use_direct_access = False if visualization_parameters == {}: visualization_parameters = None reqId = 0 # If a reqId was passed in tqx, extract it if tqx: tqx_param_list = tqx.split(";") for param in tqx_param_list: key, value = param.split(":") if key == 'reqId': reqId = value # Extract the parameters. Definitely init first query = None if visualization_parameters: #query = {'parameters':[]} query = {} # Error check and damage control. Definitely need time if 'parameters' in visualization_parameters and len(visualization_parameters['parameters']) > 0: if not 'time' in visualization_parameters['parameters']: visualization_parameters['parameters'].append('time') query['parameters'] = visualization_parameters['parameters'] # The times passed from UI are system times so convert them to NTP if 'start_time' in visualization_parameters: #query['start_time'] = int(ntplib.system_to_ntp_time(float(visualization_parameters['start_time']))) query['start_time'] = int(visualization_parameters['start_time']) if 'end_time' in visualization_parameters: #query['end_time'] = int(ntplib.system_to_ntp_time(float(visualization_parameters['end_time']))) query['end_time'] = int((visualization_parameters['end_time'])) # stride time if 'stride_time' in visualization_parameters: query['stride_time'] = int(visualization_parameters['stride_time']) else: query['stride_time'] == 1 # direct access parameter if 'use_direct_access' in visualization_parameters: if (int(visualization_parameters['use_direct_access']) == 1): use_direct_access = True else: use_direct_access = False # get the dataset_id associated with the data_product. Need it to do the data retrieval ds_ids,_ = self.clients.resource_registry.find_objects(data_product_id, PRED.hasDataset, RT.Dataset, True) if ds_ids is None or not ds_ids: raise NotFound("Could not find dataset associated with data product") if use_direct_access: retrieved_granule = DataRetrieverService.retrieve_oob(ds_ids[0], query=query) else: #replay_granule = self.clients.data_retriever.retrieve(ds_ids[0],{'start_time':0,'end_time':2}) retrieved_granule = self.clients.data_retriever.retrieve(ds_ids[0], query=query) if retrieved_granule is None: return None #temp_rdt = RecordDictionaryTool.load_from_granule(retrieved_granule) # send the granule through the transform to get the google datatable gdt_pdict_id = self.clients.dataset_management.read_parameter_dictionary_by_name('google_dt',id_only=True) gdt_stream_def = self.clients.pubsub_management.create_stream_definition('gdt', parameter_dictionary_id=gdt_pdict_id) gdt_data_granule = VizTransformGoogleDTAlgorithm.execute(retrieved_granule, params=gdt_stream_def, config=visualization_parameters) if gdt_data_granule == None: return None gdt_rdt = RecordDictionaryTool.load_from_granule(gdt_data_granule) gdt_components = get_safe(gdt_rdt, 'google_dt_components') gdt_component = gdt_components[0] temp_gdt_description = gdt_component["data_description"] temp_gdt_content = gdt_component["data_content"] # adjust the 'float' time to datetime in the content gdt_description = [('time', 'datetime', 'time')] gdt_content = [] for idx in range(1,len(temp_gdt_description)): temp_arr = temp_gdt_description[idx] if temp_arr != None and temp_arr[0] != 'time': gdt_description.append((temp_arr[0], temp_arr[1], temp_arr[2])) for tempTuple in temp_gdt_content: # sometimes there are inexplicable empty tuples in the content. Drop them if tempTuple == [] or len(tempTuple) == 0: continue varTuple = [] varTuple.append(datetime.fromtimestamp(tempTuple[0])) for idx in range(1,len(tempTuple)): # some silly numpy format won't go away so need to cast numbers to floats if(gdt_description[idx][1] == 'number'): if tempTuple[idx] == None: varTuple.append(0.0) else: # Precision hardcoded for now. Needs to be on a per parameter basis varTuple.append(round(float(tempTuple[idx]),5)) else: varTuple.append(tempTuple[idx]) gdt_content.append(varTuple) # now generate the Google datatable out of the description and content gdt = gviz_api.DataTable(gdt_description) gdt.LoadData(gdt_content) # return the json version of the table if callback == '': return gdt.ToJSonResponse(req_id = reqId) else: return callback + "(\"" + gdt.ToJSonResponse(req_id = reqId) + "\")"