def message_received(granule, h):
rdt = RecordDictionaryTool.load_from_granule(granule)
log.warn(
'Logging Record Dictionary received in logger subscription \n%s',
rdt.pretty_print())
def _validate_google_dt_results(self, results_stream_def, results):
cc = self.container
assertions = self.assertTrue
for g in results:
if isinstance(g,Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
#log.warn(tx.pretty_print())
#log.warn(rdt.pretty_print())
gdt_component = rdt['google_dt_components'][0]
assertions(gdt_component['viz_product_type'] == 'google_realtime_dt' )
gdt_description = gdt_component['data_table_description']
gdt_content = gdt_component['data_table_content']
assertions(gdt_description[0][0] == 'time')
assertions(len(gdt_description) > 1)
assertions(len(gdt_content) >= 0)
return
def _validate_google_dt_results(self, results_stream_def, results):
cc = self.container
assertions = self.assertTrue
for g in results:
if isinstance(g, Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
#log.warn(tx.pretty_print())
#log.warn(rdt.pretty_print())
gdt_component = rdt['google_dt_components'][0]
assertions(
gdt_component['viz_product_type'] == 'google_realtime_dt')
gdt_description = gdt_component['data_table_description']
gdt_content = gdt_component['data_table_content']
assertions(gdt_description[0][0] == 'time')
assertions(len(gdt_description) > 1)
assertions(len(gdt_content) >= 0)
return
def test_build_granule_and_load_from_granule(self):
#Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle
tx = TaxyTool()
tx.add_taxonomy_set('temp', 'long_temp_name')
tx.add_taxonomy_set('cond', 'long_cond_name')
tx.add_taxonomy_set('pres', 'long_pres_name')
tx.add_taxonomy_set('rdt')
# map is {<local name>: <granule name or path>}
#Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect
rdt = RecordDictionaryTool(taxonomy=tx)
#Create some arrays and fill them with random values
temp_array = numpy.random.standard_normal(100)
cond_array = numpy.random.standard_normal(100)
pres_array = numpy.random.standard_normal(100)
#Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc.
rdt['temp'] = temp_array
rdt['cond'] = cond_array
rdt['pres'] = pres_array
#You can also add in another RecordDictionaryTool, providing the taxonomies are the same.
rdt2 = RecordDictionaryTool(taxonomy=tx)
rdt2['temp'] = temp_array
rdt['rdt'] = rdt2
g = build_granule(data_producer_id='john', taxonomy=tx, record_dictionary=rdt)
l_tx = TaxyTool.load_from_granule(g)
l_rd = RecordDictionaryTool.load_from_granule(g)
# Make sure we got back the same Taxonomy Object
self.assertEquals(l_tx._t, tx._t)
self.assertEquals(l_tx.get_handles('temp'), tx.get_handles('temp'))
self.assertEquals(l_tx.get_handles('testing_2'), tx.get_handles('testing_2'))
# Now test the record dictionary object
self.assertEquals(l_rd._rd, rdt._rd)
self.assertEquals(l_rd._tx._t, rdt._tx._t)
for k, v in l_rd.iteritems():
self.assertIn(k, rdt)
if isinstance(v, numpy.ndarray):
self.assertTrue( (v == rdt[k]).all())
else:
self.assertEquals(v._rd, rdt[k]._rd)
def test_build_granule_and_load_from_granule(self):
#Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle
tx = TaxyTool()
tx.add_taxonomy_set('temp', 'long_temp_name')
tx.add_taxonomy_set('cond', 'long_cond_name')
tx.add_taxonomy_set('pres', 'long_pres_name')
tx.add_taxonomy_set('rdt')
# map is {<local name>: <granule name or path>}
#Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect
rdt = RecordDictionaryTool(taxonomy=tx)
#Create some arrays and fill them with random values
temp_array = numpy.random.standard_normal(100)
cond_array = numpy.random.standard_normal(100)
pres_array = numpy.random.standard_normal(100)
#Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc.
rdt['temp'] = temp_array
rdt['cond'] = cond_array
rdt['pres'] = pres_array
#You can also add in another RecordDictionaryTool, providing the taxonomies are the same.
rdt2 = RecordDictionaryTool(taxonomy=tx)
rdt2['temp'] = temp_array
rdt['rdt'] = rdt2
g = build_granule(data_producer_id='john',
taxonomy=tx,
record_dictionary=rdt)
l_tx = TaxyTool.load_from_granule(g)
l_rd = RecordDictionaryTool.load_from_granule(g)
# Make sure we got back the same Taxonomy Object
self.assertEquals(l_tx._t, tx._t)
self.assertEquals(l_tx.get_handles('temp'), tx.get_handles('temp'))
self.assertEquals(l_tx.get_handles('testing_2'),
tx.get_handles('testing_2'))
# Now test the record dictionary object
self.assertEquals(l_rd._rd, rdt._rd)
self.assertEquals(l_rd._tx._t, rdt._tx._t)
for k, v in l_rd.iteritems():
self.assertIn(k, rdt)
if isinstance(v, numpy.ndarray):
self.assertTrue((v == rdt[k]).all())
else:
self.assertEquals(v._rd, rdt[k]._rd)
def combine_granules(granule_a, granule_b):
"""
This is a method that combines granules in a very naive manner
"""
validate_is_instance(granule_a, Granule, "granule_a is not a proper Granule")
validate_is_instance(granule_b, Granule, "granule_b is not a proper Granule")
tt_a = TaxyTool.load_from_granule(granule_a)
tt_b = TaxyTool.load_from_granule(granule_b)
if tt_a != tt_b:
raise BadRequest("Can't combine the two granules, they do not have the same taxonomy.")
rdt_new = RecordDictionaryTool(tt_a)
rdt_a = RecordDictionaryTool.load_from_granule(granule_a)
rdt_b = RecordDictionaryTool.load_from_granule(granule_b)
for k in rdt_a.iterkeys():
rdt_new[k] = np.append(rdt_a[k], rdt_b[k])
return build_granule(granule_a.data_producer_id, tt_a, rdt_new)
def process(self, packet):
if not isinstance(packet,Granule):
log.warn('Invalid packet received: Type "%s"' % type(packet))
return
rd_in = RecordDictionaryTool.load_from_granule(packet)
tt = TaxyTool.load_from_granule(packet)
rd_out = RecordDictionaryTool(tt)
for nickname, v_sequence in rd_in.iteritems():
rd_out[nickname] = self.shift(v_sequence)
g_out = build_granule(data_producer_id='dp_id',taxonomy=tt,record_dictionary=rd_out)
self.publish(g_out)
def combine_granules(granule_a, granule_b):
"""
This is a method that combines granules in a very naive manner
"""
validate_is_instance(granule_a, Granule,
'granule_a is not a proper Granule')
validate_is_instance(granule_b, Granule,
'granule_b is not a proper Granule')
tt_a = TaxyTool.load_from_granule(granule_a)
tt_b = TaxyTool.load_from_granule(granule_b)
if tt_a != tt_b:
raise BadRequest(
'Can\'t combine the two granules, they do not have the same taxonomy.'
)
rdt_new = RecordDictionaryTool(tt_a)
rdt_a = RecordDictionaryTool.load_from_granule(granule_a)
rdt_b = RecordDictionaryTool.load_from_granule(granule_b)
for k in rdt_a.iterkeys():
rdt_new[k] = np.append(rdt_a[k], rdt_b[k])
return build_granule(granule_a.data_producer_id, tt_a, rdt_new)
def get_visualization_image(self, data_product_id='', visualization_parameters={}, callback=''):
# Error check
if not data_product_id:
raise BadRequest("The data_product_id parameter is missing")
query = None
if visualization_parameters:
if visualization_parameters.has_key('query'):
query=visualization_parameters['query']
# get the dataset_id associated with the data_product. Need it to do the data retrieval
ds_ids,_ = self.rrclient.find_objects(data_product_id, PRED.hasDataset, RT.DataSet, True)
if ds_ids == None or len(ds_ids) == 0:
return None
# Ideally just need the latest granule to figure out the list of images
#replay_granule = self.data_retriever.retrieve(ds_ids[0],{'start_time':0,'end_time':2})
retrieved_granule = self.data_retriever.retrieve(ds_ids[0])
if retrieved_granule == None:
return None
mpl_transform = VizTransformMatplotlibGraphs()
mpl_data_granule = mpl_transform.execute(retrieved_granule)
#mpl_data_granule = self.data_retriever.retrieve(ds_ids[0], module="ion.processes.data.transforms.viz.matplotlib_graphs", cls="VizTransformMatplotlibGraphs")
#if not mpl_data_granule:
# return None
mpl_rdt = RecordDictionaryTool.load_from_granule(mpl_data_granule)
temp_mpl_graph_list = get_safe(mpl_rdt, "matplotlib_graphs")
mpl_graph = temp_mpl_graph_list[0]
# restructure the mpl graphs in to a simpler dict that will be passed through
ret_dict = {}
ret_dict['content_type'] = mpl_graph['content_type']
ret_dict['image_name'] = mpl_graph['image_name']
# reason for encoding as base64 string is otherwise message pack complains about the bit stream
ret_dict['image_obj'] = base64.encodestring(mpl_graph['image_obj'])
#ret_dict['image_obj'] = mpl_graph['image_obj']
if callback == '':
return ret_dict
else:
return callback + "(" + simplejson.dumps(ret_dict) + ")"
def test_combine_granule(self):
tt = TaxyTool()
tt.add_taxonomy_set('a')
rdt = RecordDictionaryTool(tt)
rdt['a'] = numpy.array([1,2,3])
granule1 = build_granule('test',tt,rdt)
rdt = RecordDictionaryTool(tt)
rdt['a'] = numpy.array([4,5,6])
granule2 = build_granule('test',tt,rdt)
granule3 = combine_granules(granule1,granule2)
rdt = RecordDictionaryTool.load_from_granule(granule3)
self.assertTrue(numpy.allclose(rdt['a'],numpy.array([1,2,3,4,5,6])))
def test_combine_granule(self):
tt = TaxyTool()
tt.add_taxonomy_set('a')
rdt = RecordDictionaryTool(tt)
rdt['a'] = numpy.array([1, 2, 3])
granule1 = build_granule('test', tt, rdt)
rdt = RecordDictionaryTool(tt)
rdt['a'] = numpy.array([4, 5, 6])
granule2 = build_granule('test', tt, rdt)
granule3 = combine_granules(granule1, granule2)
rdt = RecordDictionaryTool.load_from_granule(granule3)
self.assertTrue(
numpy.allclose(rdt['a'], numpy.array([1, 2, 3, 4, 5, 6])))
def _validate_mpl_graphs_results(self, results_stream_def, results):
cc = self.container
assertions = self.assertTrue
for g in results:
if isinstance(g,Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
#log.warn(tx.pretty_print())
#log.warn(rdt.pretty_print())
graphs = rdt['matplotlib_graphs']
for graph in graphs:
assertions(graph['viz_product_type'] == 'matplotlib_graphs' )
# check to see if the list (numpy array) contians actual images
assertions(imghdr.what(graph['image_name'], graph['image_obj']) == 'png')
return
def _validate_mpl_graphs_results(self, results_stream_def, results):
cc = self.container
assertions = self.assertTrue
for g in results:
if isinstance(g, Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
#log.warn(tx.pretty_print())
#log.warn(rdt.pretty_print())
graphs = rdt['matplotlib_graphs']
for graph in graphs:
assertions(
graph['viz_product_type'] == 'matplotlib_graphs')
# check to see if the list (numpy array) contians actual images
assertions(
imghdr.what(graph['image_name'], graph['image_obj']) ==
'png')
return
def message_received(granule, h):
rdt = RecordDictionaryTool.load_from_granule(granule)
log.warn('Logging Record Dictionary received in logger subscription \n%s', rdt.pretty_print())
def get_visualization_data(self, data_product_id='', visualization_parameters={}, callback=''):
"""Retrieves the data for the specified DP and sends a token back which can be checked in
a non-blocking fashion till data is ready
@param data_product_id str
@param visualization_parameters str
@param callback str
@retval jsonp_visualization_data str
@throws NotFound object with specified id, query does not exist
"""
# error check
if not data_product_id:
raise BadRequest("The data_product_id parameter is missing")
gdt = None
query = None
if visualization_parameters:
if visualization_parameters.has_key('query'):
query=visualization_parameters['query']
# get the dataset_id associated with the data_product. Need it to do the data retrieval
ds_ids,_ = self.rrclient.find_objects(data_product_id, PRED.hasDataset, RT.DataSet, True)
if ds_ids == None or len(ds_ids) == 0:
return None
# Ideally just need the latest granule to figure out the list of images
#replay_granule = self.data_retriever.retrieve(ds_ids[0],{'start_time':0,'end_time':2})
retrieved_granule = self.data_retriever.retrieve(ds_ids[0])
if retrieved_granule == None:
return None
# send the granule through the transform to get the google datatable
gdt_transform = VizTransformGoogleDT()
gdt_data_granule = gdt_transform.execute(retrieved_granule)
# send the granule through the transform to get the google datatable
#gdt_data_granule = self.data_retriever.retrieve(ds_ids[0], module="ion.processes.data.transforms.viz.google_dt", cls="VizTransformGoogleDT")
#if not gdt_data_granule:
# return None
gdt_rdt = RecordDictionaryTool.load_from_granule(gdt_data_granule)
gdt_components = get_safe(gdt_rdt, "google_dt_components")
temp_gdt_description = gdt_components[0]["data_description"]
temp_gdt_content = gdt_components[0]["data_content"]
# adjust the 'float' time to datetime in the content
gdt_description = [('time', 'datetime', 'time')]
gdt_content = []
for idx in range(1,len(temp_gdt_description)):
gdt_description.append(temp_gdt_description[idx])
for tempTuple in temp_gdt_content:
# sometimes there are inexplicable empty tuples in the content. Drop them
if tempTuple == [] or len(tempTuple) == 0:
continue
varTuple = []
varTuple.append(datetime.fromtimestamp(tempTuple[0]))
for idx in range(1,len(tempTuple)):
varTuple.append(tempTuple[idx])
gdt_content.append(varTuple)
# now generate the Google datatable out of the description and content
gdt = gviz_api.DataTable(gdt_description)
gdt.LoadData(gdt_content)
# return the json version of the table
if callback == '':
return gdt.ToJSonResponse()
else:
return callback + "(\"" + gdt.ToJSonResponse() + "\")"
def _process_visualization_message(self, messages):
gdt_description = None
gdt_content = []
viz_product_type = ''
for message in messages:
message_data = message.body
if isinstance(message_data,Granule):
tx = TaxyTool.load_from_granule(message_data)
rdt = RecordDictionaryTool.load_from_granule(message_data)
gdt_components = get_safe(rdt, 'google_dt_components')
# IF this granule does not contain google dt, skip
if gdt_components is None:
continue
gdt_component = gdt_components[0]
viz_product_type = gdt_component['viz_product_type']
# Process Google DataTable messages
if viz_product_type == 'google_dt':
# If the data description is being put together for the first time,
# switch the time format from float to datetime
if (gdt_description == None):
temp_gdt_description = gdt_component['data_description']
gdt_description = [('time', 'datetime', 'time')]
for idx in range(1,len(temp_gdt_description)):
# for some weird reason need to force convert to tuples
temp_arr = temp_gdt_description[idx]
if temp_arr != None:
gdt_description.append((temp_arr[0], temp_arr[1], temp_arr[2]))
# append all content to one big array
temp_gdt_content = gdt_component['data_content']
for tempTuple in temp_gdt_content:
# sometimes there are inexplicable empty tuples in the content. Drop them
if tempTuple == [] or len(tempTuple) == 0:
continue
varTuple = []
varTuple.append(datetime.fromtimestamp(tempTuple[0]))
for idx in range(1,len(tempTuple)):
varTuple.append(tempTuple[idx])
gdt_content.append(varTuple)
#TODO - what to do if this is not a valid visualization message?
# Now that all the messages have been parsed, any last processing should be done here
if viz_product_type == "google_dt":
# Using the description and content, build the google data table
gdt = gviz_api.DataTable(gdt_description)
gdt.LoadData(gdt_content)
return gdt.ToJSonResponse()
return None
