def _validate_google_dt_results(self, results_stream_def, results): cc = self.container assertions = self.assertTrue for g in results: if isinstance(g,Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) #log.warn(tx.pretty_print()) #log.warn(rdt.pretty_print()) gdt_component = rdt['google_dt_components'][0] assertions(gdt_component['viz_product_type'] == 'google_realtime_dt' ) gdt_description = gdt_component['data_table_description'] gdt_content = gdt_component['data_table_content'] assertions(gdt_description[0][0] == 'time') assertions(len(gdt_description) > 1) assertions(len(gdt_content) >= 0) return
def _validate_google_dt_results(self, results_stream_def, results): cc = self.container assertions = self.assertTrue for g in results: if isinstance(g, Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) #log.warn(tx.pretty_print()) #log.warn(rdt.pretty_print()) gdt_component = rdt['google_dt_components'][0] assertions( gdt_component['viz_product_type'] == 'google_realtime_dt') gdt_description = gdt_component['data_table_description'] gdt_content = gdt_component['data_table_content'] assertions(gdt_description[0][0] == 'time') assertions(len(gdt_description) > 1) assertions(len(gdt_content) >= 0) return
def test_build_granule_and_load_from_granule(self): #Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle tx = TaxyTool() tx.add_taxonomy_set('temp', 'long_temp_name') tx.add_taxonomy_set('cond', 'long_cond_name') tx.add_taxonomy_set('pres', 'long_pres_name') tx.add_taxonomy_set('rdt') # map is {<local name>: <granule name or path>} #Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect rdt = RecordDictionaryTool(taxonomy=tx) #Create some arrays and fill them with random values temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) #Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc. rdt['temp'] = temp_array rdt['cond'] = cond_array rdt['pres'] = pres_array #You can also add in another RecordDictionaryTool, providing the taxonomies are the same. rdt2 = RecordDictionaryTool(taxonomy=tx) rdt2['temp'] = temp_array rdt['rdt'] = rdt2 g = build_granule(data_producer_id='john', taxonomy=tx, record_dictionary=rdt) l_tx = TaxyTool.load_from_granule(g) l_rd = RecordDictionaryTool.load_from_granule(g) # Make sure we got back the same Taxonomy Object self.assertEquals(l_tx._t, tx._t) self.assertEquals(l_tx.get_handles('temp'), tx.get_handles('temp')) self.assertEquals(l_tx.get_handles('testing_2'), tx.get_handles('testing_2')) # Now test the record dictionary object self.assertEquals(l_rd._rd, rdt._rd) self.assertEquals(l_rd._tx._t, rdt._tx._t) for k, v in l_rd.iteritems(): self.assertIn(k, rdt) if isinstance(v, numpy.ndarray): self.assertTrue( (v == rdt[k]).all()) else: self.assertEquals(v._rd, rdt[k]._rd)
def test_build_granule_and_load_from_granule(self): #Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle tx = TaxyTool() tx.add_taxonomy_set('temp', 'long_temp_name') tx.add_taxonomy_set('cond', 'long_cond_name') tx.add_taxonomy_set('pres', 'long_pres_name') tx.add_taxonomy_set('rdt') # map is {<local name>: <granule name or path>} #Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect rdt = RecordDictionaryTool(taxonomy=tx) #Create some arrays and fill them with random values temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) #Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc. rdt['temp'] = temp_array rdt['cond'] = cond_array rdt['pres'] = pres_array #You can also add in another RecordDictionaryTool, providing the taxonomies are the same. rdt2 = RecordDictionaryTool(taxonomy=tx) rdt2['temp'] = temp_array rdt['rdt'] = rdt2 g = build_granule(data_producer_id='john', taxonomy=tx, record_dictionary=rdt) l_tx = TaxyTool.load_from_granule(g) l_rd = RecordDictionaryTool.load_from_granule(g) # Make sure we got back the same Taxonomy Object self.assertEquals(l_tx._t, tx._t) self.assertEquals(l_tx.get_handles('temp'), tx.get_handles('temp')) self.assertEquals(l_tx.get_handles('testing_2'), tx.get_handles('testing_2')) # Now test the record dictionary object self.assertEquals(l_rd._rd, rdt._rd) self.assertEquals(l_rd._tx._t, rdt._tx._t) for k, v in l_rd.iteritems(): self.assertIn(k, rdt) if isinstance(v, numpy.ndarray): self.assertTrue((v == rdt[k]).all()) else: self.assertEquals(v._rd, rdt[k]._rd)
def combine_granules(granule_a, granule_b): """ This is a method that combines granules in a very naive manner """ validate_is_instance(granule_a, Granule, "granule_a is not a proper Granule") validate_is_instance(granule_b, Granule, "granule_b is not a proper Granule") tt_a = TaxyTool.load_from_granule(granule_a) tt_b = TaxyTool.load_from_granule(granule_b) if tt_a != tt_b: raise BadRequest("Can't combine the two granules, they do not have the same taxonomy.") rdt_new = RecordDictionaryTool(tt_a) rdt_a = RecordDictionaryTool.load_from_granule(granule_a) rdt_b = RecordDictionaryTool.load_from_granule(granule_b) for k in rdt_a.iterkeys(): rdt_new[k] = np.append(rdt_a[k], rdt_b[k]) return build_granule(granule_a.data_producer_id, tt_a, rdt_new)
def process(self, packet): if not isinstance(packet,Granule): log.warn('Invalid packet received: Type "%s"' % type(packet)) return rd_in = RecordDictionaryTool.load_from_granule(packet) tt = TaxyTool.load_from_granule(packet) rd_out = RecordDictionaryTool(tt) for nickname, v_sequence in rd_in.iteritems(): rd_out[nickname] = self.shift(v_sequence) g_out = build_granule(data_producer_id='dp_id',taxonomy=tt,record_dictionary=rd_out) self.publish(g_out)
def combine_granules(granule_a, granule_b): """ This is a method that combines granules in a very naive manner """ validate_is_instance(granule_a, Granule, 'granule_a is not a proper Granule') validate_is_instance(granule_b, Granule, 'granule_b is not a proper Granule') tt_a = TaxyTool.load_from_granule(granule_a) tt_b = TaxyTool.load_from_granule(granule_b) if tt_a != tt_b: raise BadRequest( 'Can\'t combine the two granules, they do not have the same taxonomy.' ) rdt_new = RecordDictionaryTool(tt_a) rdt_a = RecordDictionaryTool.load_from_granule(granule_a) rdt_b = RecordDictionaryTool.load_from_granule(granule_b) for k in rdt_a.iterkeys(): rdt_new[k] = np.append(rdt_a[k], rdt_b[k]) return build_granule(granule_a.data_producer_id, tt_a, rdt_new)
def _validate_mpl_graphs_results(self, results_stream_def, results): cc = self.container assertions = self.assertTrue for g in results: if isinstance(g,Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) #log.warn(tx.pretty_print()) #log.warn(rdt.pretty_print()) graphs = rdt['matplotlib_graphs'] for graph in graphs: assertions(graph['viz_product_type'] == 'matplotlib_graphs' ) # check to see if the list (numpy array) contians actual images assertions(imghdr.what(graph['image_name'], graph['image_obj']) == 'png') return
def _validate_mpl_graphs_results(self, results_stream_def, results): cc = self.container assertions = self.assertTrue for g in results: if isinstance(g, Granule): tx = TaxyTool.load_from_granule(g) rdt = RecordDictionaryTool.load_from_granule(g) #log.warn(tx.pretty_print()) #log.warn(rdt.pretty_print()) graphs = rdt['matplotlib_graphs'] for graph in graphs: assertions( graph['viz_product_type'] == 'matplotlib_graphs') # check to see if the list (numpy array) contians actual images assertions( imghdr.what(graph['image_name'], graph['image_obj']) == 'png') return
def _process_visualization_message(self, messages): gdt_description = None gdt_content = [] viz_product_type = '' for message in messages: message_data = message.body if isinstance(message_data,Granule): tx = TaxyTool.load_from_granule(message_data) rdt = RecordDictionaryTool.load_from_granule(message_data) gdt_components = get_safe(rdt, 'google_dt_components') # IF this granule does not contain google dt, skip if gdt_components is None: continue gdt_component = gdt_components[0] viz_product_type = gdt_component['viz_product_type'] # Process Google DataTable messages if viz_product_type == 'google_dt': # If the data description is being put together for the first time, # switch the time format from float to datetime if (gdt_description == None): temp_gdt_description = gdt_component['data_description'] gdt_description = [('time', 'datetime', 'time')] for idx in range(1,len(temp_gdt_description)): # for some weird reason need to force convert to tuples temp_arr = temp_gdt_description[idx] if temp_arr != None: gdt_description.append((temp_arr[0], temp_arr[1], temp_arr[2])) # append all content to one big array temp_gdt_content = gdt_component['data_content'] for tempTuple in temp_gdt_content: # sometimes there are inexplicable empty tuples in the content. Drop them if tempTuple == [] or len(tempTuple) == 0: continue varTuple = [] varTuple.append(datetime.fromtimestamp(tempTuple[0])) for idx in range(1,len(tempTuple)): varTuple.append(tempTuple[idx]) gdt_content.append(varTuple) #TODO - what to do if this is not a valid visualization message? # Now that all the messages have been parsed, any last processing should be done here if viz_product_type == "google_dt": # Using the description and content, build the google data table gdt = gviz_api.DataTable(gdt_description) gdt.LoadData(gdt_content) return gdt.ToJSonResponse() return None