def _get_data(cls, config): parser = get_safe(config, 'parser', None) ext_dset_res = get_safe(config, 'external_dataset_res', None) if ext_dset_res and parser: #CBM: Not in use yet... # t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension'] # x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension'] # y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension'] # z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension'] # var_lst = ext_dset_res.dataset_description.parameters['variables'] max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id', 'unknown data producer') tx_yml = get_safe(config, 'taxonomy') ttool = TaxyTool.load( tx_yml ) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool cnt = cls._calc_iter_cnt(len(parser.sensor_map), max_rec) for x in xrange(cnt): rdt = RecordDictionaryTool(taxonomy=ttool) for name in parser.sensor_map: d = parser.data_map[name][x * max_rec:(x + 1) * max_rec] rdt[name] = d g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt) yield g else: log.warn('No parser object found in config')
def _get_data(cls, config): parser = get_safe(config, 'parser', None) ext_dset_res = get_safe(config, 'external_dataset_res', None) if ext_dset_res and parser: #CBM: Not in use yet... # t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension'] # x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension'] # y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension'] # z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension'] # var_lst = ext_dset_res.dataset_description.parameters['variables'] max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id', 'unknown data producer') tx_yml = get_safe(config, 'taxonomy') ttool = TaxyTool.load(tx_yml) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool cnt = cls._calc_iter_cnt(len(parser.sensor_map), max_rec) for x in xrange(cnt): rdt = RecordDictionaryTool(taxonomy=ttool) for name in parser.sensor_map: d = parser.data_map[name][x*max_rec:(x+1)*max_rec] rdt[name]=d g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt) yield g else: log.warn('No parser object found in config')
def _trigger_func(self, stream_id): #@todo - add lots of comments in here while True: length = 10 #Explicitly make these numpy arrays... c = numpy.array([random.uniform(0.0,75.0) for i in xrange(length)]) t = numpy.array([random.uniform(-1.7, 21.0) for i in xrange(length)]) p = numpy.array([random.lognormvariate(1,2) for i in xrange(length)]) lat = numpy.array([random.uniform(-90.0, 90.0) for i in xrange(length)]) lon = numpy.array([random.uniform(0.0, 360.0) for i in xrange(length)]) tvar = numpy.array([self.last_time + i for i in xrange(1,length+1)]) self.last_time = max(tvar) rdt = RecordDictionaryTool(taxonomy=tx) # This is an example of using groups it is not a normative statement about how to use groups rdt0 = RecordDictionaryTool(taxonomy=tx) rdt0['temp'] = t rdt0['cond'] = c rdt0['pres'] = p #add a value sequence of raw bytes - not sure the type below is correct? with open('/dev/urandom','r') as rand: rdt0['raw_fixed'] = numpy.array([rand.read(32) for i in xrange(length)], dtype='a32') #add a value sequence of raw bytes - not sure the type below is correct? with open('/dev/urandom','r') as rand: rdt0['raw_blob'] = numpy.array([rand.read(random.randint(1,40)) for i in xrange(length)], dtype=object) rdt1 = RecordDictionaryTool(taxonomy=tx) rdt1['time'] = tvar rdt1['lat'] = lat rdt1['lon'] = lon rdt['group1'] = rdt1 rdt['group0'] = rdt0 log.info("logging published Record Dictionary:\n %s", rdt.pretty_print()) g = build_granule(data_producer_id='Bobs Potatoes', taxonomy=tx, record_dictionary=rdt) log.info('Sending %d values!' % length) self.publisher.publish(g) time.sleep(2.0)
def test_combine_granule(self): tt = TaxyTool() tt.add_taxonomy_set('a') rdt = RecordDictionaryTool(tt) rdt['a'] = numpy.array([1,2,3]) granule1 = build_granule('test',tt,rdt) rdt = RecordDictionaryTool(tt) rdt['a'] = numpy.array([4,5,6]) granule2 = build_granule('test',tt,rdt) granule3 = combine_granules(granule1,granule2) rdt = RecordDictionaryTool.load_from_granule(granule3) self.assertTrue(numpy.allclose(rdt['a'],numpy.array([1,2,3,4,5,6])))
def test_build_granule_and_load_from_granule(self): #Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle tx = TaxyTool() tx.add_taxonomy_set('temp', 'long_temp_name') tx.add_taxonomy_set('cond', 'long_cond_name') tx.add_taxonomy_set('pres', 'long_pres_name') tx.add_taxonomy_set('rdt') # map is {<local name>: <granule name or path>} #Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect rdt = RecordDictionaryTool(taxonomy=tx) #Create some arrays and fill them with random values temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) #Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc. rdt['temp'] = temp_array rdt['cond'] = cond_array rdt['pres'] = pres_array #You can also add in another RecordDictionaryTool, providing the taxonomies are the same. rdt2 = RecordDictionaryTool(taxonomy=tx) rdt2['temp'] = temp_array rdt['rdt'] = rdt2 g = build_granule(data_producer_id='john', taxonomy=tx, record_dictionary=rdt) l_tx = TaxyTool.load_from_granule(g) l_rd = RecordDictionaryTool.load_from_granule(g) # Make sure we got back the same Taxonomy Object self.assertEquals(l_tx._t, tx._t) self.assertEquals(l_tx.get_handles('temp'), tx.get_handles('temp')) self.assertEquals(l_tx.get_handles('testing_2'), tx.get_handles('testing_2')) # Now test the record dictionary object self.assertEquals(l_rd._rd, rdt._rd) self.assertEquals(l_rd._tx._t, rdt._tx._t) for k, v in l_rd.iteritems(): self.assertIn(k, rdt) if isinstance(v, numpy.ndarray): self.assertTrue( (v == rdt[k]).all()) else: self.assertEquals(v._rd, rdt[k]._rd)
def test_combine_granule(self): tt = TaxyTool() tt.add_taxonomy_set('a') rdt = RecordDictionaryTool(tt) rdt['a'] = numpy.array([1, 2, 3]) granule1 = build_granule('test', tt, rdt) rdt = RecordDictionaryTool(tt) rdt['a'] = numpy.array([4, 5, 6]) granule2 = build_granule('test', tt, rdt) granule3 = combine_granules(granule1, granule2) rdt = RecordDictionaryTool.load_from_granule(granule3) self.assertTrue( numpy.allclose(rdt['a'], numpy.array([1, 2, 3, 4, 5, 6])))
def test_build_granule_and_load_from_granule(self): #Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle tx = TaxyTool() tx.add_taxonomy_set('temp', 'long_temp_name') tx.add_taxonomy_set('cond', 'long_cond_name') tx.add_taxonomy_set('pres', 'long_pres_name') tx.add_taxonomy_set('rdt') # map is {<local name>: <granule name or path>} #Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect rdt = RecordDictionaryTool(taxonomy=tx) #Create some arrays and fill them with random values temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) #Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc. rdt['temp'] = temp_array rdt['cond'] = cond_array rdt['pres'] = pres_array #You can also add in another RecordDictionaryTool, providing the taxonomies are the same. rdt2 = RecordDictionaryTool(taxonomy=tx) rdt2['temp'] = temp_array rdt['rdt'] = rdt2 g = build_granule(data_producer_id='john', taxonomy=tx, record_dictionary=rdt) l_tx = TaxyTool.load_from_granule(g) l_rd = RecordDictionaryTool.load_from_granule(g) # Make sure we got back the same Taxonomy Object self.assertEquals(l_tx._t, tx._t) self.assertEquals(l_tx.get_handles('temp'), tx.get_handles('temp')) self.assertEquals(l_tx.get_handles('testing_2'), tx.get_handles('testing_2')) # Now test the record dictionary object self.assertEquals(l_rd._rd, rdt._rd) self.assertEquals(l_rd._tx._t, rdt._tx._t) for k, v in l_rd.iteritems(): self.assertIn(k, rdt) if isinstance(v, numpy.ndarray): self.assertTrue((v == rdt[k]).all()) else: self.assertEquals(v._rd, rdt[k]._rd)
def render_graphs(self): # init Matplotlib fig = Figure() ax = fig.add_subplot(111) canvas = FigureCanvas(fig) imgInMem = StringIO.StringIO() # If there's no data, wait # For the simple case of testing, lets plot all time variant variables one at a time xAxisVar = "time" xAxisFloatData = self.graph_data[xAxisVar] rdt = RecordDictionaryTool(taxonomy=tx) rdt["matplotlib_graphs"] = array([]) for varName, varData in self.graph_data.iteritems(): if varName == "time" or varName == "height" or varName == "longitude" or varName == "latitude": continue yAxisVar = varName yAxisFloatData = self.graph_data[varName] # Generate the plot ax.plot(xAxisFloatData, yAxisFloatData, "ro") ax.set_xlabel(xAxisVar) ax.set_ylabel(yAxisVar) ax.set_title(yAxisVar + " vs " + xAxisVar) ax.set_autoscale_on(False) # generate filename for the output image fileName = yAxisVar + "_vs_" + xAxisVar + ".png" # Save the figure to the in memory file canvas.print_figure(imgInMem, format="png") imgInMem.seek(0) # submit resulting table back using the out stream publisher msg = { "viz_product_type": "matplotlib_graphs", "data_product_id": "FAKE_DATAPRODUCT_ID_0001", "image_obj": imgInMem.getvalue(), "image_name": fileName, } append(rdt["matplotlib_graphs"], msg) # clear the canvas for the next image ax.clear() # Generate a list of the graph objects generated self.out_granule = build_granule( data_producer_id="matplotlib_graphs_transform", taxonomy=tx, record_dictionary=rdt )
def receive_msg(self, msg, stream_id): if msg == {}: return rdt = RecordDictionaryTool.load_from_granule(msg) conductivity = get_safe(rdt, 'cond') #psd.get_values('conductivity') pressure = get_safe(rdt, 'pres') #psd.get_values('pressure') temperature = get_safe(rdt, 'temp') #psd.get_values('temperature') longitude = get_safe(rdt, 'lon') # psd.get_values('longitude') latitude = get_safe(rdt, 'lat') #psd.get_values('latitude') time = get_safe(rdt, 'time') rdt2 = RecordDictionaryTool(rdt._tx) rdt2['cond'] = ctd_L0_algorithm.execute(conductivity) rdt2['lat'] = latitude rdt2['lon'] = longitude rdt2['time'] = time g = build_granule(data_producer_id='ctd_L0_all', record_dictionary=rdt2, taxonomy=rdt2._tx) self.publish(msg=g, stream_id=self.cond_stream) rdt2 = RecordDictionaryTool(rdt._tx) rdt2['pres'] = ctd_L0_algorithm.execute(pressure) rdt2['lat'] = latitude rdt2['lon'] = longitude rdt2['time'] = time g = build_granule(data_producer_id='ctd_L0_all', record_dictionary=rdt2, taxonomy=rdt2._tx) self.publish(msg=g, stream_id=self.pres_stream) rdt2 = RecordDictionaryTool(rdt._tx) rdt2['temp'] = ctd_L0_algorithm.execute(temperature) rdt2['lat'] = latitude rdt2['lon'] = longitude rdt2['time'] = time g = build_granule(data_producer_id='ctd_L0_all', record_dictionary=rdt2, taxonomy=rdt2._tx) self.publish(msg=g, stream_id=self.temp_stream)
def granule_example(nsdict): """ Usage: from examples.granule import granule_example granule_example(locals()) tx, g, rdt, rdt2... etc, are now local variables in your shell! """ #Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle tx = TaxyTool() tx.add_taxonomy_set('temp', 'long_temp_name') tx.add_taxonomy_set('cond', 'long_cond_name') tx.add_taxonomy_set('pres', 'long_pres_name') tx.add_taxonomy_set('rdt') # map is {<local name>: <granule name or path>} #Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect rdt = RecordDictionaryTool(taxonomy=tx) #Create some arrays and fill them with random values temp_array = numpy.random.standard_normal(100) cond_array = numpy.random.standard_normal(100) pres_array = numpy.random.standard_normal(100) #Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc. rdt['temp'] = temp_array rdt['cond'] = cond_array rdt['pres'] = pres_array #You can also add in another RecordDictionaryTool, providing the taxonomies are the same. rdt2 = RecordDictionaryTool(taxonomy=tx) rdt2['temp'] = temp_array rdt['rdt'] = rdt2 #you can get a string representation of the RecordDictionaryTool print rdt print rdt.pretty_print() #Determine the length of the RecordDictionary using the len function print len(rdt) #Delete an item in the RecordDictionary del rdt['temp'] g = build_granule(data_producer_id='john', taxonomy=tx, record_dictionary=rdt) nsdict.update(locals())
def process(self, packet): if not isinstance(packet,Granule): log.warn('Invalid packet received: Type "%s"' % type(packet)) return rd_in = RecordDictionaryTool.load_from_granule(packet) tt = TaxyTool.load_from_granule(packet) rd_out = RecordDictionaryTool(tt) for nickname, v_sequence in rd_in.iteritems(): rd_out[nickname] = self.shift(v_sequence) g_out = build_granule(data_producer_id='dp_id',taxonomy=tt,record_dictionary=rd_out) self.publish(g_out)
def launch_benchmark(transform_number=1, primer=1, message_length=4): import gevent from gevent.greenlet import Greenlet from pyon.util.containers import DotDict from pyon.net.transport import NameTrio from pyon.net.endpoint import Publisher import numpy from pyon.ion.granule.record_dictionary import RecordDictionaryTool from pyon.ion.granule.taxonomy import TaxyTool from pyon.ion.granule.granule import build_granule tt = TaxyTool() tt.add_taxonomy_set('a') import uuid num = transform_number msg_len = message_length transforms = list() pids = 1 TransformBenchTesting.message_length = message_length cc = Container.instance pub = Publisher(to_name=NameTrio(get_sys_name(), str(uuid.uuid4())[0:6])) for i in xrange(num): tbt = cc.proc_manager._create_service_instance( str(pids), 'tbt', 'prototype.transforms.linear', 'TransformInPlaceNewGranule', DotDict({ 'process': { 'name': 'tbt%d' % pids, 'transform_id': pids } })) tbt.init() tbt.start() gevent.sleep(0.2) for i in xrange(primer): rd = RecordDictionaryTool(tt, message_length) rd['a'] = numpy.arange(message_length) gran = build_granule(data_producer_id='dp_id', taxonomy=tt, record_dictionary=rd) pub.publish(gran) g = Greenlet(tbt.perf) g.start() transforms.append(tbt) pids += 1
def _trigger_func(self, stream_id): self.last_time = 0 #@todo - add lots of comments in here while True: length = 10 #Explicitly make these numpy arrays... c = numpy.array([random.uniform(0.0,75.0) for i in xrange(length)]) t = numpy.array([random.uniform(-1.7, 21.0) for i in xrange(length)]) p = numpy.array([random.lognormvariate(1,2) for i in xrange(length)]) lat = numpy.array([random.uniform(-90.0, 90.0) for i in xrange(length)]) lon = numpy.array([random.uniform(0.0, 360.0) for i in xrange(length)]) tvar = numpy.array([self.last_time + i for i in xrange(1,length+1)]) self.last_time = max(tvar) rdt = RecordDictionaryTool(taxonomy=self._tx) rdt['temp'] = ExampleDataProducer_algorithm.execute(t) rdt['cond'] = ExampleDataProducer_algorithm.execute(c) rdt['pres'] = ExampleDataProducer_algorithm.execute(p) rdt['time'] = tvar rdt['lat'] = lat rdt['lon'] = lon log.info("logging published Record Dictionary:\n %s", rdt.pretty_print()) g = build_granule(data_producer_id=stream_id, taxonomy=self._tx, record_dictionary=rdt) log.info('Sending %d values!' % length) if(isinstance(g, Granule)): self.publish(g, stream_id) time.sleep(2.0)
def _get_data(cls, config): """ Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len'] @param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count'] """ array_len = get_safe(config, 'constraints.array_len',1) max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id') tx_yml = get_safe(config, 'taxonomy') ttool = TaxyTool.load(tx_yml) arr = npr.random_sample(array_len) log.debug('Array to send using max_rec={0}: {1}'.format(max_rec, arr)) cnt = cls._calc_iter_cnt(arr.size, max_rec) for x in xrange(cnt): rdt = RecordDictionaryTool(taxonomy=ttool) d = arr[x*max_rec:(x+1)*max_rec] rdt['data'] = d g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt) yield g
def _get_data(cls, config): """ A generator that retrieves config['constraints']['count'] number of sequential Fibonacci numbers @param config Dict of configuration parameters - must contain ['constraints']['count'] """ cnt = get_safe(config, 'constraints.count', 1) max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id') tx_yml = get_safe(config, 'taxonomy') ttool = TaxyTool.load(tx_yml) def fibGenerator(): """ A Fibonacci sequence generator """ count = 0 ret = [] a, b = 1, 1 while 1: count += 1 ret.append(a) if count == max_rec: yield np.array(ret) ret = [] count = 0 a, b = b, a + b gen = fibGenerator() cnt = cls._calc_iter_cnt(cnt, max_rec) for i in xrange(cnt): rdt = RecordDictionaryTool(taxonomy=ttool) d = gen.next() rdt['data'] = d g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt) yield g
def _get_data(cls, config): """ Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len'] @param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count'] """ array_len = get_safe(config, 'constraints.array_len', 1) max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id') tx_yml = get_safe(config, 'taxonomy') ttool = TaxyTool.load(tx_yml) arr = npr.random_sample(array_len) log.debug('Array to send using max_rec={0}: {1}'.format(max_rec, arr)) cnt = cls._calc_iter_cnt(arr.size, max_rec) for x in xrange(cnt): rdt = RecordDictionaryTool(taxonomy=ttool) d = arr[x * max_rec:(x + 1) * max_rec] rdt['data'] = d g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt) yield g
def launch_benchmark(transform_number=1, primer=1,message_length=4): import gevent from gevent.greenlet import Greenlet from pyon.util.containers import DotDict from pyon.net.transport import NameTrio from pyon.net.endpoint import Publisher import numpy from pyon.ion.granule.record_dictionary import RecordDictionaryTool from pyon.ion.granule.taxonomy import TaxyTool from pyon.ion.granule.granule import build_granule tt = TaxyTool() tt.add_taxonomy_set('a') import uuid num = transform_number msg_len = message_length transforms = list() pids = 1 TransformBenchTesting.message_length = message_length cc = Container.instance pub = Publisher(to_name=NameTrio(get_sys_name(),str(uuid.uuid4())[0:6])) for i in xrange(num): tbt=cc.proc_manager._create_service_instance(str(pids), 'tbt', 'prototype.transforms.linear', 'TransformInPlaceNewGranule', DotDict({'process':{'name':'tbt%d' % pids, 'transform_id':pids}})) tbt.init() tbt.start() gevent.sleep(0.2) for i in xrange(primer): rd = RecordDictionaryTool(tt, message_length) rd['a'] = numpy.arange(message_length) gran = build_granule(data_producer_id='dp_id',taxonomy=tt, record_dictionary=rd) pub.publish(gran) g = Greenlet(tbt.perf) g.start() transforms.append(tbt) pids += 1
def _get_data(cls, config): """ A generator that retrieves config['constraints']['count'] number of sequential Fibonacci numbers @param config Dict of configuration parameters - must contain ['constraints']['count'] """ cnt = get_safe(config,'constraints.count',1) max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id') tx_yml = get_safe(config, 'taxonomy') ttool = TaxyTool.load(tx_yml) def fibGenerator(): """ A Fibonacci sequence generator """ count = 0 ret = [] a, b = 1, 1 while 1: count += 1 ret.append(a) if count == max_rec: yield np.array(ret) ret=[] count = 0 a, b = b, a + b gen=fibGenerator() cnt = cls._calc_iter_cnt(cnt, max_rec) for i in xrange(cnt): rdt = RecordDictionaryTool(taxonomy=ttool) d = gen.next() rdt['data'] = d g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt) yield g
def _get_data(cls, config): """ Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len'] @param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count'] """ ext_dset_res = get_safe(config, 'external_dataset_res', None) # Get the Dataset object from the config (should have been instantiated in _init_acquisition_cycle) ds=get_safe(config, 'dataset_object') if ext_dset_res and ds: t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension'] x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension'] y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension'] z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension'] var_lst = ext_dset_res.dataset_description.parameters['variables'] t_slice = get_safe(config, 'constraints.temporal_slice', (slice(0,1))) #TODO: Using 'eval' here is BAD - need to find a less sketchy way to pass constraints if isinstance(t_slice,str): t_slice=eval(t_slice) lon = ds.variables[x_vname][:] lat = ds.variables[y_vname][:] z = ds.variables[z_vname][:] t_arr = ds.variables[t_vname][t_slice] data_arrays = {} for varn in var_lst: data_arrays[varn] = ds.variables[varn][t_slice] max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id', 'unknown data producer') tx_yml = get_safe(config, 'taxonomy') ttool = TaxyTool.load(tx_yml) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool cnt = cls._calc_iter_cnt(t_arr.size, max_rec) for x in xrange(cnt): ta = t_arr[x*max_rec:(x+1)*max_rec] # Make a 'master' RecDict rdt = RecordDictionaryTool(taxonomy=ttool) # Make a 'coordinate' RecDict rdt_c = RecordDictionaryTool(taxonomy=ttool) # Make a 'data' RecDict rdt_d = RecordDictionaryTool(taxonomy=ttool) # Assign values to the coordinate RecDict rdt_c[x_vname] = lon rdt_c[y_vname] = lat rdt_c[z_vname] = z # Assign values to the data RecDict rdt_d[t_vname] = ta for key, arr in data_arrays.iteritems(): d = arr[x*max_rec:(x+1)*max_rec] rdt_d[key] = d # Add the coordinate and data RecDicts to the master RecDict rdt['coords'] = rdt_c rdt['data'] = rdt_d # Build and return a granule # CBM: ttool must be passed g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt) yield g ds.close()
def execute(self, granule): log.debug('(Google DT transform): Received Viz Data Packet' ) self.dataDescription = [] self.dataTableContent = [] element_count_id = 0 expected_range = [] # NOTE : Detect somehow that this is a replay stream with a set number of expected granules. Based on this # calculate the number of expected records and set the self.realtime_window_size bigger or equal to this # number. psd = PointSupplementStreamParser(stream_definition=self.incoming_stream_def, stream_granule=granule) vardict = {} arrLen = None for varname in psd.list_field_names(): vardict[varname] = psd.get_values(varname) arrLen = len(vardict[varname]) #iinit the dataTable # create data description from the variables in the message self.dataDescription = [('time', 'datetime', 'time')] # split the data string to extract variable names for varname in psd.list_field_names(): if varname == 'time': continue self.dataDescription.append((varname, 'number', varname)) # Add the records to the datatable for i in xrange(arrLen): varTuple = [] for varname,_,_ in self.dataDescription: val = float(vardict[varname][i]) if varname == 'time': #varTuple.append(datetime.fromtimestamp(val)) varTuple.append(val) else: varTuple.append(val) # Append the tuples to the data table self.dataTableContent.append (varTuple) # Maintain a sliding window for realtime transform processes if len(self.dataTableContent) > self.realtime_window_size: # always pop the first element till window size is what we want while len(self.dataTableContent) > realtime_window_size: self.dataTableContent.pop(0) """ To Do : Do we need to figure out the how many granules have been received for a replay stream ?? if not self.realtime_flag: # This is the historical view part. Make a note of how many records were received in_data_stream_id = self.incoming_stream_def.data_stream_id element_count_id = self.incoming_stream_def.identifiables[in_data_stream_id].element_count_id # From each granule you can check the constraint on the number of records expected_range = granule.identifiables[element_count_id].constraint.intervals[0] # The number of records in a given packet is: self.total_num_of_records_recvd += packet.identifiables[element_count_id].value """ # define an output container of data # submit the partial datatable to the viz service rdt = RecordDictionaryTool(taxonomy=tx) # submit resulting table back using the out stream publisher. The data_product_id is the input dp_id # responsible for the incoming data msg = {"viz_product_type": "google_realtime_dt", "data_product_id": "FAKE_DATAPRODUCT_ID_0000", "data_table_description": self.dataDescription, "data_table_content": self.dataTableContent} rdt['google_dt_components'] = numpy.array([msg]) log.debug('Google DT transform: Sending a granule') out_granule = build_granule(data_producer_id='google_dt_transform', taxonomy=tx, record_dictionary=rdt) #self.publish(out_granule) # clear the tuple for future use self.varTuple[:] = [] return out_granule