def _get_data(cls, config):
parser = get_safe(config, 'parser', None)
ext_dset_res = get_safe(config, 'external_dataset_res', None)
if ext_dset_res and parser:
#CBM: Not in use yet...
# t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension']
# x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension']
# y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension']
# z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension']
# var_lst = ext_dset_res.dataset_description.parameters['variables']
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id',
'unknown data producer')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(
tx_yml
) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool
cnt = cls._calc_iter_cnt(len(parser.sensor_map), max_rec)
for x in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
for name in parser.sensor_map:
d = parser.data_map[name][x * max_rec:(x + 1) * max_rec]
rdt[name] = d
g = build_granule(data_producer_id=dprod_id,
taxonomy=ttool,
record_dictionary=rdt)
yield g
else:
log.warn('No parser object found in config')
def _get_data(cls, config):
parser = get_safe(config, 'parser', None)
ext_dset_res = get_safe(config, 'external_dataset_res', None)
if ext_dset_res and parser:
#CBM: Not in use yet...
# t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension']
# x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension']
# y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension']
# z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension']
# var_lst = ext_dset_res.dataset_description.parameters['variables']
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id', 'unknown data producer')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool
cnt = cls._calc_iter_cnt(len(parser.sensor_map), max_rec)
for x in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
for name in parser.sensor_map:
d = parser.data_map[name][x*max_rec:(x+1)*max_rec]
rdt[name]=d
g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt)
yield g
else:
log.warn('No parser object found in config')
def _trigger_func(self, stream_id):
#@todo - add lots of comments in here
while True:
length = 10
#Explicitly make these numpy arrays...
c = numpy.array([random.uniform(0.0,75.0) for i in xrange(length)])
t = numpy.array([random.uniform(-1.7, 21.0) for i in xrange(length)])
p = numpy.array([random.lognormvariate(1,2) for i in xrange(length)])
lat = numpy.array([random.uniform(-90.0, 90.0) for i in xrange(length)])
lon = numpy.array([random.uniform(0.0, 360.0) for i in xrange(length)])
tvar = numpy.array([self.last_time + i for i in xrange(1,length+1)])
self.last_time = max(tvar)
rdt = RecordDictionaryTool(taxonomy=tx)
# This is an example of using groups it is not a normative statement about how to use groups
rdt0 = RecordDictionaryTool(taxonomy=tx)
rdt0['temp'] = t
rdt0['cond'] = c
rdt0['pres'] = p
#add a value sequence of raw bytes - not sure the type below is correct?
with open('/dev/urandom','r') as rand:
rdt0['raw_fixed'] = numpy.array([rand.read(32) for i in xrange(length)], dtype='a32')
#add a value sequence of raw bytes - not sure the type below is correct?
with open('/dev/urandom','r') as rand:
rdt0['raw_blob'] = numpy.array([rand.read(random.randint(1,40)) for i in xrange(length)], dtype=object)
rdt1 = RecordDictionaryTool(taxonomy=tx)
rdt1['time'] = tvar
rdt1['lat'] = lat
rdt1['lon'] = lon
rdt['group1'] = rdt1
rdt['group0'] = rdt0
log.info("logging published Record Dictionary:\n %s", rdt.pretty_print())
g = build_granule(data_producer_id='Bobs Potatoes', taxonomy=tx, record_dictionary=rdt)
log.info('Sending %d values!' % length)
self.publisher.publish(g)
time.sleep(2.0)
def test_combine_granule(self):
tt = TaxyTool()
tt.add_taxonomy_set('a')
rdt = RecordDictionaryTool(tt)
rdt['a'] = numpy.array([1,2,3])
granule1 = build_granule('test',tt,rdt)
rdt = RecordDictionaryTool(tt)
rdt['a'] = numpy.array([4,5,6])
granule2 = build_granule('test',tt,rdt)
granule3 = combine_granules(granule1,granule2)
rdt = RecordDictionaryTool.load_from_granule(granule3)
self.assertTrue(numpy.allclose(rdt['a'],numpy.array([1,2,3,4,5,6])))
def test_build_granule_and_load_from_granule(self):
#Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle
tx = TaxyTool()
tx.add_taxonomy_set('temp', 'long_temp_name')
tx.add_taxonomy_set('cond', 'long_cond_name')
tx.add_taxonomy_set('pres', 'long_pres_name')
tx.add_taxonomy_set('rdt')
# map is {<local name>: <granule name or path>}
#Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect
rdt = RecordDictionaryTool(taxonomy=tx)
#Create some arrays and fill them with random values
temp_array = numpy.random.standard_normal(100)
cond_array = numpy.random.standard_normal(100)
pres_array = numpy.random.standard_normal(100)
#Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc.
rdt['temp'] = temp_array
rdt['cond'] = cond_array
rdt['pres'] = pres_array
#You can also add in another RecordDictionaryTool, providing the taxonomies are the same.
rdt2 = RecordDictionaryTool(taxonomy=tx)
rdt2['temp'] = temp_array
rdt['rdt'] = rdt2
g = build_granule(data_producer_id='john', taxonomy=tx, record_dictionary=rdt)
l_tx = TaxyTool.load_from_granule(g)
l_rd = RecordDictionaryTool.load_from_granule(g)
# Make sure we got back the same Taxonomy Object
self.assertEquals(l_tx._t, tx._t)
self.assertEquals(l_tx.get_handles('temp'), tx.get_handles('temp'))
self.assertEquals(l_tx.get_handles('testing_2'), tx.get_handles('testing_2'))
# Now test the record dictionary object
self.assertEquals(l_rd._rd, rdt._rd)
self.assertEquals(l_rd._tx._t, rdt._tx._t)
for k, v in l_rd.iteritems():
self.assertIn(k, rdt)
if isinstance(v, numpy.ndarray):
self.assertTrue( (v == rdt[k]).all())
else:
self.assertEquals(v._rd, rdt[k]._rd)
def test_combine_granule(self):
tt = TaxyTool()
tt.add_taxonomy_set('a')
rdt = RecordDictionaryTool(tt)
rdt['a'] = numpy.array([1, 2, 3])
granule1 = build_granule('test', tt, rdt)
rdt = RecordDictionaryTool(tt)
rdt['a'] = numpy.array([4, 5, 6])
granule2 = build_granule('test', tt, rdt)
granule3 = combine_granules(granule1, granule2)
rdt = RecordDictionaryTool.load_from_granule(granule3)
self.assertTrue(
numpy.allclose(rdt['a'], numpy.array([1, 2, 3, 4, 5, 6])))
def test_build_granule_and_load_from_granule(self):
#Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle
tx = TaxyTool()
tx.add_taxonomy_set('temp', 'long_temp_name')
tx.add_taxonomy_set('cond', 'long_cond_name')
tx.add_taxonomy_set('pres', 'long_pres_name')
tx.add_taxonomy_set('rdt')
# map is {<local name>: <granule name or path>}
#Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect
rdt = RecordDictionaryTool(taxonomy=tx)
#Create some arrays and fill them with random values
temp_array = numpy.random.standard_normal(100)
cond_array = numpy.random.standard_normal(100)
pres_array = numpy.random.standard_normal(100)
#Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc.
rdt['temp'] = temp_array
rdt['cond'] = cond_array
rdt['pres'] = pres_array
#You can also add in another RecordDictionaryTool, providing the taxonomies are the same.
rdt2 = RecordDictionaryTool(taxonomy=tx)
rdt2['temp'] = temp_array
rdt['rdt'] = rdt2
g = build_granule(data_producer_id='john',
taxonomy=tx,
record_dictionary=rdt)
l_tx = TaxyTool.load_from_granule(g)
l_rd = RecordDictionaryTool.load_from_granule(g)
# Make sure we got back the same Taxonomy Object
self.assertEquals(l_tx._t, tx._t)
self.assertEquals(l_tx.get_handles('temp'), tx.get_handles('temp'))
self.assertEquals(l_tx.get_handles('testing_2'),
tx.get_handles('testing_2'))
# Now test the record dictionary object
self.assertEquals(l_rd._rd, rdt._rd)
self.assertEquals(l_rd._tx._t, rdt._tx._t)
for k, v in l_rd.iteritems():
self.assertIn(k, rdt)
if isinstance(v, numpy.ndarray):
self.assertTrue((v == rdt[k]).all())
else:
self.assertEquals(v._rd, rdt[k]._rd)
def render_graphs(self):
# init Matplotlib
fig = Figure()
ax = fig.add_subplot(111)
canvas = FigureCanvas(fig)
imgInMem = StringIO.StringIO()
# If there's no data, wait
# For the simple case of testing, lets plot all time variant variables one at a time
xAxisVar = "time"
xAxisFloatData = self.graph_data[xAxisVar]
rdt = RecordDictionaryTool(taxonomy=tx)
rdt["matplotlib_graphs"] = array([])
for varName, varData in self.graph_data.iteritems():
if varName == "time" or varName == "height" or varName == "longitude" or varName == "latitude":
continue
yAxisVar = varName
yAxisFloatData = self.graph_data[varName]
# Generate the plot
ax.plot(xAxisFloatData, yAxisFloatData, "ro")
ax.set_xlabel(xAxisVar)
ax.set_ylabel(yAxisVar)
ax.set_title(yAxisVar + " vs " + xAxisVar)
ax.set_autoscale_on(False)
# generate filename for the output image
fileName = yAxisVar + "_vs_" + xAxisVar + ".png"
# Save the figure to the in memory file
canvas.print_figure(imgInMem, format="png")
imgInMem.seek(0)
# submit resulting table back using the out stream publisher
msg = {
"viz_product_type": "matplotlib_graphs",
"data_product_id": "FAKE_DATAPRODUCT_ID_0001",
"image_obj": imgInMem.getvalue(),
"image_name": fileName,
}
append(rdt["matplotlib_graphs"], msg)
# clear the canvas for the next image
ax.clear()
# Generate a list of the graph objects generated
self.out_granule = build_granule(
data_producer_id="matplotlib_graphs_transform", taxonomy=tx, record_dictionary=rdt
)
def receive_msg(self, msg, stream_id):
if msg == {}:
return
rdt = RecordDictionaryTool.load_from_granule(msg)
conductivity = get_safe(rdt, 'cond') #psd.get_values('conductivity')
pressure = get_safe(rdt, 'pres') #psd.get_values('pressure')
temperature = get_safe(rdt, 'temp') #psd.get_values('temperature')
longitude = get_safe(rdt, 'lon') # psd.get_values('longitude')
latitude = get_safe(rdt, 'lat') #psd.get_values('latitude')
time = get_safe(rdt, 'time')
rdt2 = RecordDictionaryTool(rdt._tx)
rdt2['cond'] = ctd_L0_algorithm.execute(conductivity)
rdt2['lat'] = latitude
rdt2['lon'] = longitude
rdt2['time'] = time
g = build_granule(data_producer_id='ctd_L0_all', record_dictionary=rdt2, taxonomy=rdt2._tx)
self.publish(msg=g, stream_id=self.cond_stream)
rdt2 = RecordDictionaryTool(rdt._tx)
rdt2['pres'] = ctd_L0_algorithm.execute(pressure)
rdt2['lat'] = latitude
rdt2['lon'] = longitude
rdt2['time'] = time
g = build_granule(data_producer_id='ctd_L0_all', record_dictionary=rdt2, taxonomy=rdt2._tx)
self.publish(msg=g, stream_id=self.pres_stream)
rdt2 = RecordDictionaryTool(rdt._tx)
rdt2['temp'] = ctd_L0_algorithm.execute(temperature)
rdt2['lat'] = latitude
rdt2['lon'] = longitude
rdt2['time'] = time
g = build_granule(data_producer_id='ctd_L0_all', record_dictionary=rdt2, taxonomy=rdt2._tx)
self.publish(msg=g, stream_id=self.temp_stream)
def granule_example(nsdict):
"""
Usage:
from examples.granule import granule_example
granule_example(locals())
tx, g, rdt, rdt2... etc, are now local variables in your shell!
"""
#Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle
tx = TaxyTool()
tx.add_taxonomy_set('temp', 'long_temp_name')
tx.add_taxonomy_set('cond', 'long_cond_name')
tx.add_taxonomy_set('pres', 'long_pres_name')
tx.add_taxonomy_set('rdt')
# map is {<local name>: <granule name or path>}
#Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect
rdt = RecordDictionaryTool(taxonomy=tx)
#Create some arrays and fill them with random values
temp_array = numpy.random.standard_normal(100)
cond_array = numpy.random.standard_normal(100)
pres_array = numpy.random.standard_normal(100)
#Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc.
rdt['temp'] = temp_array
rdt['cond'] = cond_array
rdt['pres'] = pres_array
#You can also add in another RecordDictionaryTool, providing the taxonomies are the same.
rdt2 = RecordDictionaryTool(taxonomy=tx)
rdt2['temp'] = temp_array
rdt['rdt'] = rdt2
#you can get a string representation of the RecordDictionaryTool
print rdt
print rdt.pretty_print()
#Determine the length of the RecordDictionary using the len function
print len(rdt)
#Delete an item in the RecordDictionary
del rdt['temp']
g = build_granule(data_producer_id='john', taxonomy=tx, record_dictionary=rdt)
nsdict.update(locals())
def process(self, packet):
if not isinstance(packet,Granule):
log.warn('Invalid packet received: Type "%s"' % type(packet))
return
rd_in = RecordDictionaryTool.load_from_granule(packet)
tt = TaxyTool.load_from_granule(packet)
rd_out = RecordDictionaryTool(tt)
for nickname, v_sequence in rd_in.iteritems():
rd_out[nickname] = self.shift(v_sequence)
g_out = build_granule(data_producer_id='dp_id',taxonomy=tt,record_dictionary=rd_out)
self.publish(g_out)
def launch_benchmark(transform_number=1, primer=1, message_length=4):
import gevent
from gevent.greenlet import Greenlet
from pyon.util.containers import DotDict
from pyon.net.transport import NameTrio
from pyon.net.endpoint import Publisher
import numpy
from pyon.ion.granule.record_dictionary import RecordDictionaryTool
from pyon.ion.granule.taxonomy import TaxyTool
from pyon.ion.granule.granule import build_granule
tt = TaxyTool()
tt.add_taxonomy_set('a')
import uuid
num = transform_number
msg_len = message_length
transforms = list()
pids = 1
TransformBenchTesting.message_length = message_length
cc = Container.instance
pub = Publisher(to_name=NameTrio(get_sys_name(),
str(uuid.uuid4())[0:6]))
for i in xrange(num):
tbt = cc.proc_manager._create_service_instance(
str(pids), 'tbt', 'prototype.transforms.linear',
'TransformInPlaceNewGranule',
DotDict({
'process': {
'name': 'tbt%d' % pids,
'transform_id': pids
}
}))
tbt.init()
tbt.start()
gevent.sleep(0.2)
for i in xrange(primer):
rd = RecordDictionaryTool(tt, message_length)
rd['a'] = numpy.arange(message_length)
gran = build_granule(data_producer_id='dp_id',
taxonomy=tt,
record_dictionary=rd)
pub.publish(gran)
g = Greenlet(tbt.perf)
g.start()
transforms.append(tbt)
pids += 1
def _trigger_func(self, stream_id):
self.last_time = 0
#@todo - add lots of comments in here
while True:
length = 10
#Explicitly make these numpy arrays...
c = numpy.array([random.uniform(0.0,75.0) for i in xrange(length)])
t = numpy.array([random.uniform(-1.7, 21.0) for i in xrange(length)])
p = numpy.array([random.lognormvariate(1,2) for i in xrange(length)])
lat = numpy.array([random.uniform(-90.0, 90.0) for i in xrange(length)])
lon = numpy.array([random.uniform(0.0, 360.0) for i in xrange(length)])
tvar = numpy.array([self.last_time + i for i in xrange(1,length+1)])
self.last_time = max(tvar)
rdt = RecordDictionaryTool(taxonomy=self._tx)
rdt['temp'] = ExampleDataProducer_algorithm.execute(t)
rdt['cond'] = ExampleDataProducer_algorithm.execute(c)
rdt['pres'] = ExampleDataProducer_algorithm.execute(p)
rdt['time'] = tvar
rdt['lat'] = lat
rdt['lon'] = lon
log.info("logging published Record Dictionary:\n %s", rdt.pretty_print())
g = build_granule(data_producer_id=stream_id, taxonomy=self._tx, record_dictionary=rdt)
log.info('Sending %d values!' % length)
if(isinstance(g, Granule)):
self.publish(g, stream_id)
time.sleep(2.0)
def _get_data(cls, config):
"""
Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len']
@param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count']
"""
array_len = get_safe(config, 'constraints.array_len',1)
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml)
arr = npr.random_sample(array_len)
log.debug('Array to send using max_rec={0}: {1}'.format(max_rec, arr))
cnt = cls._calc_iter_cnt(arr.size, max_rec)
for x in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
d = arr[x*max_rec:(x+1)*max_rec]
rdt['data'] = d
g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt)
yield g
def _get_data(cls, config):
"""
A generator that retrieves config['constraints']['count'] number of sequential Fibonacci numbers
@param config Dict of configuration parameters - must contain ['constraints']['count']
"""
cnt = get_safe(config, 'constraints.count', 1)
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml)
def fibGenerator():
"""
A Fibonacci sequence generator
"""
count = 0
ret = []
a, b = 1, 1
while 1:
count += 1
ret.append(a)
if count == max_rec:
yield np.array(ret)
ret = []
count = 0
a, b = b, a + b
gen = fibGenerator()
cnt = cls._calc_iter_cnt(cnt, max_rec)
for i in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
d = gen.next()
rdt['data'] = d
g = build_granule(data_producer_id=dprod_id,
taxonomy=ttool,
record_dictionary=rdt)
yield g
def _get_data(cls, config):
"""
Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len']
@param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count']
"""
array_len = get_safe(config, 'constraints.array_len', 1)
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml)
arr = npr.random_sample(array_len)
log.debug('Array to send using max_rec={0}: {1}'.format(max_rec, arr))
cnt = cls._calc_iter_cnt(arr.size, max_rec)
for x in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
d = arr[x * max_rec:(x + 1) * max_rec]
rdt['data'] = d
g = build_granule(data_producer_id=dprod_id,
taxonomy=ttool,
record_dictionary=rdt)
yield g
def launch_benchmark(transform_number=1, primer=1,message_length=4):
import gevent
from gevent.greenlet import Greenlet
from pyon.util.containers import DotDict
from pyon.net.transport import NameTrio
from pyon.net.endpoint import Publisher
import numpy
from pyon.ion.granule.record_dictionary import RecordDictionaryTool
from pyon.ion.granule.taxonomy import TaxyTool
from pyon.ion.granule.granule import build_granule
tt = TaxyTool()
tt.add_taxonomy_set('a')
import uuid
num = transform_number
msg_len = message_length
transforms = list()
pids = 1
TransformBenchTesting.message_length = message_length
cc = Container.instance
pub = Publisher(to_name=NameTrio(get_sys_name(),str(uuid.uuid4())[0:6]))
for i in xrange(num):
tbt=cc.proc_manager._create_service_instance(str(pids), 'tbt', 'prototype.transforms.linear', 'TransformInPlaceNewGranule', DotDict({'process':{'name':'tbt%d' % pids, 'transform_id':pids}}))
tbt.init()
tbt.start()
gevent.sleep(0.2)
for i in xrange(primer):
rd = RecordDictionaryTool(tt, message_length)
rd['a'] = numpy.arange(message_length)
gran = build_granule(data_producer_id='dp_id',taxonomy=tt, record_dictionary=rd)
pub.publish(gran)
g = Greenlet(tbt.perf)
g.start()
transforms.append(tbt)
pids += 1
def _get_data(cls, config):
"""
A generator that retrieves config['constraints']['count'] number of sequential Fibonacci numbers
@param config Dict of configuration parameters - must contain ['constraints']['count']
"""
cnt = get_safe(config,'constraints.count',1)
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml)
def fibGenerator():
"""
A Fibonacci sequence generator
"""
count = 0
ret = []
a, b = 1, 1
while 1:
count += 1
ret.append(a)
if count == max_rec:
yield np.array(ret)
ret=[]
count = 0
a, b = b, a + b
gen=fibGenerator()
cnt = cls._calc_iter_cnt(cnt, max_rec)
for i in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
d = gen.next()
rdt['data'] = d
g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt)
yield g
def _get_data(cls, config):
"""
Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len']
@param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count']
"""
ext_dset_res = get_safe(config, 'external_dataset_res', None)
# Get the Dataset object from the config (should have been instantiated in _init_acquisition_cycle)
ds=get_safe(config, 'dataset_object')
if ext_dset_res and ds:
t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension']
x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension']
y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension']
z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension']
var_lst = ext_dset_res.dataset_description.parameters['variables']
t_slice = get_safe(config, 'constraints.temporal_slice', (slice(0,1)))
#TODO: Using 'eval' here is BAD - need to find a less sketchy way to pass constraints
if isinstance(t_slice,str):
t_slice=eval(t_slice)
lon = ds.variables[x_vname][:]
lat = ds.variables[y_vname][:]
z = ds.variables[z_vname][:]
t_arr = ds.variables[t_vname][t_slice]
data_arrays = {}
for varn in var_lst:
data_arrays[varn] = ds.variables[varn][t_slice]
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id', 'unknown data producer')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool
cnt = cls._calc_iter_cnt(t_arr.size, max_rec)
for x in xrange(cnt):
ta = t_arr[x*max_rec:(x+1)*max_rec]
# Make a 'master' RecDict
rdt = RecordDictionaryTool(taxonomy=ttool)
# Make a 'coordinate' RecDict
rdt_c = RecordDictionaryTool(taxonomy=ttool)
# Make a 'data' RecDict
rdt_d = RecordDictionaryTool(taxonomy=ttool)
# Assign values to the coordinate RecDict
rdt_c[x_vname] = lon
rdt_c[y_vname] = lat
rdt_c[z_vname] = z
# Assign values to the data RecDict
rdt_d[t_vname] = ta
for key, arr in data_arrays.iteritems():
d = arr[x*max_rec:(x+1)*max_rec]
rdt_d[key] = d
# Add the coordinate and data RecDicts to the master RecDict
rdt['coords'] = rdt_c
rdt['data'] = rdt_d
# Build and return a granule
# CBM: ttool must be passed
g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt)
yield g
ds.close()
def execute(self, granule):
log.debug('(Google DT transform): Received Viz Data Packet' )
self.dataDescription = []
self.dataTableContent = []
element_count_id = 0
expected_range = []
# NOTE : Detect somehow that this is a replay stream with a set number of expected granules. Based on this
# calculate the number of expected records and set the self.realtime_window_size bigger or equal to this
# number.
psd = PointSupplementStreamParser(stream_definition=self.incoming_stream_def, stream_granule=granule)
vardict = {}
arrLen = None
for varname in psd.list_field_names():
vardict[varname] = psd.get_values(varname)
arrLen = len(vardict[varname])
#iinit the dataTable
# create data description from the variables in the message
self.dataDescription = [('time', 'datetime', 'time')]
# split the data string to extract variable names
for varname in psd.list_field_names():
if varname == 'time':
continue
self.dataDescription.append((varname, 'number', varname))
# Add the records to the datatable
for i in xrange(arrLen):
varTuple = []
for varname,_,_ in self.dataDescription:
val = float(vardict[varname][i])
if varname == 'time':
#varTuple.append(datetime.fromtimestamp(val))
varTuple.append(val)
else:
varTuple.append(val)
# Append the tuples to the data table
self.dataTableContent.append (varTuple)
# Maintain a sliding window for realtime transform processes
if len(self.dataTableContent) > self.realtime_window_size:
# always pop the first element till window size is what we want
while len(self.dataTableContent) > realtime_window_size:
self.dataTableContent.pop(0)
""" To Do : Do we need to figure out the how many granules have been received for a replay stream ??
if not self.realtime_flag:
# This is the historical view part. Make a note of how many records were received
in_data_stream_id = self.incoming_stream_def.data_stream_id
element_count_id = self.incoming_stream_def.identifiables[in_data_stream_id].element_count_id
# From each granule you can check the constraint on the number of records
expected_range = granule.identifiables[element_count_id].constraint.intervals[0]
# The number of records in a given packet is:
self.total_num_of_records_recvd += packet.identifiables[element_count_id].value
"""
# define an output container of data
# submit the partial datatable to the viz service
rdt = RecordDictionaryTool(taxonomy=tx)
# submit resulting table back using the out stream publisher. The data_product_id is the input dp_id
# responsible for the incoming data
msg = {"viz_product_type": "google_realtime_dt",
"data_product_id": "FAKE_DATAPRODUCT_ID_0000",
"data_table_description": self.dataDescription,
"data_table_content": self.dataTableContent}
rdt['google_dt_components'] = numpy.array([msg])
log.debug('Google DT transform: Sending a granule')
out_granule = build_granule(data_producer_id='google_dt_transform', taxonomy=tx, record_dictionary=rdt)
#self.publish(out_granule)
# clear the tuple for future use
self.varTuple[:] = []
return out_granule
