def __init__(self, taxonomy, shape=None):
"""
@brief Initialize a new instance of Record Dictionary Tool with a taxonomy and an optional fixed length
@param taxonomy is an instance of a TaxonomyTool or Taxonomy (IonObject) used in this record dictionary
@param length is an optional fixed length for the value sequences of this record dictionary
"""
if not isinstance(shape, (_NoneType, int, tuple)):
raise TypeError(
'Invalid shape argument, received type "%s"; should be None or int or tuple'
% type(shape))
self._rd = {}
self._shp = shape
if isinstance(self._shp, int):
self._shp = (self._shp, )
# hold onto the taxonomy - we need it to build the granule...
if isinstance(taxonomy, TaxyTool):
self._tx = taxonomy
elif isinstance(taxonomy, Taxonomy):
self._tx = TaxyTool(taxonomy)
else:
raise TypeError(
'Invalid taxonomy argument, received type "%s"; should be Taxonomy or TaxyTool'
% type(taxonomy))
def test_build_granule_and_load_from_granule(self):
#Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle
tx = TaxyTool()
tx.add_taxonomy_set('temp', 'long_temp_name')
tx.add_taxonomy_set('cond', 'long_cond_name')
tx.add_taxonomy_set('pres', 'long_pres_name')
tx.add_taxonomy_set('rdt')
# map is {<local name>: <granule name or path>}
#Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect
rdt = RecordDictionaryTool(taxonomy=tx)
#Create some arrays and fill them with random values
temp_array = numpy.random.standard_normal(100)
cond_array = numpy.random.standard_normal(100)
pres_array = numpy.random.standard_normal(100)
#Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc.
rdt['temp'] = temp_array
rdt['cond'] = cond_array
rdt['pres'] = pres_array
#You can also add in another RecordDictionaryTool, providing the taxonomies are the same.
rdt2 = RecordDictionaryTool(taxonomy=tx)
rdt2['temp'] = temp_array
rdt['rdt'] = rdt2
g = build_granule(data_producer_id='john', taxonomy=tx, record_dictionary=rdt)
l_tx = TaxyTool.load_from_granule(g)
l_rd = RecordDictionaryTool.load_from_granule(g)
# Make sure we got back the same Taxonomy Object
self.assertEquals(l_tx._t, tx._t)
self.assertEquals(l_tx.get_handles('temp'), tx.get_handles('temp'))
self.assertEquals(l_tx.get_handles('testing_2'), tx.get_handles('testing_2'))
# Now test the record dictionary object
self.assertEquals(l_rd._rd, rdt._rd)
self.assertEquals(l_rd._tx._t, rdt._tx._t)
for k, v in l_rd.iteritems():
self.assertIn(k, rdt)
if isinstance(v, numpy.ndarray):
self.assertTrue( (v == rdt[k]).all())
else:
self.assertEquals(v._rd, rdt[k]._rd)
def _setup_resources(self):
stream_id = self.create_stream_and_logger(name='fibonacci_stream')
tx = TaxyTool()
tx.add_taxonomy_set('data', 'external_data')
#TG: Build TaxonomyTool & add to dh_cfg.taxonomy
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id': stream_id,
'data_producer_id': 'fibonacci_data_producer_id',
'taxonomy': tx.dump(),
'max_records': 4,
}
def _setup_resources(self):
stream_id = self.create_stream_and_logger(name='fibonacci_stream')
tx = TaxyTool()
tx.add_taxonomy_set('data', 'external_data')
#TG: Build TaxonomyTool & add to dh_cfg.taxonomy
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,
'data_producer_id':'fibonacci_data_producer_id',
'taxonomy':tx.dump(),
'max_records':4,
}
def _setup_resources(self):
stream_id = self.create_stream_and_logger(name='dummydata_stream')
tx = TaxyTool()
tx.add_taxonomy_set('data', 'external_data')
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,#TODO: This should probably be a 'stream_config' dict with stream_name:stream_id members
'data_producer_id':'dummy_data_producer_id',
'taxonomy':tx.dump(),
'max_records':4,
}
def _setup_resources(self):
stream_id = self.create_stream_and_logger(name='dummydata_stream')
tx = TaxyTool()
tx.add_taxonomy_set('data', 'external_data')
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id':
stream_id, #TODO: This should probably be a 'stream_config' dict with stream_name:stream_id members
'data_producer_id': 'dummy_data_producer_id',
'taxonomy': tx.dump(),
'max_records': 4,
}
def launch_benchmark(transform_number=1, primer=1, message_length=4):
import gevent
from gevent.greenlet import Greenlet
from pyon.util.containers import DotDict
from pyon.net.transport import NameTrio
from pyon.net.endpoint import Publisher
import numpy
from pyon.ion.granule.record_dictionary import RecordDictionaryTool
from pyon.ion.granule.taxonomy import TaxyTool
from pyon.ion.granule.granule import build_granule
tt = TaxyTool()
tt.add_taxonomy_set('a')
import uuid
num = transform_number
msg_len = message_length
transforms = list()
pids = 1
TransformBenchTesting.message_length = message_length
cc = Container.instance
pub = Publisher(to_name=NameTrio(get_sys_name(),
str(uuid.uuid4())[0:6]))
for i in xrange(num):
tbt = cc.proc_manager._create_service_instance(
str(pids), 'tbt', 'prototype.transforms.linear',
'TransformInPlaceNewGranule',
DotDict({
'process': {
'name': 'tbt%d' % pids,
'transform_id': pids
}
}))
tbt.init()
tbt.start()
gevent.sleep(0.2)
for i in xrange(primer):
rd = RecordDictionaryTool(tt, message_length)
rd['a'] = numpy.arange(message_length)
gran = build_granule(data_producer_id='dp_id',
taxonomy=tt,
record_dictionary=rd)
pub.publish(gran)
g = Greenlet(tbt.perf)
g.start()
transforms.append(tbt)
pids += 1
def load_from_granule(cls, g):
"""
@brief return an instance of Record Dictionary Tool from a granule. Used when a granule is received in a message
"""
result = cls(TaxyTool(g.taxonomy))
result._rd = g.record_dictionary
return result
def _get_data(cls, config):
parser = get_safe(config, 'parser', None)
ext_dset_res = get_safe(config, 'external_dataset_res', None)
if ext_dset_res and parser:
#CBM: Not in use yet...
# t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension']
# x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension']
# y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension']
# z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension']
# var_lst = ext_dset_res.dataset_description.parameters['variables']
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id', 'unknown data producer')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool
cnt = cls._calc_iter_cnt(len(parser.sensor_map), max_rec)
for x in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
for name in parser.sensor_map:
d = parser.data_map[name][x*max_rec:(x+1)*max_rec]
rdt[name]=d
g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt)
yield g
else:
log.warn('No parser object found in config')
def _validate_google_dt_results(self, results_stream_def, results):
cc = self.container
assertions = self.assertTrue
for g in results:
if isinstance(g,Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
#log.warn(tx.pretty_print())
#log.warn(rdt.pretty_print())
gdt_component = rdt['google_dt_components'][0]
assertions(gdt_component['viz_product_type'] == 'google_realtime_dt' )
gdt_description = gdt_component['data_table_description']
gdt_content = gdt_component['data_table_content']
assertions(gdt_description[0][0] == 'time')
assertions(len(gdt_description) > 1)
assertions(len(gdt_content) >= 0)
return
def _validate_google_dt_results(self, results_stream_def, results):
cc = self.container
assertions = self.assertTrue
for g in results:
if isinstance(g, Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
#log.warn(tx.pretty_print())
#log.warn(rdt.pretty_print())
gdt_component = rdt['google_dt_components'][0]
assertions(
gdt_component['viz_product_type'] == 'google_realtime_dt')
gdt_description = gdt_component['data_table_description']
gdt_content = gdt_component['data_table_content']
assertions(gdt_description[0][0] == 'time')
assertions(len(gdt_description) > 1)
assertions(len(gdt_content) >= 0)
return
def _get_data(cls, config):
parser = get_safe(config, 'parser', None)
ext_dset_res = get_safe(config, 'external_dataset_res', None)
if ext_dset_res and parser:
#CBM: Not in use yet...
# t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension']
# x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension']
# y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension']
# z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension']
# var_lst = ext_dset_res.dataset_description.parameters['variables']
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id',
'unknown data producer')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(
tx_yml
) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool
cnt = cls._calc_iter_cnt(len(parser.sensor_map), max_rec)
for x in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
for name in parser.sensor_map:
d = parser.data_map[name][x * max_rec:(x + 1) * max_rec]
rdt[name] = d
g = build_granule(data_producer_id=dprod_id,
taxonomy=ttool,
record_dictionary=rdt)
yield g
else:
log.warn('No parser object found in config')
def test_get_nick_names(self):
tc = TaxyTool()
tc.add_taxonomy_set('1', 'a')
tc.add_taxonomy_set('2', 'a')
self.assertEquals(tc.get_nick_name(0), '1')
self.assertEquals(tc.get_nick_names('a'), ['1', '2'])
self.assertEquals(tc.get_handle('1'), 0)
self.assertEquals(tc.get_handles('a'), {0, 1})
def init(self):
### Taxonomies are defined before hand out of band... somehow.
self._tx = TaxyTool()
self._tx.add_taxonomy_set('temp','long name for temp')
self._tx.add_taxonomy_set('cond','long name for cond')
self._tx.add_taxonomy_set('lat','long name for latitude')
self._tx.add_taxonomy_set('lon','long name for longitude')
self._tx.add_taxonomy_set('pres','long name for pres')
self._tx.add_taxonomy_set('time','long name for time')
def test_combine_granule(self):
tt = TaxyTool()
tt.add_taxonomy_set('a')
rdt = RecordDictionaryTool(tt)
rdt['a'] = numpy.array([1,2,3])
granule1 = build_granule('test',tt,rdt)
rdt = RecordDictionaryTool(tt)
rdt['a'] = numpy.array([4,5,6])
granule2 = build_granule('test',tt,rdt)
granule3 = combine_granules(granule1,granule2)
rdt = RecordDictionaryTool.load_from_granule(granule3)
self.assertTrue(numpy.allclose(rdt['a'],numpy.array([1,2,3,4,5,6])))
def test_init(self):
"""
test initialization of the TaxyCab
"""
tc = TaxyTool()
self.assertRaises(KeyError, tc.get_handle, 'nick_name')
self.assertRaises(KeyError,tc.get_names_by_handle, 0)
tx = Taxonomy(map={1:('nick_name',{'nick_name','a'})})
tc2 = TaxyTool(taxonomy=tx)
self.assertEquals(tc2._cnt,1)
self.assertEquals(tc2.get_handles('a'),{1,})
self.assertEquals(tc2.get_handle('nick_name'),1)
tc3 = TaxyTool(tx)
self.assertEquals(tc3.get_names_by_handle(1),{'nick_name','a'})
def test_get_nick_names(self):
tc = TaxyTool()
tc.add_taxonomy_set('1','a')
tc.add_taxonomy_set('2','a')
self.assertEquals(tc.get_nick_name(0),'1')
self.assertEquals(tc.get_nick_names('a'),['1', '2'])
self.assertEquals(tc.get_handle('1'),0)
self.assertEquals(tc.get_handles('a'),{0,1})
def combine_granules(granule_a, granule_b):
"""
This is a method that combines granules in a very naive manner
"""
validate_is_instance(granule_a, Granule, "granule_a is not a proper Granule")
validate_is_instance(granule_b, Granule, "granule_b is not a proper Granule")
tt_a = TaxyTool.load_from_granule(granule_a)
tt_b = TaxyTool.load_from_granule(granule_b)
if tt_a != tt_b:
raise BadRequest("Can't combine the two granules, they do not have the same taxonomy.")
rdt_new = RecordDictionaryTool(tt_a)
rdt_a = RecordDictionaryTool.load_from_granule(granule_a)
rdt_b = RecordDictionaryTool.load_from_granule(granule_b)
for k in rdt_a.iterkeys():
rdt_new[k] = np.append(rdt_a[k], rdt_b[k])
return build_granule(granule_a.data_producer_id, tt_a, rdt_new)
def test_combine_granule(self):
tt = TaxyTool()
tt.add_taxonomy_set('a')
rdt = RecordDictionaryTool(tt)
rdt['a'] = numpy.array([1, 2, 3])
granule1 = build_granule('test', tt, rdt)
rdt = RecordDictionaryTool(tt)
rdt['a'] = numpy.array([4, 5, 6])
granule2 = build_granule('test', tt, rdt)
granule3 = combine_granules(granule1, granule2)
rdt = RecordDictionaryTool.load_from_granule(granule3)
self.assertTrue(
numpy.allclose(rdt['a'], numpy.array([1, 2, 3, 4, 5, 6])))
def test_yamlize(self):
tc = TaxyTool()
tc.add_taxonomy_set('1','a')
tc.add_taxonomy_set('2','b')
tc.extend_names_by_nick_name('1','x')
tc.extend_names_by_anyname('a','z')
tc.extend_names_by_anyname('b','c')
s = tc.dump()
tc2 = TaxyTool.load(s)
#@todo - a list is not a set and the yaml dump/ion serialization can not handle sets...
self.assertEquals(tc2._cnt,1)
self.assertEquals(tc2.get_names_by_handle(0),{'1','x','a','z',})
self.assertEquals(tc2.get_names_by_handle(1),{'2','b','c',})
self.assertEquals(tc._cnt,1)
self.assertEquals(tc.get_names_by_handle(0),{'1','x','a','z',})
self.assertEquals(tc.get_names_by_handle(1),{'2','b','c',})
def process(self, packet):
if not isinstance(packet,Granule):
log.warn('Invalid packet received: Type "%s"' % type(packet))
return
rd_in = RecordDictionaryTool.load_from_granule(packet)
tt = TaxyTool.load_from_granule(packet)
rd_out = RecordDictionaryTool(tt)
for nickname, v_sequence in rd_in.iteritems():
rd_out[nickname] = self.shift(v_sequence)
g_out = build_granule(data_producer_id='dp_id',taxonomy=tt,record_dictionary=rd_out)
self.publish(g_out)
def launch_benchmark(transform_number=1, primer=1,message_length=4):
import gevent
from gevent.greenlet import Greenlet
from pyon.util.containers import DotDict
from pyon.net.transport import NameTrio
from pyon.net.endpoint import Publisher
import numpy
from pyon.ion.granule.record_dictionary import RecordDictionaryTool
from pyon.ion.granule.taxonomy import TaxyTool
from pyon.ion.granule.granule import build_granule
tt = TaxyTool()
tt.add_taxonomy_set('a')
import uuid
num = transform_number
msg_len = message_length
transforms = list()
pids = 1
TransformBenchTesting.message_length = message_length
cc = Container.instance
pub = Publisher(to_name=NameTrio(get_sys_name(),str(uuid.uuid4())[0:6]))
for i in xrange(num):
tbt=cc.proc_manager._create_service_instance(str(pids), 'tbt', 'prototype.transforms.linear', 'TransformInPlaceNewGranule', DotDict({'process':{'name':'tbt%d' % pids, 'transform_id':pids}}))
tbt.init()
tbt.start()
gevent.sleep(0.2)
for i in xrange(primer):
rd = RecordDictionaryTool(tt, message_length)
rd['a'] = numpy.arange(message_length)
gran = build_granule(data_producer_id='dp_id',taxonomy=tt, record_dictionary=rd)
pub.publish(gran)
g = Greenlet(tbt.perf)
g.start()
transforms.append(tbt)
pids += 1
def granule_example(nsdict):
"""
Usage:
from examples.granule import granule_example
granule_example(locals())
tx, g, rdt, rdt2... etc, are now local variables in your shell!
"""
#Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle
tx = TaxyTool()
tx.add_taxonomy_set('temp', 'long_temp_name')
tx.add_taxonomy_set('cond', 'long_cond_name')
tx.add_taxonomy_set('pres', 'long_pres_name')
tx.add_taxonomy_set('rdt')
# map is {<local name>: <granule name or path>}
#Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect
rdt = RecordDictionaryTool(taxonomy=tx)
#Create some arrays and fill them with random values
temp_array = numpy.random.standard_normal(100)
cond_array = numpy.random.standard_normal(100)
pres_array = numpy.random.standard_normal(100)
#Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc.
rdt['temp'] = temp_array
rdt['cond'] = cond_array
rdt['pres'] = pres_array
#You can also add in another RecordDictionaryTool, providing the taxonomies are the same.
rdt2 = RecordDictionaryTool(taxonomy=tx)
rdt2['temp'] = temp_array
rdt['rdt'] = rdt2
#you can get a string representation of the RecordDictionaryTool
print rdt
print rdt.pretty_print()
#Determine the length of the RecordDictionary using the len function
print len(rdt)
#Delete an item in the RecordDictionary
del rdt['temp']
g = build_granule(data_producer_id='john', taxonomy=tx, record_dictionary=rdt)
nsdict.update(locals())
def combine_granules(granule_a, granule_b):
"""
This is a method that combines granules in a very naive manner
"""
validate_is_instance(granule_a, Granule,
'granule_a is not a proper Granule')
validate_is_instance(granule_b, Granule,
'granule_b is not a proper Granule')
tt_a = TaxyTool.load_from_granule(granule_a)
tt_b = TaxyTool.load_from_granule(granule_b)
if tt_a != tt_b:
raise BadRequest(
'Can\'t combine the two granules, they do not have the same taxonomy.'
)
rdt_new = RecordDictionaryTool(tt_a)
rdt_a = RecordDictionaryTool.load_from_granule(granule_a)
rdt_b = RecordDictionaryTool.load_from_granule(granule_b)
for k in rdt_a.iterkeys():
rdt_new[k] = np.append(rdt_a[k], rdt_b[k])
return build_granule(granule_a.data_producer_id, tt_a, rdt_new)
def test_init(self):
"""
test initialization of the TaxyCab
"""
tc = TaxyTool()
self.assertRaises(KeyError, tc.get_handle, 'nick_name')
self.assertRaises(KeyError, tc.get_names_by_handle, 0)
tx = Taxonomy(map={1: ('nick_name', {'nick_name', 'a'})})
tc2 = TaxyTool(taxonomy=tx)
self.assertEquals(tc2._cnt, 1)
self.assertEquals(tc2.get_handles('a'), {
1,
})
self.assertEquals(tc2.get_handle('nick_name'), 1)
tc3 = TaxyTool(tx)
self.assertEquals(tc3.get_names_by_handle(1), {'nick_name', 'a'})
def _get_data(cls, config):
"""
Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len']
@param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count']
"""
array_len = get_safe(config, 'constraints.array_len',1)
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml)
arr = npr.random_sample(array_len)
log.debug('Array to send using max_rec={0}: {1}'.format(max_rec, arr))
cnt = cls._calc_iter_cnt(arr.size, max_rec)
for x in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
d = arr[x*max_rec:(x+1)*max_rec]
rdt['data'] = d
g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt)
yield g
def _get_data(cls, config):
"""
A generator that retrieves config['constraints']['count'] number of sequential Fibonacci numbers
@param config Dict of configuration parameters - must contain ['constraints']['count']
"""
cnt = get_safe(config, 'constraints.count', 1)
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml)
def fibGenerator():
"""
A Fibonacci sequence generator
"""
count = 0
ret = []
a, b = 1, 1
while 1:
count += 1
ret.append(a)
if count == max_rec:
yield np.array(ret)
ret = []
count = 0
a, b = b, a + b
gen = fibGenerator()
cnt = cls._calc_iter_cnt(cnt, max_rec)
for i in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
d = gen.next()
rdt['data'] = d
g = build_granule(data_producer_id=dprod_id,
taxonomy=ttool,
record_dictionary=rdt)
yield g
def _validate_mpl_graphs_results(self, results_stream_def, results):
cc = self.container
assertions = self.assertTrue
for g in results:
if isinstance(g,Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
#log.warn(tx.pretty_print())
#log.warn(rdt.pretty_print())
graphs = rdt['matplotlib_graphs']
for graph in graphs:
assertions(graph['viz_product_type'] == 'matplotlib_graphs' )
# check to see if the list (numpy array) contians actual images
assertions(imghdr.what(graph['image_name'], graph['image_obj']) == 'png')
return
def _get_data(cls, config):
"""
Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len']
@param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count']
"""
array_len = get_safe(config, 'constraints.array_len', 1)
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml)
arr = npr.random_sample(array_len)
log.debug('Array to send using max_rec={0}: {1}'.format(max_rec, arr))
cnt = cls._calc_iter_cnt(arr.size, max_rec)
for x in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
d = arr[x * max_rec:(x + 1) * max_rec]
rdt['data'] = d
g = build_granule(data_producer_id=dprod_id,
taxonomy=ttool,
record_dictionary=rdt)
yield g
def _get_data(cls, config):
"""
A generator that retrieves config['constraints']['count'] number of sequential Fibonacci numbers
@param config Dict of configuration parameters - must contain ['constraints']['count']
"""
cnt = get_safe(config,'constraints.count',1)
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml)
def fibGenerator():
"""
A Fibonacci sequence generator
"""
count = 0
ret = []
a, b = 1, 1
while 1:
count += 1
ret.append(a)
if count == max_rec:
yield np.array(ret)
ret=[]
count = 0
a, b = b, a + b
gen=fibGenerator()
cnt = cls._calc_iter_cnt(cnt, max_rec)
for i in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
d = gen.next()
rdt['data'] = d
g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt)
yield g
def _validate_mpl_graphs_results(self, results_stream_def, results):
cc = self.container
assertions = self.assertTrue
for g in results:
if isinstance(g, Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
#log.warn(tx.pretty_print())
#log.warn(rdt.pretty_print())
graphs = rdt['matplotlib_graphs']
for graph in graphs:
assertions(
graph['viz_product_type'] == 'matplotlib_graphs')
# check to see if the list (numpy array) contians actual images
assertions(
imghdr.what(graph['image_name'], graph['image_obj']) ==
'png')
return
def __init__(self,taxonomy, shape=None):
"""
@brief Initialize a new instance of Record Dictionary Tool with a taxonomy and an optional fixed length
@param taxonomy is an instance of a TaxonomyTool or Taxonomy (IonObject) used in this record dictionary
@param length is an optional fixed length for the value sequences of this record dictionary
"""
if not isinstance(shape, (_NoneType, int, tuple)):
raise TypeError('Invalid shape argument, received type "%s"; should be None or int or tuple' % type(shape))
self._rd = {}
self._shp = shape
if isinstance(self._shp, int):
self._shp = (self._shp,)
# hold onto the taxonomy - we need it to build the granule...
if isinstance(taxonomy, TaxyTool):
self._tx = taxonomy
elif isinstance(taxonomy, Taxonomy):
self._tx = TaxyTool(taxonomy)
else:
raise TypeError('Invalid taxonomy argument, received type "%s"; should be Taxonomy or TaxyTool' % type(taxonomy))
def test_extend_names(self):
tc = TaxyTool()
tc.add_taxonomy_set('1', 'a')
tc.add_taxonomy_set('2', 'a')
self.assertEquals(tc.get_handles('1'), {
0,
})
self.assertEquals(tc.get_handles('2'), {
1,
})
self.assertEquals(tc.get_names_by_handle(0), {
'1',
'a',
})
self.assertEquals(tc.get_names_by_handle(1), {
'2',
'a',
})
tc.extend_names_by_nick_name('1', 'c', 'e')
tc.extend_names_by_nick_name('2', 'z', 'x')
tc.extend_names_by_nick_name('1', 'd', 'f')
self.assertEquals(tc.get_handles('a'), {0, 1})
self.assertEquals(tc.get_handles('z'), {
1,
})
self.assertEquals(tc.get_handles('c'), {
0,
})
#Test for a name that isn't in the taxonomy
self.assertEquals(tc.get_handles('b'), {
-1,
})
self.assertEquals(tc.get_names_by_handle(0), {
'1',
'a',
'c',
'e',
'd',
'f',
})
self.assertEquals(tc.get_names_by_handle(1), {
'2',
'a',
'z',
'x',
})
tc.extend_names_by_anyname('a', 'extend')
self.assertEquals(tc.get_handles('extend'), {
0,
1,
})
def _get_data(cls, config):
"""
Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len']
@param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count']
"""
ext_dset_res = get_safe(config, 'external_dataset_res', None)
# Get the Dataset object from the config (should have been instantiated in _init_acquisition_cycle)
ds=get_safe(config, 'dataset_object')
if ext_dset_res and ds:
t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension']
x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension']
y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension']
z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension']
var_lst = ext_dset_res.dataset_description.parameters['variables']
t_slice = get_safe(config, 'constraints.temporal_slice', (slice(0,1)))
#TODO: Using 'eval' here is BAD - need to find a less sketchy way to pass constraints
if isinstance(t_slice,str):
t_slice=eval(t_slice)
lon = ds.variables[x_vname][:]
lat = ds.variables[y_vname][:]
z = ds.variables[z_vname][:]
t_arr = ds.variables[t_vname][t_slice]
data_arrays = {}
for varn in var_lst:
data_arrays[varn] = ds.variables[varn][t_slice]
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id', 'unknown data producer')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool
cnt = cls._calc_iter_cnt(t_arr.size, max_rec)
for x in xrange(cnt):
ta = t_arr[x*max_rec:(x+1)*max_rec]
# Make a 'master' RecDict
rdt = RecordDictionaryTool(taxonomy=ttool)
# Make a 'coordinate' RecDict
rdt_c = RecordDictionaryTool(taxonomy=ttool)
# Make a 'data' RecDict
rdt_d = RecordDictionaryTool(taxonomy=ttool)
# Assign values to the coordinate RecDict
rdt_c[x_vname] = lon
rdt_c[y_vname] = lat
rdt_c[z_vname] = z
# Assign values to the data RecDict
rdt_d[t_vname] = ta
for key, arr in data_arrays.iteritems():
d = arr[x*max_rec:(x+1)*max_rec]
rdt_d[key] = d
# Add the coordinate and data RecDicts to the master RecDict
rdt['coords'] = rdt_c
rdt['data'] = rdt_d
# Build and return a granule
# CBM: ttool must be passed
g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt)
yield g
ds.close()
def test_eq(self):
tx1 = Taxonomy(map={1:('nick_name',{'nick_name','a'}),
2:('nick2',{'nick2','a','b','cc'})} )
# pass with tt1._t.map "is"
tt1 = TaxyTool(tx1)
tt2 = TaxyTool(tx1)
self.assertEquals(tt1, tt2)
# after changes - thy are still the same
tt2.add_taxonomy_set('new_name','p','q','r')
self.assertEquals(tt2,tt1)
# pass with 'is'
tt3 = tt1
self.assertEquals(tt3, tt1)
# after changes - thy are still the same
tt3.add_taxonomy_set('new_name','p','q','r')
self.assertEquals(tt1,tt3)
# pass with tt1._t.map '=='
tx1 = Taxonomy(map={1:('nick_name',{'nick_name','a'}),
2:('nick2',{'nick2','a','b','cc'})} )
tx2 = Taxonomy(map={1:('nick_name',{'nick_name','a'}),
2:('nick2',{'nick2','a','b','cc'})} )
self.assertNotEquals(tx1, tx2)
self.assertEquals(tx1.map, tx2.map)
tt1 = TaxyTool(tx1)
tt2 = TaxyTool(tx2)
self.assertEquals(tt1, tt2)
# fail with tt1._t.map '=='
tx2 = Taxonomy(map={1:('nick_name',{'nick_name','as'}),
2:('nick2',{'nick2','a','b','cc'})} )
tt1 = TaxyTool(tx1)
tt2 = TaxyTool(tx2)
self.assertNotEquals(tt1, tt2)
# Use the interface to build a complex one and test equality as they go in and out of sync...
tt1 = TaxyTool()
tt2 = TaxyTool()
tt1.add_taxonomy_set('a name','junk','1','2')
tt2.add_taxonomy_set('a name','junk','1','2')
self.assertEquals(tt1, tt2)
tt2.add_taxonomy_set('new_name','1')
self.assertNotEquals(tt1,tt2)
tt1.extend_names_by_nick_name('a name','3')
tt2.extend_names_by_nick_name('a name','3')
tt1.add_taxonomy_set('new_name','1')
self.assertEquals(tt1, tt2)
class RecordDictionaryTool(object):
"""
A granule is a unit of information which conveys part of a coverage. It is composed of a taxonomy and a nested
structure of record dictionaries.
The record dictionary is composed of named value sequences and other nested record dictionaries.
The fact that all of the keys in the record dictionary are handles mapped by the taxonomy should never be exposed.
The application user refers to everything in the record dictionary by the unique nick name from the taxonomy.
"""
def __init__(self,taxonomy, shape=None):
"""
@brief Initialize a new instance of Record Dictionary Tool with a taxonomy and an optional fixed length
@param taxonomy is an instance of a TaxonomyTool or Taxonomy (IonObject) used in this record dictionary
@param length is an optional fixed length for the value sequences of this record dictionary
"""
if not isinstance(shape, (_NoneType, int, tuple)):
raise TypeError('Invalid shape argument, received type "%s"; should be None or int or tuple' % type(shape))
self._rd = {}
self._shp = shape
if isinstance(self._shp, int):
self._shp = (self._shp,)
# hold onto the taxonomy - we need it to build the granule...
if isinstance(taxonomy, TaxyTool):
self._tx = taxonomy
elif isinstance(taxonomy, Taxonomy):
self._tx = TaxyTool(taxonomy)
else:
raise TypeError('Invalid taxonomy argument, received type "%s"; should be Taxonomy or TaxyTool' % type(taxonomy))
@classmethod
def load_from_granule(cls, g):
"""
@brief return an instance of Record Dictionary Tool from a granule. Used when a granule is received in a message
"""
result = cls(TaxyTool(g.taxonomy))
result._rd = g.record_dictionary
return result
def __setitem__(self, name, vals):
"""
Set an item by nick name in the record dictionary
"""
if isinstance(vals, RecordDictionaryTool):
assert vals._tx == self._tx
self._rd[self._tx.get_handle(name)] = vals._rd
elif isinstance(vals, numpy.ndarray):
#Otherwise it is a value sequence which should have the correct length
# Matthew says: Assert only equal shape 5/17/12
if vals.ndim == 0:
raise ValueError('The rank of a value sequence array in a record dictionary must be greater than zero. Got name "%s" with rank "%d"' % (name, vals.ndim))
# Set _shp if it is None
if self._shp is None:
self._shp = vals.shape
# Test new value sequence length
if self._shp != vals.shape:
raise ValueError('Invalid array shape "%s" for name "%s"; Record dictionary defined shape is "%s"' % (vals.shape, name, self._shp))
self._rd[self._tx.get_handle(name)] = vals
else:
raise TypeError('Invalid type "%s" in Record Dictionary Tool setitem with name "%s". Valid types are numpy.ndarray and RecordDictionaryTool' % (type(vals),name))
def __getitem__(self, name):
"""
Get an item by nick name from the record dictionary.
"""
if isinstance(self._rd[self._tx.get_handle(name)], dict):
result = RecordDictionaryTool(taxonomy=self._tx)
result._rd = self._rd[self._tx.get_handle(name)]
return result
else:
return self._rd[self._tx.get_handle(name)]
def iteritems(self):
""" D.iteritems() -> an iterator over the (key, value) items of D """
for k, v in self._rd.iteritems():
if isinstance(v, dict):
result = RecordDictionaryTool(taxonomy=self._tx)
result._rd = v
yield self._tx.get_nick_name(k), result
else:
yield self._tx.get_nick_name(k), v
def iterkeys(self):
""" D.iterkeys() -> an iterator over the keys of D """
for k in self._rd.iterkeys():
yield self._tx.get_nick_name(k)
def itervalues(self):
""" D.itervalues() -> an iterator over the values of D """
for v in self._rd.itervalues():
if isinstance(v, dict):
result = RecordDictionaryTool(taxonomy=self._tx)
result._rd = v
yield result
else:
yield v
def update(self, E=None, **F):
"""
@brief Dictionary update method exposed for Record Dictionaries
@param E is another record dictionary
@param F is a dictionary of nicknames and value sequences
"""
if E:
if hasattr(E, "keys"):
for k in E:
self[k] = E[k]
else:
for k, v in E.iteritems():
self[k] = v
if F:
for k in F.keys():
self[k] = F[k]
def __contains__(self, nick_name):
""" D.__contains__(k) -> True if D has a key k, else False """
try:
handle = self._tx.get_handle(nick_name)
except KeyError as ke:
# if the nick_name is not in the taxonomy, it is certainly not in the record dictionary
return False
return handle in self._rd
def __delitem__(self, y):
""" x.__delitem__(y) <==> del x[y] """
#not sure if this is right, might just have to remove the name, not the whole handle
del self._rd[self._tx.get_handle(y)]
#will probably need to delete the name from _tx
def __iter__(self):
""" x.__iter__() <==> iter(x) """
for k in self._rd.iterkeys():
yield self._tx.get_nick_name(k)
def __len__(self):
""" x.__len__() <==> len(x) """
return len(self._rd)
def __repr__(self):
""" x.__repr__() <==> repr(x) """
result = "{"
for k, v in self.iteritems():
result += "\'{0}\': {1},".format(k, v)
if len(result) > 1:
result = result[:-1] + "}"
return result
def __str__(self):
result = "{"
for k, v in self.iteritems():
result += "\'{0}\': {1},".format(k, v)
if len(result) > 1:
result = result[:-1] + "}"
return result
__hash__ = None
def pretty_print(self):
"""
@brief Pretty Print the record dictionary for debug or log purposes.
"""
fid = StringIO.StringIO()
# Use string IO inside a try block in case of exceptions or a large return value.
try:
fid.write('Start Pretty Print Record Dictionary:\n')
self._pprint(fid,offset='')
fid.write('End of Pretty Print')
except Exception, ex:
log.exception('Unexpected Exception in Pretty Print Wrapper!')
fid.write('Exception! %s' % ex)
finally:
def test_activateDatasetAgent(self):
# Create ExternalDatasetModel
datsetModel_obj = IonObject(RT.ExternalDatasetModel,
name='ExampleDatasetModel',
description="ExampleDatasetModel",
datset_type="FibSeries")
try:
datasetModel_id = self.damsclient.create_external_dataset_model(
datsetModel_obj)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetModel: %s" % ex)
log.debug(
"TestExternalDatasetAgentMgmt: new ExternalDatasetModel id = %s",
str(datasetModel_id))
# Create ExternalDatasetAgent
datasetAgent_obj = IonObject(RT.ExternalDatasetAgent,
name='datasetagent007',
description="datasetagent007",
handler_module=EDA_MOD,
handler_class=EDA_CLS)
try:
datasetAgent_id = self.damsclient.create_external_dataset_agent(
datasetAgent_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetAgent: %s" % ex)
log.debug(
"TestExternalDatasetAgentMgmt: new ExternalDatasetAgent id = %s",
str(datasetAgent_id))
# Create ExternalDataset
log.debug(
'TestExternalDatasetAgentMgmt: Create external dataset resource ')
extDataset_obj = IonObject(RT.ExternalDataset,
name='ExtDataset',
description="ExtDataset")
try:
extDataset_id = self.damsclient.create_external_dataset(
extDataset_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new external dataset resource: %s" %
ex)
log.debug(
"TestExternalDatasetAgentMgmt: new ExternalDataset id = %s ",
str(extDataset_id))
#register the dataset as a data producer
dproducer_id = self.damsclient.register_external_data_set(
extDataset_id)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
container=ctd_stream_def)
log.debug(
"TestExternalDatasetAgentMgmt: new Stream Definition id = %s",
str(ctd_stream_def_id))
log.debug(
"TestExternalDatasetAgentMgmt: Creating new data product with a stream definition"
)
dp_obj = IonObject(RT.DataProduct,
name='eoi dataset data',
description=' stream test')
try:
data_product_id1 = self.dpclient.create_data_product(
dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
log.debug("TestExternalDatasetAgentMgmt: new dp_id = %s",
str(data_product_id1))
self.damsclient.assign_data_product(input_resource_id=extDataset_id,
data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1,
persist_data=True,
persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
log.debug("TestExternalDatasetAgentMgmt: Data product streams1 = %s",
str(stream_ids))
stream_id = stream_ids[0]
# Build a taxonomy for the dataset
tx = TaxyTool()
tx.add_taxonomy_set('data', 'external_data')
# Augment the DVR_CONFIG with the necessary pieces
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id':
stream_id, #TODO: This should probably be a 'stream_config' dict with stream_name:stream_id members
'data_producer_id': dproducer_id,
# 'external_dataset_res':extDataset_obj, # Not needed - retrieved by EDA based on resource_id
'taxonomy': tx.dump(), #TODO: Currently does not support sets
'max_records': 4,
}
# Create agent config.
self._stream_config = {}
agent_config = {
'driver_config': self.DVR_CONFIG,
'stream_config': self._stream_config,
'agent': {
'resource_id': EDA_RESOURCE_ID
},
'test_mode': True
}
extDatasetAgentInstance_obj = IonObject(
RT.ExternalDatasetAgentInstance,
name='DatasetAgentInstance',
description="DatasetAgentInstance",
dataset_driver_config=self.DVR_CONFIG,
dataset_agent_config=agent_config)
extDatasetAgentInstance_id = self.damsclient.create_external_dataset_agent_instance(
external_dataset_agent_instance=extDatasetAgentInstance_obj,
external_dataset_agent_id=datasetAgent_id,
external_dataset_id=extDataset_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s",
str(extDatasetAgentInstance_obj))
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s",
str(extDatasetAgentInstance_id))
#Check that the instance is currently not active
id, active = self.damsclient.retrieve_external_dataset_agent_instance(
extDataset_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 1 = %s ",
str(id), str(active))
self.damsclient.start_external_dataset_agent_instance(
extDatasetAgentInstance_id)
dataset_agent_instance_obj = self.damsclient.read_external_dataset_agent_instance(
extDatasetAgentInstance_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s",
str(dataset_agent_instance_obj))
# now the instance process should be active
id, active = self.damsclient.retrieve_external_dataset_agent_instance(
extDataset_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 2 = %s ",
str(id), str(active))
# Start a resource agent client to talk with the instrument agent.
self._dsa_client = ResourceAgentClient(extDataset_id,
process=FakeProcess())
print 'TestExternalDatasetAgentMgmt: got ia client %s', self._dsa_client
log.debug("TestExternalDatasetAgentMgmt: got dataset client %s",
str(self._dsa_client))
# cmd=AgentCommand(command='initialize')
# _ = self._dsa_client.execute_agent(cmd)
#
# cmd = AgentCommand(command='go_active')
# _ = self._dsa_client.execute_agent(cmd)
#
# cmd = AgentCommand(command='run')
# _ = self._dsa_client.execute_agent(cmd)
#
# log.info('Send an unconstrained request for data (\'new data\')')
# cmd = AgentCommand(command='acquire_data')
# self._dsa_client.execute(cmd)
#
# log.info('Send a second unconstrained request for data (\'new data\'), should be rejected')
# cmd = AgentCommand(command='acquire_data')
# self._dsa_client.execute(cmd)
#
# cmd = AgentCommand(command='reset')
# _ = self._dsa_client.execute_agent(cmd)
# cmd = AgentCommand(command='get_current_state')
# retval = self._dsa_client.execute_agent(cmd)
# state = retval.result
# TODO: Think about what we really should be testing at this point
# The following is taken from ion.agents.data.test.test_external_dataset_agent.ExternalDatasetAgentTestBase.test_states()
# TODO: Do we also need to show data retrieval?
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='resume')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='clear')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='clear')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='reset')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.damsclient.stop_external_dataset_agent_instance(
extDatasetAgentInstance_id)
from pyon.ion.granule.granule import build_granule
from pyon.ion.granule.taxonomy import TaxyTool
from pyon.ion.granule.record_dictionary import RecordDictionaryTool
from prototype.sci_data.stream_defs import SBE37_CDM_stream_definition, SBE37_RAW_stream_definition
from prototype.sci_data.stream_parser import PointSupplementStreamParser
from prototype.sci_data.constructor_apis import PointSupplementConstructor, RawSupplementConstructor
import StringIO
from numpy import array, append
# Matplotlib related imports
from matplotlib.backends.backend_agg import FigureCanvasAgg as FigureCanvas
from matplotlib.figure import Figure
tx = TaxyTool()
tx.add_taxonomy_set("matplotlib_graphs", "Matplotlib generated graphs for a particular data product")
class VizTransformMatplotlibGraphs(TransformFunction):
"""
This class is used for instantiating worker processes that have subscriptions to data streams and convert
incoming data from CDM format to Matplotlib graphs
"""
outgoing_stream_def = SBE37_RAW_stream_definition()
incoming_stream_def = SBE37_CDM_stream_definition()
def on_start(self):
def test_taxonomy_set(self):
nick_name = 'nick_name'
a = 'a'
b = 'b'
c = 'c'
tc = TaxyTool()
tc.add_taxonomy_set(nick_name=nick_name)
self.assertEquals(tc.get_handles('a name'), {
-1,
})
self.assertRaises(KeyError, tc.get_names_by_handle, 5)
self.assertEquals(tc.get_names_by_handle(0), {
nick_name,
})
tc = TaxyTool()
tc.add_taxonomy_set(a)
self.assertEquals(tc.get_handle(a), 0)
self.assertEquals(tc.get_names_by_handle(0), {
a,
})
self.assertEquals(tc.get_names_by_nick_name(a), {
a,
})
tc = TaxyTool()
tc.add_taxonomy_set(a)
tc.add_taxonomy_set(b)
self.assertEquals(tc.get_handle(a), 0)
self.assertEquals(tc.get_names_by_handle(0), {
a,
})
self.assertEquals(tc.get_handle(b), 1)
self.assertEquals(tc.get_names_by_handle(1), {
b,
})
tc = TaxyTool()
tc.add_taxonomy_set(nick_name, a, b, c)
self.assertEquals(tc.get_handle(nick_name), 0)
self.assertEquals(tc.get_handles(a), {
0,
})
self.assertEquals(tc.get_handles(b), {
0,
})
self.assertEquals(tc.get_handles(c), {
0,
})
self.assertEquals(tc.get_names_by_handle(0), {
nick_name,
a,
b,
c,
})
self.assertEquals(tc.get_names_by_nick_name(nick_name), {
nick_name,
a,
b,
c,
})
def test_taxonomy_set(self):
nick_name = 'nick_name'
a = 'a'
b = 'b'
c = 'c'
tc = TaxyTool()
tc.add_taxonomy_set(nick_name=nick_name)
self.assertEquals(tc.get_handles('a name'), {-1,})
self.assertRaises(KeyError,tc.get_names_by_handle,5)
self.assertEquals(tc.get_names_by_handle(0), {nick_name,})
tc = TaxyTool()
tc.add_taxonomy_set(a)
self.assertEquals(tc.get_handle(a),0)
self.assertEquals(tc.get_names_by_handle(0),{a,})
self.assertEquals(tc.get_names_by_nick_name(a),{a,})
tc = TaxyTool()
tc.add_taxonomy_set(a)
tc.add_taxonomy_set(b)
self.assertEquals(tc.get_handle(a),0)
self.assertEquals(tc.get_names_by_handle(0),{a,})
self.assertEquals(tc.get_handle(b),1)
self.assertEquals(tc.get_names_by_handle(1),{b,})
tc = TaxyTool()
tc.add_taxonomy_set(nick_name, a, b, c)
self.assertEquals(tc.get_handle(nick_name),0)
self.assertEquals(tc.get_handles(a),{0,})
self.assertEquals(tc.get_handles(b),{0,})
self.assertEquals(tc.get_handles(c),{0,})
self.assertEquals(tc.get_names_by_handle(0),{nick_name,a,b,c,})
self.assertEquals(tc.get_names_by_nick_name(nick_name),{nick_name,a,b,c,})
def test_activateDatasetAgent(self):
# Create ExternalDatasetModel
datsetModel_obj = IonObject(RT.ExternalDatasetModel, name='ExampleDatasetModel', description="ExampleDatasetModel", datset_type="FibSeries" )
try:
datasetModel_id = self.damsclient.create_external_dataset_model(datsetModel_obj)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetModel: %s" %ex)
log.debug("TestExternalDatasetAgentMgmt: new ExternalDatasetModel id = %s", str(datasetModel_id) )
# Create ExternalDatasetAgent
datasetAgent_obj = IonObject(RT.ExternalDatasetAgent, name='datasetagent007', description="datasetagent007", handler_module=EDA_MOD, handler_class=EDA_CLS )
try:
datasetAgent_id = self.damsclient.create_external_dataset_agent(datasetAgent_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetAgent: %s" %ex)
log.debug("TestExternalDatasetAgentMgmt: new ExternalDatasetAgent id = %s", str(datasetAgent_id) )
# Create ExternalDataset
log.debug('TestExternalDatasetAgentMgmt: Create external dataset resource ')
extDataset_obj = IonObject(RT.ExternalDataset, name='ExtDataset', description="ExtDataset" )
try:
extDataset_id = self.damsclient.create_external_dataset(extDataset_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new external dataset resource: %s" %ex)
log.debug("TestExternalDatasetAgentMgmt: new ExternalDataset id = %s ", str(extDataset_id))
#register the dataset as a data producer
dproducer_id = self.damsclient.register_external_data_set(extDataset_id)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubcli.create_stream_definition(container=ctd_stream_def)
log.debug("TestExternalDatasetAgentMgmt: new Stream Definition id = %s", str(ctd_stream_def_id))
log.debug("TestExternalDatasetAgentMgmt: Creating new data product with a stream definition")
dp_obj = IonObject(RT.DataProduct,name='eoi dataset data',description=' stream test')
try:
data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
log.debug("TestExternalDatasetAgentMgmt: new dp_id = %s", str(data_product_id1) )
self.damsclient.assign_data_product(input_resource_id=extDataset_id, data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug("TestExternalDatasetAgentMgmt: Data product streams1 = %s", str(stream_ids) )
stream_id = stream_ids[0]
# Build a taxonomy for the dataset
tx = TaxyTool()
tx.add_taxonomy_set('data', 'external_data')
# Augment the DVR_CONFIG with the necessary pieces
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,#TODO: This should probably be a 'stream_config' dict with stream_name:stream_id members
'data_producer_id':dproducer_id,
# 'external_dataset_res':extDataset_obj, # Not needed - retrieved by EDA based on resource_id
'taxonomy':tx.dump(), #TODO: Currently does not support sets
'max_records':4,
}
# Create agent config.
self._stream_config = {}
agent_config = {
'driver_config' : self.DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': EDA_RESOURCE_ID},
'test_mode' : True
}
extDatasetAgentInstance_obj = IonObject(RT.ExternalDatasetAgentInstance, name='DatasetAgentInstance', description="DatasetAgentInstance", dataset_driver_config = self.DVR_CONFIG, dataset_agent_config = agent_config)
extDatasetAgentInstance_id = self.damsclient.create_external_dataset_agent_instance(external_dataset_agent_instance=extDatasetAgentInstance_obj, external_dataset_agent_id=datasetAgent_id, external_dataset_id=extDataset_id)
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(extDatasetAgentInstance_obj) )
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s", str(extDatasetAgentInstance_id) )
#Check that the instance is currently not active
id, active = self.damsclient.retrieve_external_dataset_agent_instance(extDataset_id)
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 1 = %s ", str(id), str(active) )
self.damsclient.start_external_dataset_agent_instance(extDatasetAgentInstance_id)
dataset_agent_instance_obj= self.damsclient.read_external_dataset_agent_instance(extDatasetAgentInstance_id)
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(dataset_agent_instance_obj) )
# now the instance process should be active
id, active = self.damsclient.retrieve_external_dataset_agent_instance(extDataset_id)
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 2 = %s ", str(id), str(active) )
# Start a resource agent client to talk with the instrument agent.
self._dsa_client = ResourceAgentClient(extDataset_id, process=FakeProcess())
print 'TestExternalDatasetAgentMgmt: got ia client %s', self._dsa_client
log.debug("TestExternalDatasetAgentMgmt: got dataset client %s", str(self._dsa_client))
# cmd=AgentCommand(command='initialize')
# _ = self._dsa_client.execute_agent(cmd)
#
# cmd = AgentCommand(command='go_active')
# _ = self._dsa_client.execute_agent(cmd)
#
# cmd = AgentCommand(command='run')
# _ = self._dsa_client.execute_agent(cmd)
#
# log.info('Send an unconstrained request for data (\'new data\')')
# cmd = AgentCommand(command='acquire_data')
# self._dsa_client.execute(cmd)
#
# log.info('Send a second unconstrained request for data (\'new data\'), should be rejected')
# cmd = AgentCommand(command='acquire_data')
# self._dsa_client.execute(cmd)
#
# cmd = AgentCommand(command='reset')
# _ = self._dsa_client.execute_agent(cmd)
# cmd = AgentCommand(command='get_current_state')
# retval = self._dsa_client.execute_agent(cmd)
# state = retval.result
# TODO: Think about what we really should be testing at this point
# The following is taken from ion.agents.data.test.test_external_dataset_agent.ExternalDatasetAgentTestBase.test_states()
# TODO: Do we also need to show data retrieval?
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='resume')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='clear')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='clear')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='reset')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.damsclient.stop_external_dataset_agent_instance(extDatasetAgentInstance_id)
def test_build_granule_and_load_from_granule(self):
#Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle
tx = TaxyTool()
tx.add_taxonomy_set('temp', 'long_temp_name')
tx.add_taxonomy_set('cond', 'long_cond_name')
tx.add_taxonomy_set('pres', 'long_pres_name')
tx.add_taxonomy_set('rdt')
# map is {<local name>: <granule name or path>}
#Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect
rdt = RecordDictionaryTool(taxonomy=tx)
#Create some arrays and fill them with random values
temp_array = numpy.random.standard_normal(100)
cond_array = numpy.random.standard_normal(100)
pres_array = numpy.random.standard_normal(100)
#Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc.
rdt['temp'] = temp_array
rdt['cond'] = cond_array
rdt['pres'] = pres_array
#You can also add in another RecordDictionaryTool, providing the taxonomies are the same.
rdt2 = RecordDictionaryTool(taxonomy=tx)
rdt2['temp'] = temp_array
rdt['rdt'] = rdt2
g = build_granule(data_producer_id='john',
taxonomy=tx,
record_dictionary=rdt)
l_tx = TaxyTool.load_from_granule(g)
l_rd = RecordDictionaryTool.load_from_granule(g)
# Make sure we got back the same Taxonomy Object
self.assertEquals(l_tx._t, tx._t)
self.assertEquals(l_tx.get_handles('temp'), tx.get_handles('temp'))
self.assertEquals(l_tx.get_handles('testing_2'),
tx.get_handles('testing_2'))
# Now test the record dictionary object
self.assertEquals(l_rd._rd, rdt._rd)
self.assertEquals(l_rd._tx._t, rdt._tx._t)
for k, v in l_rd.iteritems():
self.assertIn(k, rdt)
if isinstance(v, numpy.ndarray):
self.assertTrue((v == rdt[k]).all())
else:
self.assertEquals(v._rd, rdt[k]._rd)
def _process_visualization_message(self, messages):
gdt_description = None
gdt_content = []
viz_product_type = ''
for message in messages:
message_data = message.body
if isinstance(message_data,Granule):
tx = TaxyTool.load_from_granule(message_data)
rdt = RecordDictionaryTool.load_from_granule(message_data)
gdt_components = get_safe(rdt, 'google_dt_components')
# IF this granule does not contain google dt, skip
if gdt_components is None:
continue
gdt_component = gdt_components[0]
viz_product_type = gdt_component['viz_product_type']
# Process Google DataTable messages
if viz_product_type == 'google_dt':
# If the data description is being put together for the first time,
# switch the time format from float to datetime
if (gdt_description == None):
temp_gdt_description = gdt_component['data_description']
gdt_description = [('time', 'datetime', 'time')]
for idx in range(1,len(temp_gdt_description)):
# for some weird reason need to force convert to tuples
temp_arr = temp_gdt_description[idx]
if temp_arr != None:
gdt_description.append((temp_arr[0], temp_arr[1], temp_arr[2]))
# append all content to one big array
temp_gdt_content = gdt_component['data_content']
for tempTuple in temp_gdt_content:
# sometimes there are inexplicable empty tuples in the content. Drop them
if tempTuple == [] or len(tempTuple) == 0:
continue
varTuple = []
varTuple.append(datetime.fromtimestamp(tempTuple[0]))
for idx in range(1,len(tempTuple)):
varTuple.append(tempTuple[idx])
gdt_content.append(varTuple)
#TODO - what to do if this is not a valid visualization message?
# Now that all the messages have been parsed, any last processing should be done here
if viz_product_type == "google_dt":
# Using the description and content, build the google data table
gdt = gviz_api.DataTable(gdt_description)
gdt.LoadData(gdt_content)
return gdt.ToJSonResponse()
return None
# Inherit some old machinery for this example
from ion.processes.data.ctd_stream_publisher import SimpleCtdPublisher
### For new granule and stream interface
from pyon.ion.granule.record_dictionary import RecordDictionaryTool
from pyon.ion.granule.taxonomy import TaxyTool
from pyon.ion.granule.granule import build_granule
from pyon.public import log
import numpy
import random
import time
### Taxonomies are defined before hand out of band... somehow.
tx = TaxyTool()
tx.add_taxonomy_set('temp','long name for temp')
tx.add_taxonomy_set('cond','long name for cond')
tx.add_taxonomy_set('lat','long name for latitude')
tx.add_taxonomy_set('lon','long name for longitude')
tx.add_taxonomy_set('pres','long name for pres')
tx.add_taxonomy_set('time','long name for time')
# This is an example of using groups it is not a normative statement about how to use groups
tx.add_taxonomy_set('group1','This group contains coordinates...')
tx.add_taxonomy_set('group0','This group contains data...')
tx.add_taxonomy_set('raw_fixed','Fixed length bytes in an array of records')
tx.add_taxonomy_set('raw_blob','Unlimited length bytes in an array')
class ExampleDataProducer(SimpleCtdPublisher):
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
eda = ExternalDatasetAgent()
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance()
eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.name = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(protocol_type='FILE', institution=Institution(), contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.name='Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'slocum_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
dset.dataset_description.parameters['dataset_path'] = 'test_data/ru05-2012-021-0-0-sbd.dat'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = [
'c_wpt_y_lmc',
'sci_water_cond',
'm_y_lmc',
'u_hd_fin_ap_inflection_holdoff',
'sci_m_present_time',
'm_leakdetect_voltage_forward',
'sci_bb3slo_b660_scaled',
'c_science_send_all',
'm_gps_status',
'm_water_vx',
'm_water_vy',
'c_heading',
'sci_fl3slo_chlor_units',
'u_hd_fin_ap_gain',
'm_vacuum',
'u_min_water_depth',
'm_gps_lat',
'm_veh_temp',
'f_fin_offset',
'u_hd_fin_ap_hardover_holdoff',
'c_alt_time',
'm_present_time',
'm_heading',
'sci_bb3slo_b532_scaled',
'sci_fl3slo_cdom_units',
'm_fin',
'x_cycle_overrun_in_ms',
'sci_water_pressure',
'u_hd_fin_ap_igain',
'sci_fl3slo_phyco_units',
'm_battpos',
'sci_bb3slo_b470_scaled',
'm_lat',
'm_gps_lon',
'sci_ctd41cp_timestamp',
'm_pressure',
'c_wpt_x_lmc',
'c_ballast_pumped',
'x_lmc_xy_source',
'm_lon',
'm_avg_speed',
'sci_water_temp',
'u_pitch_ap_gain',
'm_roll',
'm_tot_num_inflections',
'm_x_lmc',
'u_pitch_ap_deadband',
'm_final_water_vy',
'm_final_water_vx',
'm_water_depth',
'm_leakdetect_voltage',
'u_pitch_max_delta_battpos',
'm_coulomb_amphr',
'm_pitch',
]
# Create DataSourceModel
dsrc_model = DataSourceModel(name='slocum_model')
dsrc_model.model = 'SLOCUM'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
# dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id)
# Generate the data product and associate it to the ExternalDataset
dprod = DataProduct(name='slocum_parsed_product', description='parsed slocum product')
dproduct_id = dpms_cli.create_data_product(data_product=dprod)
dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id, create_stream=True)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({'ExternalDataset':ds_id, 'ExternalDataProvider':ext_dprov_id, 'DataSource':ext_dsrc_id, 'DataSourceModel':ext_dsrc_model_id, 'DataProducer':dproducer_id, 'DataProduct':dproduct_id, 'Stream':stream_id}))
#CBM: Use CF standard_names
ttool = TaxyTool()
ttool.add_taxonomy_set('c_wpt_y_lmc'),
ttool.add_taxonomy_set('sci_water_cond'),
ttool.add_taxonomy_set('m_y_lmc'),
ttool.add_taxonomy_set('u_hd_fin_ap_inflection_holdoff'),
ttool.add_taxonomy_set('sci_m_present_time'),
ttool.add_taxonomy_set('m_leakdetect_voltage_forward'),
ttool.add_taxonomy_set('sci_bb3slo_b660_scaled'),
ttool.add_taxonomy_set('c_science_send_all'),
ttool.add_taxonomy_set('m_gps_status'),
ttool.add_taxonomy_set('m_water_vx'),
ttool.add_taxonomy_set('m_water_vy'),
ttool.add_taxonomy_set('c_heading'),
ttool.add_taxonomy_set('sci_fl3slo_chlor_units'),
ttool.add_taxonomy_set('u_hd_fin_ap_gain'),
ttool.add_taxonomy_set('m_vacuum'),
ttool.add_taxonomy_set('u_min_water_depth'),
ttool.add_taxonomy_set('m_gps_lat'),
ttool.add_taxonomy_set('m_veh_temp'),
ttool.add_taxonomy_set('f_fin_offset'),
ttool.add_taxonomy_set('u_hd_fin_ap_hardover_holdoff'),
ttool.add_taxonomy_set('c_alt_time'),
ttool.add_taxonomy_set('m_present_time'),
ttool.add_taxonomy_set('m_heading'),
ttool.add_taxonomy_set('sci_bb3slo_b532_scaled'),
ttool.add_taxonomy_set('sci_fl3slo_cdom_units'),
ttool.add_taxonomy_set('m_fin'),
ttool.add_taxonomy_set('x_cycle_overrun_in_ms'),
ttool.add_taxonomy_set('sci_water_pressure'),
ttool.add_taxonomy_set('u_hd_fin_ap_igain'),
ttool.add_taxonomy_set('sci_fl3slo_phyco_units'),
ttool.add_taxonomy_set('m_battpos'),
ttool.add_taxonomy_set('sci_bb3slo_b470_scaled'),
ttool.add_taxonomy_set('m_lat'),
ttool.add_taxonomy_set('m_gps_lon'),
ttool.add_taxonomy_set('sci_ctd41cp_timestamp'),
ttool.add_taxonomy_set('m_pressure'),
ttool.add_taxonomy_set('c_wpt_x_lmc'),
ttool.add_taxonomy_set('c_ballast_pumped'),
ttool.add_taxonomy_set('x_lmc_xy_source'),
ttool.add_taxonomy_set('m_lon'),
ttool.add_taxonomy_set('m_avg_speed'),
ttool.add_taxonomy_set('sci_water_temp'),
ttool.add_taxonomy_set('u_pitch_ap_gain'),
ttool.add_taxonomy_set('m_roll'),
ttool.add_taxonomy_set('m_tot_num_inflections'),
ttool.add_taxonomy_set('m_x_lmc'),
ttool.add_taxonomy_set('u_pitch_ap_deadband'),
ttool.add_taxonomy_set('m_final_water_vy'),
ttool.add_taxonomy_set('m_final_water_vx'),
ttool.add_taxonomy_set('m_water_depth'),
ttool.add_taxonomy_set('m_leakdetect_voltage'),
ttool.add_taxonomy_set('u_pitch_max_delta_battpos'),
ttool.add_taxonomy_set('m_coulomb_amphr'),
ttool.add_taxonomy_set('m_pitch'),
#CBM: Eventually, probably want to group this crap somehow - not sure how yet...
# Create the logger for receiving publications
self.create_stream_and_logger(name='slocum',stream_id=stream_id)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,
'external_dataset_res':dset,
'taxonomy':ttool.dump(),
'data_producer_id':dproducer_id,#CBM: Should this be put in the main body of the config - with mod & cls?
'max_records':20,
}
def test_extend_names(self):
tc = TaxyTool()
tc.add_taxonomy_set('1','a')
tc.add_taxonomy_set('2','a')
self.assertEquals(tc.get_handles('1'),{0,})
self.assertEquals(tc.get_handles('2'),{1,})
self.assertEquals(tc.get_names_by_handle(0),{'1', 'a',})
self.assertEquals(tc.get_names_by_handle(1),{'2', 'a',})
tc.extend_names_by_nick_name('1', 'c', 'e')
tc.extend_names_by_nick_name('2', 'z', 'x')
tc.extend_names_by_nick_name('1', 'd', 'f')
self.assertEquals(tc.get_handles('a'),{0,1})
self.assertEquals(tc.get_handles('z'),{1,})
self.assertEquals(tc.get_handles('c'),{0,})
#Test for a name that isn't in the taxonomy
self.assertEquals(tc.get_handles('b'),{-1,})
self.assertEquals(tc.get_names_by_handle(0),{'1', 'a', 'c', 'e', 'd', 'f',})
self.assertEquals(tc.get_names_by_handle(1),{'2', 'a', 'z', 'x',})
tc.extend_names_by_anyname('a', 'extend')
self.assertEquals(tc.get_handles('extend'),{0,1,})
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
eda = ExternalDatasetAgent()
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance()
eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.name = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(protocol_type='FILE', institution=Institution(), contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.name='Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'ruv_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
dset.dataset_description.parameters['dataset_path'] = 'test_data/RDLi_SEAB_2011_08_24_1600.ruv'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = [
]
# Create DataSourceModel
dsrc_model = DataSourceModel(name='ruv_model')
dsrc_model.model = 'RUV'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
# dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id)
# Generate the data product and associate it to the ExternalDataset
dprod = DataProduct(name='ruv_parsed_product', description='parsed ruv product')
dproduct_id = dpms_cli.create_data_product(data_product=dprod)
dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id, create_stream=True)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({'ExternalDataset':ds_id, 'ExternalDataProvider':ext_dprov_id, 'DataSource':ext_dsrc_id, 'DataSourceModel':ext_dsrc_model_id, 'DataProducer':dproducer_id, 'DataProduct':dproduct_id, 'Stream':stream_id}))
#CBM: Use CF standard_names
ttool = TaxyTool()
ttool.add_taxonomy_set('data','test data')
#CBM: Eventually, probably want to group this crap somehow - not sure how yet...
# Create the logger for receiving publications
self.create_stream_and_logger(name='ruv',stream_id=stream_id)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,
'external_dataset_res':dset,
'taxonomy':ttool.dump(),
'data_producer_id':dproducer_id,#CBM: Should this be put in the main body of the config - with mod & cls?
'max_records':20,
}
def test_eq(self):
tx1 = Taxonomy(
map={
1: ('nick_name', {'nick_name', 'a'}),
2: ('nick2', {'nick2', 'a', 'b', 'cc'})
})
# pass with tt1._t.map "is"
tt1 = TaxyTool(tx1)
tt2 = TaxyTool(tx1)
self.assertEquals(tt1, tt2)
# after changes - thy are still the same
tt2.add_taxonomy_set('new_name', 'p', 'q', 'r')
self.assertEquals(tt2, tt1)
# pass with 'is'
tt3 = tt1
self.assertEquals(tt3, tt1)
# after changes - thy are still the same
tt3.add_taxonomy_set('new_name', 'p', 'q', 'r')
self.assertEquals(tt1, tt3)
# pass with tt1._t.map '=='
tx1 = Taxonomy(
map={
1: ('nick_name', {'nick_name', 'a'}),
2: ('nick2', {'nick2', 'a', 'b', 'cc'})
})
tx2 = Taxonomy(
map={
1: ('nick_name', {'nick_name', 'a'}),
2: ('nick2', {'nick2', 'a', 'b', 'cc'})
})
self.assertNotEquals(tx1, tx2)
self.assertEquals(tx1.map, tx2.map)
tt1 = TaxyTool(tx1)
tt2 = TaxyTool(tx2)
self.assertEquals(tt1, tt2)
# fail with tt1._t.map '=='
tx2 = Taxonomy(
map={
1: ('nick_name', {'nick_name', 'as'}),
2: ('nick2', {'nick2', 'a', 'b', 'cc'})
})
tt1 = TaxyTool(tx1)
tt2 = TaxyTool(tx2)
self.assertNotEquals(tt1, tt2)
# Use the interface to build a complex one and test equality as they go in and out of sync...
tt1 = TaxyTool()
tt2 = TaxyTool()
tt1.add_taxonomy_set('a name', 'junk', '1', '2')
tt2.add_taxonomy_set('a name', 'junk', '1', '2')
self.assertEquals(tt1, tt2)
tt2.add_taxonomy_set('new_name', '1')
self.assertNotEquals(tt1, tt2)
tt1.extend_names_by_nick_name('a name', '3')
tt2.extend_names_by_nick_name('a name', '3')
tt1.add_taxonomy_set('new_name', '1')
self.assertEquals(tt1, tt2)
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
eda = ExternalDatasetAgent()
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance()
eda_inst_id = dams_cli.create_external_dataset_agent_instance(
eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(),
contact=ContactInformation())
dprov.contact.name = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(protocol_type='DAP',
institution=Institution(),
contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.name = 'Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'usgs_test_dataset'
dset = ExternalDataset(name=ds_name,
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
# The usgs.nc test dataset is a download of the R1 dataset found here:
# http://thredds-test.oceanobservatories.org/thredds/dodsC/ooiciData/E66B1A74-A684-454A-9ADE-8388C2C634E5.ncml
dset.dataset_description.parameters[
'dataset_path'] = 'test_data/usgs.nc'
dset.dataset_description.parameters['temporal_dimension'] = 'time'
dset.dataset_description.parameters['zonal_dimension'] = 'lon'
dset.dataset_description.parameters['meridional_dimension'] = 'lat'
dset.dataset_description.parameters['vertical_dimension'] = 'z'
dset.dataset_description.parameters['variables'] = [
'water_temperature',
'streamflow',
'water_temperature_bottom',
'water_temperature_middle',
'specific_conductance',
'data_qualifier',
]
# Create DataSourceModel
dsrc_model = DataSourceModel(name='dap_model')
dsrc_model.model = 'DAP'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(
external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(
external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(
data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(
data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(
external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(
external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
# dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id)
# Generate the data product and associate it to the ExternalDataset
dprod = DataProduct(name='usgs_parsed_product',
description='parsed usgs product')
dproduct_id = dpms_cli.create_data_product(data_product=dprod)
dams_cli.assign_data_product(input_resource_id=ds_id,
data_product_id=dproduct_id,
create_stream=True)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id,
predicate=PRED.hasStream,
object_type=RT.Stream,
id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({
'ExternalDataset': ds_id,
'ExternalDataProvider': ext_dprov_id,
'DataSource': ext_dsrc_id,
'DataSourceModel': ext_dsrc_model_id,
'DataProducer': dproducer_id,
'DataProduct': dproduct_id,
'Stream': stream_id
}))
#CBM: Use CF standard_names
ttool = TaxyTool()
ttool.add_taxonomy_set('time', 'time')
ttool.add_taxonomy_set('lon', 'longitude')
ttool.add_taxonomy_set('lat', 'latitude')
ttool.add_taxonomy_set('z', 'water depth')
ttool.add_taxonomy_set('water_temperature',
'average water temperature')
ttool.add_taxonomy_set('water_temperature_bottom',
'water temperature at bottom of water column')
ttool.add_taxonomy_set('water_temperature_middle',
'water temperature at middle of water column')
ttool.add_taxonomy_set('streamflow', 'flow velocity of stream')
ttool.add_taxonomy_set('specific_conductance',
'specific conductance of water')
ttool.add_taxonomy_set('data_qualifier', 'data qualifier flag')
ttool.add_taxonomy_set('coords',
'This group contains coordinate parameters')
ttool.add_taxonomy_set('data', 'This group contains data parameters')
# Create the logger for receiving publications
self.create_stream_and_logger(name='usgs', stream_id=stream_id)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id': stream_id,
'taxonomy': ttool.dump(),
'data_producer_id':
dproducer_id, #CBM: Should this be put in the main body of the config - with mod & cls?
'max_records': 4,
}
def test_yamlize(self):
tc = TaxyTool()
tc.add_taxonomy_set('1', 'a')
tc.add_taxonomy_set('2', 'b')
tc.extend_names_by_nick_name('1', 'x')
tc.extend_names_by_anyname('a', 'z')
tc.extend_names_by_anyname('b', 'c')
s = tc.dump()
tc2 = TaxyTool.load(s)
#@todo - a list is not a set and the yaml dump/ion serialization can not handle sets...
self.assertEquals(tc2._cnt, 1)
self.assertEquals(tc2.get_names_by_handle(0), {
'1',
'x',
'a',
'z',
})
self.assertEquals(tc2.get_names_by_handle(1), {
'2',
'b',
'c',
})
self.assertEquals(tc._cnt, 1)
self.assertEquals(tc.get_names_by_handle(0), {
'1',
'x',
'a',
'z',
})
self.assertEquals(tc.get_names_by_handle(1), {
'2',
'b',
'c',
})
class RecordDictionaryTool(object):
"""
A granule is a unit of information which conveys part of a coverage. It is composed of a taxonomy and a nested
structure of record dictionaries.
The record dictionary is composed of named value sequences and other nested record dictionaries.
The fact that all of the keys in the record dictionary are handles mapped by the taxonomy should never be exposed.
The application user refers to everything in the record dictionary by the unique nick name from the taxonomy.
"""
def __init__(self, taxonomy, shape=None):
"""
@brief Initialize a new instance of Record Dictionary Tool with a taxonomy and an optional fixed length
@param taxonomy is an instance of a TaxonomyTool or Taxonomy (IonObject) used in this record dictionary
@param length is an optional fixed length for the value sequences of this record dictionary
"""
if not isinstance(shape, (_NoneType, int, tuple)):
raise TypeError(
'Invalid shape argument, received type "%s"; should be None or int or tuple'
% type(shape))
self._rd = {}
self._shp = shape
if isinstance(self._shp, int):
self._shp = (self._shp, )
# hold onto the taxonomy - we need it to build the granule...
if isinstance(taxonomy, TaxyTool):
self._tx = taxonomy
elif isinstance(taxonomy, Taxonomy):
self._tx = TaxyTool(taxonomy)
else:
raise TypeError(
'Invalid taxonomy argument, received type "%s"; should be Taxonomy or TaxyTool'
% type(taxonomy))
@classmethod
def load_from_granule(cls, g):
"""
@brief return an instance of Record Dictionary Tool from a granule. Used when a granule is received in a message
"""
result = cls(TaxyTool(g.taxonomy))
result._rd = g.record_dictionary
return result
def __setitem__(self, name, vals):
"""
Set an item by nick name in the record dictionary
"""
if isinstance(vals, RecordDictionaryTool):
assert vals._tx == self._tx
self._rd[self._tx.get_handle(name)] = vals._rd
elif isinstance(vals, numpy.ndarray):
#Otherwise it is a value sequence which should have the correct length
# Matthew says: Assert only equal shape 5/17/12
if vals.ndim == 0:
raise ValueError(
'The rank of a value sequence array in a record dictionary must be greater than zero. Got name "%s" with rank "%d"'
% (name, vals.ndim))
# Set _shp if it is None
if self._shp is None:
self._shp = vals.shape
# Test new value sequence length
if self._shp != vals.shape:
raise ValueError(
'Invalid array shape "%s" for name "%s"; Record dictionary defined shape is "%s"'
% (vals.shape, name, self._shp))
self._rd[self._tx.get_handle(name)] = vals
else:
raise TypeError(
'Invalid type "%s" in Record Dictionary Tool setitem with name "%s". Valid types are numpy.ndarray and RecordDictionaryTool'
% (type(vals), name))
def __getitem__(self, name):
"""
Get an item by nick name from the record dictionary.
"""
if isinstance(self._rd[self._tx.get_handle(name)], dict):
result = RecordDictionaryTool(taxonomy=self._tx)
result._rd = self._rd[self._tx.get_handle(name)]
return result
else:
return self._rd[self._tx.get_handle(name)]
def iteritems(self):
""" D.iteritems() -> an iterator over the (key, value) items of D """
for k, v in self._rd.iteritems():
if isinstance(v, dict):
result = RecordDictionaryTool(taxonomy=self._tx)
result._rd = v
yield self._tx.get_nick_name(k), result
else:
yield self._tx.get_nick_name(k), v
def iterkeys(self):
""" D.iterkeys() -> an iterator over the keys of D """
for k in self._rd.iterkeys():
yield self._tx.get_nick_name(k)
def itervalues(self):
""" D.itervalues() -> an iterator over the values of D """
for v in self._rd.itervalues():
if isinstance(v, dict):
result = RecordDictionaryTool(taxonomy=self._tx)
result._rd = v
yield result
else:
yield v
def update(self, E=None, **F):
"""
@brief Dictionary update method exposed for Record Dictionaries
@param E is another record dictionary
@param F is a dictionary of nicknames and value sequences
"""
if E:
if hasattr(E, "keys"):
for k in E:
self[k] = E[k]
else:
for k, v in E.iteritems():
self[k] = v
if F:
for k in F.keys():
self[k] = F[k]
def __contains__(self, nick_name):
""" D.__contains__(k) -> True if D has a key k, else False """
try:
handle = self._tx.get_handle(nick_name)
except KeyError as ke:
# if the nick_name is not in the taxonomy, it is certainly not in the record dictionary
return False
return handle in self._rd
def __delitem__(self, y):
""" x.__delitem__(y) <==> del x[y] """
#not sure if this is right, might just have to remove the name, not the whole handle
del self._rd[self._tx.get_handle(y)]
#will probably need to delete the name from _tx
def __iter__(self):
""" x.__iter__() <==> iter(x) """
for k in self._rd.iterkeys():
yield self._tx.get_nick_name(k)
def __len__(self):
""" x.__len__() <==> len(x) """
return len(self._rd)
def __repr__(self):
""" x.__repr__() <==> repr(x) """
result = "{"
for k, v in self.iteritems():
result += "\'{0}\': {1},".format(k, v)
if len(result) > 1:
result = result[:-1] + "}"
return result
def __str__(self):
result = "{"
for k, v in self.iteritems():
result += "\'{0}\': {1},".format(k, v)
if len(result) > 1:
result = result[:-1] + "}"
return result
__hash__ = None
def pretty_print(self):
"""
@brief Pretty Print the record dictionary for debug or log purposes.
"""
fid = StringIO.StringIO()
# Use string IO inside a try block in case of exceptions or a large return value.
try:
fid.write('Start Pretty Print Record Dictionary:\n')
self._pprint(fid, offset='')
fid.write('End of Pretty Print')
except Exception, ex:
log.exception('Unexpected Exception in Pretty Print Wrapper!')
fid.write('Exception! %s' % ex)
finally:
from pyon.service.service import BaseService
from pyon.core.exception import BadRequest
from pyon.public import IonObject, RT, log
from datetime import datetime
import numpy
from pyon.ion.granule.granule import build_granule
from pyon.ion.granule.taxonomy import TaxyTool
from pyon.ion.granule.record_dictionary import RecordDictionaryTool
from prototype.sci_data.stream_defs import SBE37_CDM_stream_definition, SBE37_RAW_stream_definition
from prototype.sci_data.stream_parser import PointSupplementStreamParser
from prototype.sci_data.constructor_apis import PointSupplementConstructor, RawSupplementConstructor
tx = TaxyTool()
tx.add_taxonomy_set('google_dt_components','Google DT components for part or entire datatable')
class VizTransformGoogleDT(TransformFunction):
"""
This class is used for converting incoming data from CDM format to JSON style Google DataTables
Note: One behaviour that this class is expected to achieve specifically is to determine if its supposed
to work as a realtime transform (exists indefinitely and maintains a sliding window of data) or as
a replay transform (one-shot).
[2] This transform behaves as an instantaneous forwarder. There is no waiting for the entire stream
to create the complete datatable. As the granules come in, they are translated to the datatable
'components'. Components, because we are not creating the actual datatable in this code. That's the job