def _get_data(cls, config):
parser = get_safe(config, 'parser', None)
ext_dset_res = get_safe(config, 'external_dataset_res', None)
if ext_dset_res and parser:
#CBM: Not in use yet...
# t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension']
# x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension']
# y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension']
# z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension']
# var_lst = ext_dset_res.dataset_description.parameters['variables']
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id', 'unknown data producer')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool
cnt = cls._calc_iter_cnt(len(parser.sensor_map), max_rec)
for x in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
for name in parser.sensor_map:
d = parser.data_map[name][x*max_rec:(x+1)*max_rec]
rdt[name]=d
g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt)
yield g
else:
log.warn('No parser object found in config')
def _get_data(cls, config):
parser = get_safe(config, 'parser', None)
ext_dset_res = get_safe(config, 'external_dataset_res', None)
if ext_dset_res and parser:
#CBM: Not in use yet...
# t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension']
# x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension']
# y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension']
# z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension']
# var_lst = ext_dset_res.dataset_description.parameters['variables']
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id',
'unknown data producer')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(
tx_yml
) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool
cnt = cls._calc_iter_cnt(len(parser.sensor_map), max_rec)
for x in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
for name in parser.sensor_map:
d = parser.data_map[name][x * max_rec:(x + 1) * max_rec]
rdt[name] = d
g = build_granule(data_producer_id=dprod_id,
taxonomy=ttool,
record_dictionary=rdt)
yield g
else:
log.warn('No parser object found in config')
def _get_data(cls, config):
"""
Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len']
@param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count']
"""
array_len = get_safe(config, 'constraints.array_len',1)
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml)
arr = npr.random_sample(array_len)
log.debug('Array to send using max_rec={0}: {1}'.format(max_rec, arr))
cnt = cls._calc_iter_cnt(arr.size, max_rec)
for x in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
d = arr[x*max_rec:(x+1)*max_rec]
rdt['data'] = d
g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt)
yield g
def test_yamlize(self):
tc = TaxyTool()
tc.add_taxonomy_set('1', 'a')
tc.add_taxonomy_set('2', 'b')
tc.extend_names_by_nick_name('1', 'x')
tc.extend_names_by_anyname('a', 'z')
tc.extend_names_by_anyname('b', 'c')
s = tc.dump()
tc2 = TaxyTool.load(s)
#@todo - a list is not a set and the yaml dump/ion serialization can not handle sets...
self.assertEquals(tc2._cnt, 1)
self.assertEquals(tc2.get_names_by_handle(0), {
'1',
'x',
'a',
'z',
})
self.assertEquals(tc2.get_names_by_handle(1), {
'2',
'b',
'c',
})
self.assertEquals(tc._cnt, 1)
self.assertEquals(tc.get_names_by_handle(0), {
'1',
'x',
'a',
'z',
})
self.assertEquals(tc.get_names_by_handle(1), {
'2',
'b',
'c',
})
def _get_data(cls, config):
"""
A generator that retrieves config['constraints']['count'] number of sequential Fibonacci numbers
@param config Dict of configuration parameters - must contain ['constraints']['count']
"""
cnt = get_safe(config, 'constraints.count', 1)
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml)
def fibGenerator():
"""
A Fibonacci sequence generator
"""
count = 0
ret = []
a, b = 1, 1
while 1:
count += 1
ret.append(a)
if count == max_rec:
yield np.array(ret)
ret = []
count = 0
a, b = b, a + b
gen = fibGenerator()
cnt = cls._calc_iter_cnt(cnt, max_rec)
for i in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
d = gen.next()
rdt['data'] = d
g = build_granule(data_producer_id=dprod_id,
taxonomy=ttool,
record_dictionary=rdt)
yield g
def test_yamlize(self):
tc = TaxyTool()
tc.add_taxonomy_set('1','a')
tc.add_taxonomy_set('2','b')
tc.extend_names_by_nick_name('1','x')
tc.extend_names_by_anyname('a','z')
tc.extend_names_by_anyname('b','c')
s = tc.dump()
tc2 = TaxyTool.load(s)
#@todo - a list is not a set and the yaml dump/ion serialization can not handle sets...
self.assertEquals(tc2._cnt,1)
self.assertEquals(tc2.get_names_by_handle(0),{'1','x','a','z',})
self.assertEquals(tc2.get_names_by_handle(1),{'2','b','c',})
self.assertEquals(tc._cnt,1)
self.assertEquals(tc.get_names_by_handle(0),{'1','x','a','z',})
self.assertEquals(tc.get_names_by_handle(1),{'2','b','c',})
def _get_data(cls, config):
"""
Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len']
@param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count']
"""
array_len = get_safe(config, 'constraints.array_len', 1)
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml)
arr = npr.random_sample(array_len)
log.debug('Array to send using max_rec={0}: {1}'.format(max_rec, arr))
cnt = cls._calc_iter_cnt(arr.size, max_rec)
for x in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
d = arr[x * max_rec:(x + 1) * max_rec]
rdt['data'] = d
g = build_granule(data_producer_id=dprod_id,
taxonomy=ttool,
record_dictionary=rdt)
yield g
def _get_data(cls, config):
"""
A generator that retrieves config['constraints']['count'] number of sequential Fibonacci numbers
@param config Dict of configuration parameters - must contain ['constraints']['count']
"""
cnt = get_safe(config,'constraints.count',1)
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml)
def fibGenerator():
"""
A Fibonacci sequence generator
"""
count = 0
ret = []
a, b = 1, 1
while 1:
count += 1
ret.append(a)
if count == max_rec:
yield np.array(ret)
ret=[]
count = 0
a, b = b, a + b
gen=fibGenerator()
cnt = cls._calc_iter_cnt(cnt, max_rec)
for i in xrange(cnt):
rdt = RecordDictionaryTool(taxonomy=ttool)
d = gen.next()
rdt['data'] = d
g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt)
yield g
def _get_data(cls, config):
"""
Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len']
@param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count']
"""
ext_dset_res = get_safe(config, 'external_dataset_res', None)
# Get the Dataset object from the config (should have been instantiated in _init_acquisition_cycle)
ds=get_safe(config, 'dataset_object')
if ext_dset_res and ds:
t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension']
x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension']
y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension']
z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension']
var_lst = ext_dset_res.dataset_description.parameters['variables']
t_slice = get_safe(config, 'constraints.temporal_slice', (slice(0,1)))
#TODO: Using 'eval' here is BAD - need to find a less sketchy way to pass constraints
if isinstance(t_slice,str):
t_slice=eval(t_slice)
lon = ds.variables[x_vname][:]
lat = ds.variables[y_vname][:]
z = ds.variables[z_vname][:]
t_arr = ds.variables[t_vname][t_slice]
data_arrays = {}
for varn in var_lst:
data_arrays[varn] = ds.variables[varn][t_slice]
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id', 'unknown data producer')
tx_yml = get_safe(config, 'taxonomy')
ttool = TaxyTool.load(tx_yml) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool
cnt = cls._calc_iter_cnt(t_arr.size, max_rec)
for x in xrange(cnt):
ta = t_arr[x*max_rec:(x+1)*max_rec]
# Make a 'master' RecDict
rdt = RecordDictionaryTool(taxonomy=ttool)
# Make a 'coordinate' RecDict
rdt_c = RecordDictionaryTool(taxonomy=ttool)
# Make a 'data' RecDict
rdt_d = RecordDictionaryTool(taxonomy=ttool)
# Assign values to the coordinate RecDict
rdt_c[x_vname] = lon
rdt_c[y_vname] = lat
rdt_c[z_vname] = z
# Assign values to the data RecDict
rdt_d[t_vname] = ta
for key, arr in data_arrays.iteritems():
d = arr[x*max_rec:(x+1)*max_rec]
rdt_d[key] = d
# Add the coordinate and data RecDicts to the master RecDict
rdt['coords'] = rdt_c
rdt['data'] = rdt_d
# Build and return a granule
# CBM: ttool must be passed
g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt)
yield g
ds.close()
