def __init__(self,param_dictionary=None, stream_definition_id='', locator=None):
"""
"""
if type(param_dictionary) == dict:
self._pdict = ParameterDictionary.load(param_dictionary)
elif isinstance(param_dictionary,ParameterDictionary):
self._pdict = param_dictionary
elif stream_definition_id:
pdict = RecordDictionaryTool.pdict_from_stream_def(stream_definition_id)
self._pdict = ParameterDictionary.load(pdict)
self._stream_def = stream_definition_id
else:
raise BadRequest('Unable to create record dictionary with improper ParameterDictionary')
if stream_definition_id:
self._stream_def=stream_definition_id
self._shp = None
self._rd = {}
self._locator = locator
self._setup_params()
def _construct_stream_and_publisher(self, stream_name, stream_config):
if log.isEnabledFor(logging.TRACE): # pragma: no cover
log.trace("%r: _construct_stream_and_publisher: "
"stream_name:%r, stream_config:\n%s",
self._platform_id, stream_name,
self._pp.pformat(stream_config))
decoder = IonObjectDeserializer(obj_registry=get_obj_registry())
if 'stream_def_dict' not in stream_config:
# should not happen: PlatformAgent._validate_configuration validates this.
log.error("'stream_def_dict' key not in configuration for stream %r" % stream_name)
return
stream_def_dict = stream_config['stream_def_dict']
stream_def_dict['type_'] = 'StreamDefinition'
stream_def_obj = decoder.deserialize(stream_def_dict)
self._stream_defs[stream_name] = stream_def_obj
routing_key = stream_config['routing_key']
stream_id = stream_config['stream_id']
exchange_point = stream_config['exchange_point']
parameter_dictionary = stream_def_dict['parameter_dictionary']
log.debug("%r: got parameter_dictionary from stream_def_dict", self._platform_id)
self._data_streams[stream_name] = stream_id
self._param_dicts[stream_name] = ParameterDictionary.load(parameter_dictionary)
stream_route = StreamRoute(exchange_point=exchange_point, routing_key=routing_key)
publisher = self._create_publisher(stream_id, stream_route)
self._data_publishers[stream_name] = publisher
log.debug("%r: created publisher for stream_name=%r", self._platform_id, stream_name)
def _construct_stream_and_publisher(self, stream_name, stream_config):
if log.isEnabledFor(logging.TRACE): # pragma: no cover
log.trace("%r: _construct_stream_and_publisher: "
"stream_name:%r, stream_config:\n%s",
self._platform_id, stream_name,
self._pp.pformat(stream_config))
decoder = IonObjectDeserializer(obj_registry=get_obj_registry())
if 'stream_def_dict' not in stream_config:
# should not happen: PlatformAgent._validate_configuration validates this.
log.error("'stream_def_dict' key not in configuration for stream %r" % stream_name)
return
stream_def_dict = stream_config['stream_def_dict']
stream_def_dict['type_'] = 'StreamDefinition'
stream_def_obj = decoder.deserialize(stream_def_dict)
self._stream_defs[stream_name] = stream_def_obj
routing_key = stream_config['routing_key']
stream_id = stream_config['stream_id']
exchange_point = stream_config['exchange_point']
parameter_dictionary = stream_def_dict['parameter_dictionary']
log.debug("%r: got parameter_dictionary from stream_def_dict", self._platform_id)
self._data_streams[stream_name] = stream_id
self._param_dicts[stream_name] = ParameterDictionary.load(parameter_dictionary)
stream_route = StreamRoute(exchange_point=exchange_point, routing_key=routing_key)
publisher = self._create_publisher(stream_id, stream_route)
self._data_publishers[stream_name] = publisher
log.debug("%r: created publisher for stream_name=%r", self._platform_id, stream_name)
def _get_data(cls, config):
new_flst = get_safe(config, 'constraints.new_files', [])
hdr_cnt = get_safe(config, 'header_count', SlocumParser.DEFAULT_HEADER_SIZE)
for f in new_flst:
try:
parser = SlocumParser(f[0], hdr_cnt)
#CBM: Not in use yet...
# ext_dset_res = get_safe(config, 'external_dataset_res', None)
# t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension']
# x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension']
# y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension']
# z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension']
# var_lst = ext_dset_res.dataset_description.parameters['variables']
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id', 'unknown data producer')
#tx_yml = get_safe(config, 'taxonomy')
#ttool = TaxyTool.load(tx_yml) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool
pdict = ParameterDictionary.load(get_safe(config, 'param_dictionary'))
cnt = calculate_iteration_count(len(parser.sensor_map), max_rec)
for x in xrange(cnt):
#rdt = RecordDictionaryTool(taxonomy=ttool)
rdt = RecordDictionaryTool(param_dictionary=pdict)
for name in parser.sensor_map:
d = parser.data_map[name][x*max_rec:(x+1)*max_rec]
rdt[name]=d
#g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt)
g = build_granule(data_producer_id=dprod_id, record_dictionary=rdt, param_dictionary=pdict)
yield g
except SlocumParseException as spe:
# TODO: Decide what to do here, raise an exception or carry on
log.error('Error parsing data file: \'{0}\''.format(f))
def load_from_granule(cls, g):
"""
@brief return an instance of Record Dictionary Tool from a granule. Used when a granule is received in a message
"""
if g.param_dictionary:
result = cls(param_dictionary=ParameterDictionary.load(g.param_dictionary))
else:
result = cls(TaxyTool(g.taxonomy))
result._rd = g.record_dictionary
if result._rd.has_key(0):
result._shp = result._rd[0].shape
return result
def load_from_granule(cls, g):
if isinstance(g.param_dictionary, str):
instance = cls(stream_definition_id=g.param_dictionary, locator=g.locator)
pdict = RecordDictionaryTool.pdict_from_stream_def(g.param_dictionary)
instance._pdict = ParameterDictionary.load(pdict)
else:
instance = cls(param_dictionary=g.param_dictionary, locator=g.locator)
instance._pdict = ParameterDictionary.load(g.param_dictionary)
if g.domain:
instance._shp = (g.domain[0],)
for k,v in g.record_dictionary.iteritems():
if v is not None:
ptype = instance._pdict.get_context(k).param_type
paramval = get_value_class(ptype, domain_set = instance.domain)
paramval[:] = v
paramval.storage._storage.flags.writeable = False
instance._rd[k] = paramval
return instance
def _get_data(cls, config):
"""
Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len']
@param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count']
"""
array_len = get_safe(config, 'constraints.array_len',1)
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id')
#tx_yml = get_safe(config, 'taxonomy')
#ttool = TaxyTool.load(tx_yml)
pdict = ParameterDictionary.load(get_safe(config, 'param_dictionary'))
arr = npr.random_sample(array_len)
log.debug('Array to send using max_rec={0}: {1}'.format(max_rec, arr))
cnt = calculate_iteration_count(arr.size, max_rec)
for x in xrange(cnt):
rdt = RecordDictionaryTool(param_dictionary=pdict)
d = arr[x*max_rec:(x+1)*max_rec]
rdt['dummy'] = d
g = rdt.to_granule()
yield g
def _get_data(cls, config):
"""
A generator that retrieves config['constraints']['count'] number of sequential Fibonacci numbers
@param config Dict of configuration parameters - must contain ['constraints']['count']
"""
cnt = get_safe(config,'constraints.count',1)
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id')
#tx_yml = get_safe(config, 'taxonomy')
#ttool = TaxyTool.load(tx_yml)
pdict = ParameterDictionary.load(get_safe(config, 'param_dictionary'))
def fibGenerator():
"""
A Fibonacci sequence generator
"""
count = 0
ret = []
a, b = 1, 1
while 1:
count += 1
ret.append(a)
if count == max_rec:
yield np.array(ret)
ret=[]
count = 0
a, b = b, a + b
gen=fibGenerator()
cnt = calculate_iteration_count(cnt, max_rec)
for i in xrange(cnt):
rdt = RecordDictionaryTool(param_dictionary=pdict)
d = gen.next()
rdt['data'] = d
g = rdt.to_granule()
yield g
def _construct_stream_and_publisher(self, stream_name, stream_config):
# granule_publish_rate
# records_per_granule
if log.isEnabledFor(logging.TRACE): # pragma: no cover
log.trace("%r: _construct_stream_and_publisher: "
"stream_name:%r, stream_config:\n%s",
self._platform_id, stream_name,
self._pp.pformat(stream_config))
decoder = IonObjectDeserializer(obj_registry=get_obj_registry())
if 'stream_def_dict' in stream_config:
stream_def_dict = stream_config['stream_def_dict']
stream_def_dict['type_'] = 'StreamDefinition'
stream_def_obj = decoder.deserialize(stream_def_dict)
self._stream_defs[stream_name] = stream_def_obj
log.debug("%r: using stream_def_dict", self._platform_id)
else: # TODO this case to be removed.
stream_definition_ref = stream_config['stream_definition_ref']
self._stream_defs[stream_name] = stream_definition_ref
log.debug("%r: using stream_definition_ref", self._platform_id)
routing_key = stream_config['routing_key']
stream_id = stream_config['stream_id']
exchange_point = stream_config['exchange_point']
parameter_dictionary = stream_config['parameter_dictionary']
self._data_streams[stream_name] = stream_id
self._param_dicts[stream_name] = ParameterDictionary.load(parameter_dictionary)
stream_route = StreamRoute(exchange_point=exchange_point, routing_key=routing_key)
publisher = self._create_publisher(stream_id, stream_route)
self._data_publishers[stream_name] = publisher
log.debug("%r: created publisher for stream_name=%r", self._platform_id, stream_name)
def _get_data(cls, config):
"""
Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len']
@param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count']
"""
ext_dset_res = get_safe(config, 'external_dataset_res', None)
# Get the Dataset object from the config (should have been instantiated in _init_acquisition_cycle)
ds=get_safe(config, 'dataset_object')
if ext_dset_res and ds:
t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension']
x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension']
y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension']
z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension']
var_lst = ext_dset_res.dataset_description.parameters['variables']
t_slice = get_safe(config, 'constraints.temporal_slice', (slice(0,1)))
#TODO: Using 'eval' here is BAD - need to find a less sketchy way to pass constraints
if isinstance(t_slice,str):
t_slice=eval(t_slice)
lon = ds.variables[x_vname][:]
lat = ds.variables[y_vname][:]
z = ds.variables[z_vname][:]
t_arr = ds.variables[t_vname][t_slice]
data_arrays = {}
for varn in var_lst:
data_arrays[varn] = ds.variables[varn][t_slice]
max_rec = get_safe(config, 'max_records', 1)
dprod_id = get_safe(config, 'data_producer_id', 'unknown data producer')
#tx_yml = get_safe(config, 'taxonomy')
#ttool = TaxyTool.load(tx_yml) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool
pdict = ParameterDictionary.load(get_safe(config, 'param_dictionary'))
cnt = calculate_iteration_count(t_arr.size, max_rec)
for x in xrange(cnt):
ta = t_arr[x*max_rec:(x+1)*max_rec]
# Make a 'master' RecDict
#rdt = RecordDictionaryTool(taxonomy=ttool)
rdt = RecordDictionaryTool(param_dictionary=pdict)
# Make a 'coordinate' RecDict
#rdt_c = RecordDictionaryTool(taxonomy=ttool)
#rdt_c = RecordDictionaryTool(param_dictionary=pdict)
# Make a 'data' RecDict
#rdt_d = RecordDictionaryTool(taxonomy=ttool)
#rdt_d = RecordDictionaryTool(param_dictionary=pdict)
# Assign values to the coordinate RecDict
rdt[x_vname] = lon
rdt[y_vname] = lat
rdt[z_vname] = z
# Assign values to the data RecDict
rdt[t_vname] = ta
for key, arr in data_arrays.iteritems():
d = arr[x*max_rec:(x+1)*max_rec]
rdt[key] = d
# Add the coordinate and data RecDicts to the master RecDict
#rdt['coords'] = rdt_c
#rdt['data'] = rdt_d
# Build and return a granule
# CBM: ttool must be passed
#g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt)
g = build_granule(data_producer_id=dprod_id, record_dictionary=rdt, param_dictionary=pdict)
yield g
ds.close()
def test_build_granule_and_load_from_granule(self):
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.reference_frame = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.dtype('float32')))
lat_ctxt.reference_frame = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.dtype('float32')))
lon_ctxt.reference_frame = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
pdict.add_context(lon_ctxt)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.dtype('float32')))
cond_ctxt.uom = 'unknown'
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('pres', param_type=QuantityType(value_encoding=np.dtype('float32')))
pres_ctxt.uom = 'unknown'
pdict.add_context(pres_ctxt)
rdt = RecordDictionaryTool(param_dictionary=pdict)
#Create some arrays and fill them with random values
temp_array = np.random.standard_normal(100)
cond_array = np.random.standard_normal(100)
pres_array = np.random.standard_normal(100)
time_array = np.random.standard_normal(100)
lat_array = np.random.standard_normal(100)
lon_array = np.random.standard_normal(100)
#Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc.
rdt['temp'] = temp_array
rdt['conductivity'] = cond_array
rdt['pres'] = pres_array
rdt['time'] = time_array
rdt['lat'] = lat_array
rdt['lon'] = lon_array
g = build_granule(data_producer_id='john', record_dictionary=rdt, param_dictionary=pdict)
l_pd = ParameterDictionary.load(g.param_dictionary)
#l_tx = TaxyTool.load_from_granule(g)
l_rd = RecordDictionaryTool.load_from_granule(g)
# Make sure we got back the same Taxonomy Object
#self.assertEquals(l_pd, pdict)
self.assertEquals(l_pd.ord_from_key('temp'), pdict.ord_from_key('temp'))
self.assertEquals(l_pd.ord_from_key('conductivity'), pdict.ord_from_key('conductivity'))
# Now test the record dictionary object
self.assertEquals(l_rd._rd, rdt._rd)
#self.assertEquals(l_rd._param_dict, rdt._param_dict)
for k, v in l_rd.iteritems():
self.assertIn(k, rdt)
if isinstance(v, np.ndarray):
self.assertTrue( (v == rdt[k]).all())
else:
self.assertEquals(v._rd, rdt[k]._rd)
