def __init__(self,param_dictionary=None, stream_definition_id='', locator=None): """ """ if type(param_dictionary) == dict: self._pdict = ParameterDictionary.load(param_dictionary) elif isinstance(param_dictionary,ParameterDictionary): self._pdict = param_dictionary elif stream_definition_id: pdict = RecordDictionaryTool.pdict_from_stream_def(stream_definition_id) self._pdict = ParameterDictionary.load(pdict) self._stream_def = stream_definition_id else: raise BadRequest('Unable to create record dictionary with improper ParameterDictionary') if stream_definition_id: self._stream_def=stream_definition_id self._shp = None self._rd = {} self._locator = locator self._setup_params()
def _construct_stream_and_publisher(self, stream_name, stream_config): if log.isEnabledFor(logging.TRACE): # pragma: no cover log.trace("%r: _construct_stream_and_publisher: " "stream_name:%r, stream_config:\n%s", self._platform_id, stream_name, self._pp.pformat(stream_config)) decoder = IonObjectDeserializer(obj_registry=get_obj_registry()) if 'stream_def_dict' not in stream_config: # should not happen: PlatformAgent._validate_configuration validates this. log.error("'stream_def_dict' key not in configuration for stream %r" % stream_name) return stream_def_dict = stream_config['stream_def_dict'] stream_def_dict['type_'] = 'StreamDefinition' stream_def_obj = decoder.deserialize(stream_def_dict) self._stream_defs[stream_name] = stream_def_obj routing_key = stream_config['routing_key'] stream_id = stream_config['stream_id'] exchange_point = stream_config['exchange_point'] parameter_dictionary = stream_def_dict['parameter_dictionary'] log.debug("%r: got parameter_dictionary from stream_def_dict", self._platform_id) self._data_streams[stream_name] = stream_id self._param_dicts[stream_name] = ParameterDictionary.load(parameter_dictionary) stream_route = StreamRoute(exchange_point=exchange_point, routing_key=routing_key) publisher = self._create_publisher(stream_id, stream_route) self._data_publishers[stream_name] = publisher log.debug("%r: created publisher for stream_name=%r", self._platform_id, stream_name)
def _get_data(cls, config): new_flst = get_safe(config, 'constraints.new_files', []) hdr_cnt = get_safe(config, 'header_count', SlocumParser.DEFAULT_HEADER_SIZE) for f in new_flst: try: parser = SlocumParser(f[0], hdr_cnt) #CBM: Not in use yet... # ext_dset_res = get_safe(config, 'external_dataset_res', None) # t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension'] # x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension'] # y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension'] # z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension'] # var_lst = ext_dset_res.dataset_description.parameters['variables'] max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id', 'unknown data producer') #tx_yml = get_safe(config, 'taxonomy') #ttool = TaxyTool.load(tx_yml) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool pdict = ParameterDictionary.load(get_safe(config, 'param_dictionary')) cnt = calculate_iteration_count(len(parser.sensor_map), max_rec) for x in xrange(cnt): #rdt = RecordDictionaryTool(taxonomy=ttool) rdt = RecordDictionaryTool(param_dictionary=pdict) for name in parser.sensor_map: d = parser.data_map[name][x*max_rec:(x+1)*max_rec] rdt[name]=d #g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt) g = build_granule(data_producer_id=dprod_id, record_dictionary=rdt, param_dictionary=pdict) yield g except SlocumParseException as spe: # TODO: Decide what to do here, raise an exception or carry on log.error('Error parsing data file: \'{0}\''.format(f))
def load_from_granule(cls, g): """ @brief return an instance of Record Dictionary Tool from a granule. Used when a granule is received in a message """ if g.param_dictionary: result = cls(param_dictionary=ParameterDictionary.load(g.param_dictionary)) else: result = cls(TaxyTool(g.taxonomy)) result._rd = g.record_dictionary if result._rd.has_key(0): result._shp = result._rd[0].shape return result
def load_from_granule(cls, g): if isinstance(g.param_dictionary, str): instance = cls(stream_definition_id=g.param_dictionary, locator=g.locator) pdict = RecordDictionaryTool.pdict_from_stream_def(g.param_dictionary) instance._pdict = ParameterDictionary.load(pdict) else: instance = cls(param_dictionary=g.param_dictionary, locator=g.locator) instance._pdict = ParameterDictionary.load(g.param_dictionary) if g.domain: instance._shp = (g.domain[0],) for k,v in g.record_dictionary.iteritems(): if v is not None: ptype = instance._pdict.get_context(k).param_type paramval = get_value_class(ptype, domain_set = instance.domain) paramval[:] = v paramval.storage._storage.flags.writeable = False instance._rd[k] = paramval return instance
def _get_data(cls, config): """ Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len'] @param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count'] """ array_len = get_safe(config, 'constraints.array_len',1) max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id') #tx_yml = get_safe(config, 'taxonomy') #ttool = TaxyTool.load(tx_yml) pdict = ParameterDictionary.load(get_safe(config, 'param_dictionary')) arr = npr.random_sample(array_len) log.debug('Array to send using max_rec={0}: {1}'.format(max_rec, arr)) cnt = calculate_iteration_count(arr.size, max_rec) for x in xrange(cnt): rdt = RecordDictionaryTool(param_dictionary=pdict) d = arr[x*max_rec:(x+1)*max_rec] rdt['dummy'] = d g = rdt.to_granule() yield g
def _get_data(cls, config): """ A generator that retrieves config['constraints']['count'] number of sequential Fibonacci numbers @param config Dict of configuration parameters - must contain ['constraints']['count'] """ cnt = get_safe(config,'constraints.count',1) max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id') #tx_yml = get_safe(config, 'taxonomy') #ttool = TaxyTool.load(tx_yml) pdict = ParameterDictionary.load(get_safe(config, 'param_dictionary')) def fibGenerator(): """ A Fibonacci sequence generator """ count = 0 ret = [] a, b = 1, 1 while 1: count += 1 ret.append(a) if count == max_rec: yield np.array(ret) ret=[] count = 0 a, b = b, a + b gen=fibGenerator() cnt = calculate_iteration_count(cnt, max_rec) for i in xrange(cnt): rdt = RecordDictionaryTool(param_dictionary=pdict) d = gen.next() rdt['data'] = d g = rdt.to_granule() yield g
def _construct_stream_and_publisher(self, stream_name, stream_config): # granule_publish_rate # records_per_granule if log.isEnabledFor(logging.TRACE): # pragma: no cover log.trace("%r: _construct_stream_and_publisher: " "stream_name:%r, stream_config:\n%s", self._platform_id, stream_name, self._pp.pformat(stream_config)) decoder = IonObjectDeserializer(obj_registry=get_obj_registry()) if 'stream_def_dict' in stream_config: stream_def_dict = stream_config['stream_def_dict'] stream_def_dict['type_'] = 'StreamDefinition' stream_def_obj = decoder.deserialize(stream_def_dict) self._stream_defs[stream_name] = stream_def_obj log.debug("%r: using stream_def_dict", self._platform_id) else: # TODO this case to be removed. stream_definition_ref = stream_config['stream_definition_ref'] self._stream_defs[stream_name] = stream_definition_ref log.debug("%r: using stream_definition_ref", self._platform_id) routing_key = stream_config['routing_key'] stream_id = stream_config['stream_id'] exchange_point = stream_config['exchange_point'] parameter_dictionary = stream_config['parameter_dictionary'] self._data_streams[stream_name] = stream_id self._param_dicts[stream_name] = ParameterDictionary.load(parameter_dictionary) stream_route = StreamRoute(exchange_point=exchange_point, routing_key=routing_key) publisher = self._create_publisher(stream_id, stream_route) self._data_publishers[stream_name] = publisher log.debug("%r: created publisher for stream_name=%r", self._platform_id, stream_name)
def _get_data(cls, config): """ Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len'] @param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count'] """ ext_dset_res = get_safe(config, 'external_dataset_res', None) # Get the Dataset object from the config (should have been instantiated in _init_acquisition_cycle) ds=get_safe(config, 'dataset_object') if ext_dset_res and ds: t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension'] x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension'] y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension'] z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension'] var_lst = ext_dset_res.dataset_description.parameters['variables'] t_slice = get_safe(config, 'constraints.temporal_slice', (slice(0,1))) #TODO: Using 'eval' here is BAD - need to find a less sketchy way to pass constraints if isinstance(t_slice,str): t_slice=eval(t_slice) lon = ds.variables[x_vname][:] lat = ds.variables[y_vname][:] z = ds.variables[z_vname][:] t_arr = ds.variables[t_vname][t_slice] data_arrays = {} for varn in var_lst: data_arrays[varn] = ds.variables[varn][t_slice] max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id', 'unknown data producer') #tx_yml = get_safe(config, 'taxonomy') #ttool = TaxyTool.load(tx_yml) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool pdict = ParameterDictionary.load(get_safe(config, 'param_dictionary')) cnt = calculate_iteration_count(t_arr.size, max_rec) for x in xrange(cnt): ta = t_arr[x*max_rec:(x+1)*max_rec] # Make a 'master' RecDict #rdt = RecordDictionaryTool(taxonomy=ttool) rdt = RecordDictionaryTool(param_dictionary=pdict) # Make a 'coordinate' RecDict #rdt_c = RecordDictionaryTool(taxonomy=ttool) #rdt_c = RecordDictionaryTool(param_dictionary=pdict) # Make a 'data' RecDict #rdt_d = RecordDictionaryTool(taxonomy=ttool) #rdt_d = RecordDictionaryTool(param_dictionary=pdict) # Assign values to the coordinate RecDict rdt[x_vname] = lon rdt[y_vname] = lat rdt[z_vname] = z # Assign values to the data RecDict rdt[t_vname] = ta for key, arr in data_arrays.iteritems(): d = arr[x*max_rec:(x+1)*max_rec] rdt[key] = d # Add the coordinate and data RecDicts to the master RecDict #rdt['coords'] = rdt_c #rdt['data'] = rdt_d # Build and return a granule # CBM: ttool must be passed #g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt) g = build_granule(data_producer_id=dprod_id, record_dictionary=rdt, param_dictionary=pdict) yield g ds.close()
def test_build_granule_and_load_from_granule(self): pdict = ParameterDictionary() t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.reference_frame = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.dtype('float32'))) lat_ctxt.reference_frame = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.dtype('float32'))) lon_ctxt.reference_frame = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.dtype('float32'))) cond_ctxt.uom = 'unknown' pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext('pres', param_type=QuantityType(value_encoding=np.dtype('float32'))) pres_ctxt.uom = 'unknown' pdict.add_context(pres_ctxt) rdt = RecordDictionaryTool(param_dictionary=pdict) #Create some arrays and fill them with random values temp_array = np.random.standard_normal(100) cond_array = np.random.standard_normal(100) pres_array = np.random.standard_normal(100) time_array = np.random.standard_normal(100) lat_array = np.random.standard_normal(100) lon_array = np.random.standard_normal(100) #Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc. rdt['temp'] = temp_array rdt['conductivity'] = cond_array rdt['pres'] = pres_array rdt['time'] = time_array rdt['lat'] = lat_array rdt['lon'] = lon_array g = build_granule(data_producer_id='john', record_dictionary=rdt, param_dictionary=pdict) l_pd = ParameterDictionary.load(g.param_dictionary) #l_tx = TaxyTool.load_from_granule(g) l_rd = RecordDictionaryTool.load_from_granule(g) # Make sure we got back the same Taxonomy Object #self.assertEquals(l_pd, pdict) self.assertEquals(l_pd.ord_from_key('temp'), pdict.ord_from_key('temp')) self.assertEquals(l_pd.ord_from_key('conductivity'), pdict.ord_from_key('conductivity')) # Now test the record dictionary object self.assertEquals(l_rd._rd, rdt._rd) #self.assertEquals(l_rd._param_dict, rdt._param_dict) for k, v in l_rd.iteritems(): self.assertIn(k, rdt) if isinstance(v, np.ndarray): self.assertTrue( (v == rdt[k]).all()) else: self.assertEquals(v._rd, rdt[k]._rd)