def _get_data(cls, config): parser = get_safe(config, 'parser', None) ext_dset_res = get_safe(config, 'external_dataset_res', None) if ext_dset_res and parser: #CBM: Not in use yet... # t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension'] # x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension'] # y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension'] # z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension'] # var_lst = ext_dset_res.dataset_description.parameters['variables'] max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id', 'unknown data producer') tx_yml = get_safe(config, 'taxonomy') ttool = TaxyTool.load(tx_yml) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool cnt = cls._calc_iter_cnt(len(parser.sensor_map), max_rec) for x in xrange(cnt): rdt = RecordDictionaryTool(taxonomy=ttool) for name in parser.sensor_map: d = parser.data_map[name][x*max_rec:(x+1)*max_rec] rdt[name]=d g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt) yield g else: log.warn('No parser object found in config')
def _get_data(cls, config): parser = get_safe(config, 'parser', None) ext_dset_res = get_safe(config, 'external_dataset_res', None) if ext_dset_res and parser: #CBM: Not in use yet... # t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension'] # x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension'] # y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension'] # z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension'] # var_lst = ext_dset_res.dataset_description.parameters['variables'] max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id', 'unknown data producer') tx_yml = get_safe(config, 'taxonomy') ttool = TaxyTool.load( tx_yml ) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool cnt = cls._calc_iter_cnt(len(parser.sensor_map), max_rec) for x in xrange(cnt): rdt = RecordDictionaryTool(taxonomy=ttool) for name in parser.sensor_map: d = parser.data_map[name][x * max_rec:(x + 1) * max_rec] rdt[name] = d g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt) yield g else: log.warn('No parser object found in config')
def _get_data(cls, config): """ Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len'] @param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count'] """ array_len = get_safe(config, 'constraints.array_len',1) max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id') tx_yml = get_safe(config, 'taxonomy') ttool = TaxyTool.load(tx_yml) arr = npr.random_sample(array_len) log.debug('Array to send using max_rec={0}: {1}'.format(max_rec, arr)) cnt = cls._calc_iter_cnt(arr.size, max_rec) for x in xrange(cnt): rdt = RecordDictionaryTool(taxonomy=ttool) d = arr[x*max_rec:(x+1)*max_rec] rdt['data'] = d g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt) yield g
def test_yamlize(self): tc = TaxyTool() tc.add_taxonomy_set('1', 'a') tc.add_taxonomy_set('2', 'b') tc.extend_names_by_nick_name('1', 'x') tc.extend_names_by_anyname('a', 'z') tc.extend_names_by_anyname('b', 'c') s = tc.dump() tc2 = TaxyTool.load(s) #@todo - a list is not a set and the yaml dump/ion serialization can not handle sets... self.assertEquals(tc2._cnt, 1) self.assertEquals(tc2.get_names_by_handle(0), { '1', 'x', 'a', 'z', }) self.assertEquals(tc2.get_names_by_handle(1), { '2', 'b', 'c', }) self.assertEquals(tc._cnt, 1) self.assertEquals(tc.get_names_by_handle(0), { '1', 'x', 'a', 'z', }) self.assertEquals(tc.get_names_by_handle(1), { '2', 'b', 'c', })
def _get_data(cls, config): """ A generator that retrieves config['constraints']['count'] number of sequential Fibonacci numbers @param config Dict of configuration parameters - must contain ['constraints']['count'] """ cnt = get_safe(config, 'constraints.count', 1) max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id') tx_yml = get_safe(config, 'taxonomy') ttool = TaxyTool.load(tx_yml) def fibGenerator(): """ A Fibonacci sequence generator """ count = 0 ret = [] a, b = 1, 1 while 1: count += 1 ret.append(a) if count == max_rec: yield np.array(ret) ret = [] count = 0 a, b = b, a + b gen = fibGenerator() cnt = cls._calc_iter_cnt(cnt, max_rec) for i in xrange(cnt): rdt = RecordDictionaryTool(taxonomy=ttool) d = gen.next() rdt['data'] = d g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt) yield g
def test_yamlize(self): tc = TaxyTool() tc.add_taxonomy_set('1','a') tc.add_taxonomy_set('2','b') tc.extend_names_by_nick_name('1','x') tc.extend_names_by_anyname('a','z') tc.extend_names_by_anyname('b','c') s = tc.dump() tc2 = TaxyTool.load(s) #@todo - a list is not a set and the yaml dump/ion serialization can not handle sets... self.assertEquals(tc2._cnt,1) self.assertEquals(tc2.get_names_by_handle(0),{'1','x','a','z',}) self.assertEquals(tc2.get_names_by_handle(1),{'2','b','c',}) self.assertEquals(tc._cnt,1) self.assertEquals(tc.get_names_by_handle(0),{'1','x','a','z',}) self.assertEquals(tc.get_names_by_handle(1),{'2','b','c',})
def _get_data(cls, config): """ Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len'] @param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count'] """ array_len = get_safe(config, 'constraints.array_len', 1) max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id') tx_yml = get_safe(config, 'taxonomy') ttool = TaxyTool.load(tx_yml) arr = npr.random_sample(array_len) log.debug('Array to send using max_rec={0}: {1}'.format(max_rec, arr)) cnt = cls._calc_iter_cnt(arr.size, max_rec) for x in xrange(cnt): rdt = RecordDictionaryTool(taxonomy=ttool) d = arr[x * max_rec:(x + 1) * max_rec] rdt['data'] = d g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt) yield g
def _get_data(cls, config): """ A generator that retrieves config['constraints']['count'] number of sequential Fibonacci numbers @param config Dict of configuration parameters - must contain ['constraints']['count'] """ cnt = get_safe(config,'constraints.count',1) max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id') tx_yml = get_safe(config, 'taxonomy') ttool = TaxyTool.load(tx_yml) def fibGenerator(): """ A Fibonacci sequence generator """ count = 0 ret = [] a, b = 1, 1 while 1: count += 1 ret.append(a) if count == max_rec: yield np.array(ret) ret=[] count = 0 a, b = b, a + b gen=fibGenerator() cnt = cls._calc_iter_cnt(cnt, max_rec) for i in xrange(cnt): rdt = RecordDictionaryTool(taxonomy=ttool) d = gen.next() rdt['data'] = d g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt) yield g
def _get_data(cls, config): """ Retrieves config['constraints']['count'] number of random samples of length config['constraints']['array_len'] @param config Dict of configuration parameters - must contain ['constraints']['count'] and ['constraints']['count'] """ ext_dset_res = get_safe(config, 'external_dataset_res', None) # Get the Dataset object from the config (should have been instantiated in _init_acquisition_cycle) ds=get_safe(config, 'dataset_object') if ext_dset_res and ds: t_vname = ext_dset_res.dataset_description.parameters['temporal_dimension'] x_vname = ext_dset_res.dataset_description.parameters['zonal_dimension'] y_vname = ext_dset_res.dataset_description.parameters['meridional_dimension'] z_vname = ext_dset_res.dataset_description.parameters['vertical_dimension'] var_lst = ext_dset_res.dataset_description.parameters['variables'] t_slice = get_safe(config, 'constraints.temporal_slice', (slice(0,1))) #TODO: Using 'eval' here is BAD - need to find a less sketchy way to pass constraints if isinstance(t_slice,str): t_slice=eval(t_slice) lon = ds.variables[x_vname][:] lat = ds.variables[y_vname][:] z = ds.variables[z_vname][:] t_arr = ds.variables[t_vname][t_slice] data_arrays = {} for varn in var_lst: data_arrays[varn] = ds.variables[varn][t_slice] max_rec = get_safe(config, 'max_records', 1) dprod_id = get_safe(config, 'data_producer_id', 'unknown data producer') tx_yml = get_safe(config, 'taxonomy') ttool = TaxyTool.load(tx_yml) #CBM: Assertion inside RDT.__setitem__ requires same instance of TaxyTool cnt = cls._calc_iter_cnt(t_arr.size, max_rec) for x in xrange(cnt): ta = t_arr[x*max_rec:(x+1)*max_rec] # Make a 'master' RecDict rdt = RecordDictionaryTool(taxonomy=ttool) # Make a 'coordinate' RecDict rdt_c = RecordDictionaryTool(taxonomy=ttool) # Make a 'data' RecDict rdt_d = RecordDictionaryTool(taxonomy=ttool) # Assign values to the coordinate RecDict rdt_c[x_vname] = lon rdt_c[y_vname] = lat rdt_c[z_vname] = z # Assign values to the data RecDict rdt_d[t_vname] = ta for key, arr in data_arrays.iteritems(): d = arr[x*max_rec:(x+1)*max_rec] rdt_d[key] = d # Add the coordinate and data RecDicts to the master RecDict rdt['coords'] = rdt_c rdt['data'] = rdt_d # Build and return a granule # CBM: ttool must be passed g = build_granule(data_producer_id=dprod_id, taxonomy=ttool, record_dictionary=rdt) yield g ds.close()