def __getitem__(self, name): """ Get an item by nick name from the record dictionary. """ if not self._shp: return None if self._available_fields and name not in self._available_fields: raise KeyError(name) ptype = self._pdict.get_context(name).param_type if isinstance(ptype, ParameterFunctionType): if self._rd[name] is not None and getattr( self._rd[name], 'memoized_values', None) is not None: return self._rd[name].memoized_values[:] try: pfv = get_value_class(ptype, self.domain) pfv._pval_callback = self._pval_callback return pfv[:] except ParameterFunctionException: log.debug('failed to get parameter function field: %s (%s)', name, self._pdict.keys(), exc_info=True) if self._rd[name] is not None: return self._rd[name][:] return None
def get_paramval(cls, ptype, domain, values): paramval = get_value_class(ptype, domain_set=domain) if isinstance(ptype, ParameterFunctionType): paramval.memoized_values = values else: paramval[:] = values paramval.storage._storage.flags.writeable = False return paramval
def get_paramval(cls, ptype, domain, values): paramval = get_value_class(ptype, domain_set=domain) if isinstance(ptype,ParameterFunctionType): paramval.memoized_values = values else: paramval[:] = values paramval.storage._storage.flags.writeable = False return paramval
def test_quantity_values(self): num_rec = 10 dom = cm.SimpleDomainSet((num_rec,)) qtype = ptypes.QuantityType(value_encoding=np.dtype('float32')) qval = cm.get_value_class(qtype, domain_set=dom) data = np.arange(10) qval[:] = data self.assertTrue(np.array_equal(data, qval[:]))
def test_constant_values(self): num_rec = 10 dom = cm.SimpleDomainSet((num_rec,)) ctype = ptypes.ConstantType(ptypes.QuantityType(value_encoding=np.dtype('int32'))) cval = cm.get_value_class(ctype, domain_set=dom) cval[0] = 200 # Doesn't matter what index (or indices) you assign this to - it's used everywhere!! self.assertEqual(cval[0], 200) self.assertEqual(cval[7], 200) self.assertEqual(cval[2,9], 200) self.assertTrue(np.array_equal(cval[[2,7],], np.array([200,200], dtype='int32')))
def test_array_values(self): num_rec = 10 dom = cm.SimpleDomainSet((num_rec,)) atype = ptypes.ArrayType() aval = cm.get_value_class(atype, domain_set=dom) for x in xrange(num_rec): aval[x] = np.random.bytes(np.random.randint(1,20)) # One value (which is a byte string) for each member of the domain self.assertIsInstance(aval[0], str) self.assertTrue(1 <= len(aval[0]) <= 20)
def pval_callback(name, slice_): result = None #search for input data...first level input if name in rdt_in._available_fields: result = rdt_in[name] else: #not first level data so continue to evaluate n,pc = merged_pdict[name] pv = get_value_class(pc.param_type, rdt_in.domain) pv._pval_callback = pval_callback result = pv[:] return result
def get_paramval(cls, ptype, domain, values): paramval = get_value_class(ptype, domain_set=domain) if isinstance(ptype,ParameterFunctionType): paramval.memoized_values = values if isinstance(ptype,SparseConstantType): values = np.atleast_1d(values) spans = cls.spanify(values) paramval.storage._storage = np.array([spans],dtype='object') else: paramval[:] = values paramval.storage._storage.flags.writeable = False return paramval
def get_paramval(cls, ptype, domain, values): paramval = get_value_class(ptype, domain_set=domain) if isinstance(ptype, ParameterFunctionType): paramval.memoized_values = values if isinstance(ptype, SparseConstantType): values = np.atleast_1d(values) spans = cls.spanify(values) paramval.storage._storage = np.array([spans], dtype='object') else: paramval[:] = values paramval.storage._storage.flags.writeable = False return paramval
def test_record_values(self): num_rec = 10 dom = cm.SimpleDomainSet((num_rec,)) rtype = ptypes.RecordType() rval = cm.get_value_class(rtype, domain_set=dom) letts='abcdefghij' for x in xrange(num_rec): rval[x] = {letts[x]: letts[x:]} # One value (which is a dict) for each member of the domain self.assertIsInstance(rval[0], dict)
def _execute_transform(self, msg, streams): stream_in_id,stream_out_id = streams stream_def_in = self._read_stream_def(stream_in_id) stream_def_out = self._read_stream_def(stream_out_id) incoming_pdict_dump = stream_def_in.parameter_dictionary outgoing_pdict_dump = stream_def_out.parameter_dictionary incoming_pdict = ParameterDictionary.load(incoming_pdict_dump) outgoing_pdict = ParameterDictionary.load(outgoing_pdict_dump) merged_pdict = dict([(k,v) for k,v in incoming_pdict.iteritems()] + [(k,v) for k,v in outgoing_pdict.iteritems()]) rdt_in = RecordDictionaryTool.load_from_granule(msg) rdt_out = RecordDictionaryTool(stream_definition_id=stream_def_out._id) #modify the shape of the rdt out since we are using _rd then _shp will never get set rdt_out._shp = rdt_in._shp if rdt_out._available_fields is None: rdt_out._available_fields = [] if rdt_in._available_fields is None: rdt_in._available_fields = [] for key,pctup in merged_pdict.iteritems(): n,pc = pctup #if function then a transform is applied to calculate values if isinstance(pc.param_type, ParameterFunctionType): #apply transform pv = get_value_class(pc.param_type, rdt_in.domain) #recursive function applies values def pval_callback(name, slice_): result = None #search for input data...first level input if name in rdt_in._available_fields: result = rdt_in[name] else: #not first level data so continue to evaluate n,pc = merged_pdict[name] pv = get_value_class(pc.param_type, rdt_in.domain) pv._pval_callback = pval_callback result = pv[:] return result #set the evaluation callback so it can find values in the input stream pv._pval_callback = pval_callback if key in rdt_out._available_fields: #rdt to and from granule wraps result in a paramval so no need #paramval = rdt_out.get_paramval(pc.param_type, rdt_in.domain, pv[:]) #paramval._pval_callback = pval_callback #rdt_out._rd[key] = paramval rdt_out._rd[key] = pv[:] else: #field exists in both the in and the out stream so pass it along to the output stream if key in rdt_in._available_fields and key in rdt_out._available_fields: #pass through rdt_out._rd[key] = rdt_in._rd[key][:] return rdt_out
def test_function_values(self): num_rec = 10 dom = cm.SimpleDomainSet((num_rec,)) ftype = ptypes.FunctionType(ptypes.QuantityType(value_encoding=np.dtype('float32'))) fval = cm.get_value_class(ftype, domain_set=dom) fval[:] = cm.make_range_expr(100, min=0, max=4, min_incl=True, max_incl=False, else_val=-9999) fval[:] = cm.make_range_expr(200, min=4, max=6, min_incl=True, else_val=-9999) fval[:] = cm.make_range_expr(300, min=6, else_val=-9999) self.assertEqual(fval[0], 100) self.assertEqual(fval[5], 200) self.assertEqual(fval[9], 300)
def pval_callback(name, slice_): result = None #search for input data...first level input if name in rdt_in._available_fields: result = rdt_in[name] else: #is it cached if it is then skip recursive evaluation try: result = store[name] except KeyError: #not first level data so continue to evaluate n,pc = merged_pdict[name] pv = get_value_class(pc.param_type, rdt_in.domain) pv._pval_callback = pval_callback result = pv[:] store[name] = result return result
def test_category_values(self): num_rec = 10 dom = cm.SimpleDomainSet((num_rec,)) # CategoryType example cat = {0:'turkey',1:'duck',2:'chicken',99:'None'} cattype = ptypes.CategoryType(categories=cat) catval = cm.get_value_class(cattype, domain_set=dom) catkeys = cat.keys() for x in xrange(num_rec): catval[x] = [random.choice(catkeys)] with self.assertRaises(IndexError): catval[20] self.assertTrue(catval[0] in cat.values())
def __getitem__(self, name): """ Get an item by nick name from the record dictionary. """ if not self._shp: return None if self._available_fields and name not in self._available_fields: raise KeyError(name) ptype = self._pdict.get_context(name).param_type if isinstance(ptype, ParameterFunctionType): if self._rd[name] is not None and getattr(self._rd[name],'memoized_values',None) is not None: return self._rd[name].memoized_values[:] try: pfv = get_value_class(ptype, self.domain) pfv._pval_callback = self._pval_callback return pfv[:] except ParameterFunctionException: log.debug('failed to get parameter function field: %s (%s)', name, self._pdict.keys(), exc_info=True) if self._rd[name] is not None: return self._rd[name][:] return None
def __getitem__(self, name): """ Get an item by nick name from the record dictionary. """ if self._available_fields and name not in self._available_fields: raise KeyError(name) if self._rd[name] is not None: context = self._pdict.get_context(name) if isinstance(context.param_type, ParameterFunctionType): return self._rd[name].memoized_values[:] return self._rd[name][:] ptype = self._pdict.get_context(name).param_type if isinstance(ptype, ParameterFunctionType): try: pfv = get_value_class(ptype, self.domain) pfv._pval_callback = self._pval_callback retval = pfv[:] return retval except Exception as e: log.warning(e.message) return None else: return None