def __getitem__(self, name):
"""
Get an item by nick name from the record dictionary.
"""
if not self._shp:
return None
if self._available_fields and name not in self._available_fields:
raise KeyError(name)
ptype = self._pdict.get_context(name).param_type
if isinstance(ptype, ParameterFunctionType):
if self._rd[name] is not None and getattr(
self._rd[name], 'memoized_values', None) is not None:
return self._rd[name].memoized_values[:]
try:
pfv = get_value_class(ptype, self.domain)
pfv._pval_callback = self._pval_callback
return pfv[:]
except ParameterFunctionException:
log.debug('failed to get parameter function field: %s (%s)',
name,
self._pdict.keys(),
exc_info=True)
if self._rd[name] is not None:
return self._rd[name][:]
return None
def get_paramval(cls, ptype, domain, values):
paramval = get_value_class(ptype, domain_set=domain)
if isinstance(ptype, ParameterFunctionType):
paramval.memoized_values = values
else:
paramval[:] = values
paramval.storage._storage.flags.writeable = False
return paramval
def get_paramval(cls, ptype, domain, values):
paramval = get_value_class(ptype, domain_set=domain)
if isinstance(ptype,ParameterFunctionType):
paramval.memoized_values = values
else:
paramval[:] = values
paramval.storage._storage.flags.writeable = False
return paramval
def test_quantity_values(self):
num_rec = 10
dom = cm.SimpleDomainSet((num_rec,))
qtype = ptypes.QuantityType(value_encoding=np.dtype('float32'))
qval = cm.get_value_class(qtype, domain_set=dom)
data = np.arange(10)
qval[:] = data
self.assertTrue(np.array_equal(data, qval[:]))
def test_constant_values(self):
num_rec = 10
dom = cm.SimpleDomainSet((num_rec,))
ctype = ptypes.ConstantType(ptypes.QuantityType(value_encoding=np.dtype('int32')))
cval = cm.get_value_class(ctype, domain_set=dom)
cval[0] = 200 # Doesn't matter what index (or indices) you assign this to - it's used everywhere!!
self.assertEqual(cval[0], 200)
self.assertEqual(cval[7], 200)
self.assertEqual(cval[2,9], 200)
self.assertTrue(np.array_equal(cval[[2,7],], np.array([200,200], dtype='int32')))
def test_array_values(self):
num_rec = 10
dom = cm.SimpleDomainSet((num_rec,))
atype = ptypes.ArrayType()
aval = cm.get_value_class(atype, domain_set=dom)
for x in xrange(num_rec):
aval[x] = np.random.bytes(np.random.randint(1,20)) # One value (which is a byte string) for each member of the domain
self.assertIsInstance(aval[0], str)
self.assertTrue(1 <= len(aval[0]) <= 20)
def pval_callback(name, slice_):
result = None
#search for input data...first level input
if name in rdt_in._available_fields:
result = rdt_in[name]
else:
#not first level data so continue to evaluate
n,pc = merged_pdict[name]
pv = get_value_class(pc.param_type, rdt_in.domain)
pv._pval_callback = pval_callback
result = pv[:]
return result
def get_paramval(cls, ptype, domain, values):
paramval = get_value_class(ptype, domain_set=domain)
if isinstance(ptype,ParameterFunctionType):
paramval.memoized_values = values
if isinstance(ptype,SparseConstantType):
values = np.atleast_1d(values)
spans = cls.spanify(values)
paramval.storage._storage = np.array([spans],dtype='object')
else:
paramval[:] = values
paramval.storage._storage.flags.writeable = False
return paramval
def get_paramval(cls, ptype, domain, values):
paramval = get_value_class(ptype, domain_set=domain)
if isinstance(ptype, ParameterFunctionType):
paramval.memoized_values = values
if isinstance(ptype, SparseConstantType):
values = np.atleast_1d(values)
spans = cls.spanify(values)
paramval.storage._storage = np.array([spans], dtype='object')
else:
paramval[:] = values
paramval.storage._storage.flags.writeable = False
return paramval
def test_record_values(self):
num_rec = 10
dom = cm.SimpleDomainSet((num_rec,))
rtype = ptypes.RecordType()
rval = cm.get_value_class(rtype, domain_set=dom)
letts='abcdefghij'
for x in xrange(num_rec):
rval[x] = {letts[x]: letts[x:]} # One value (which is a dict) for each member of the domain
self.assertIsInstance(rval[0], dict)
def _execute_transform(self, msg, streams):
stream_in_id,stream_out_id = streams
stream_def_in = self._read_stream_def(stream_in_id)
stream_def_out = self._read_stream_def(stream_out_id)
incoming_pdict_dump = stream_def_in.parameter_dictionary
outgoing_pdict_dump = stream_def_out.parameter_dictionary
incoming_pdict = ParameterDictionary.load(incoming_pdict_dump)
outgoing_pdict = ParameterDictionary.load(outgoing_pdict_dump)
merged_pdict = dict([(k,v) for k,v in incoming_pdict.iteritems()] + [(k,v) for k,v in outgoing_pdict.iteritems()])
rdt_in = RecordDictionaryTool.load_from_granule(msg)
rdt_out = RecordDictionaryTool(stream_definition_id=stream_def_out._id)
#modify the shape of the rdt out since we are using _rd then _shp will never get set
rdt_out._shp = rdt_in._shp
if rdt_out._available_fields is None: rdt_out._available_fields = []
if rdt_in._available_fields is None: rdt_in._available_fields = []
for key,pctup in merged_pdict.iteritems():
n,pc = pctup
#if function then a transform is applied to calculate values
if isinstance(pc.param_type, ParameterFunctionType):
#apply transform
pv = get_value_class(pc.param_type, rdt_in.domain)
#recursive function applies values
def pval_callback(name, slice_):
result = None
#search for input data...first level input
if name in rdt_in._available_fields:
result = rdt_in[name]
else:
#not first level data so continue to evaluate
n,pc = merged_pdict[name]
pv = get_value_class(pc.param_type, rdt_in.domain)
pv._pval_callback = pval_callback
result = pv[:]
return result
#set the evaluation callback so it can find values in the input stream
pv._pval_callback = pval_callback
if key in rdt_out._available_fields:
#rdt to and from granule wraps result in a paramval so no need
#paramval = rdt_out.get_paramval(pc.param_type, rdt_in.domain, pv[:])
#paramval._pval_callback = pval_callback
#rdt_out._rd[key] = paramval
rdt_out._rd[key] = pv[:]
else:
#field exists in both the in and the out stream so pass it along to the output stream
if key in rdt_in._available_fields and key in rdt_out._available_fields:
#pass through
rdt_out._rd[key] = rdt_in._rd[key][:]
return rdt_out
def test_function_values(self):
num_rec = 10
dom = cm.SimpleDomainSet((num_rec,))
ftype = ptypes.FunctionType(ptypes.QuantityType(value_encoding=np.dtype('float32')))
fval = cm.get_value_class(ftype, domain_set=dom)
fval[:] = cm.make_range_expr(100, min=0, max=4, min_incl=True, max_incl=False, else_val=-9999)
fval[:] = cm.make_range_expr(200, min=4, max=6, min_incl=True, else_val=-9999)
fval[:] = cm.make_range_expr(300, min=6, else_val=-9999)
self.assertEqual(fval[0], 100)
self.assertEqual(fval[5], 200)
self.assertEqual(fval[9], 300)
def pval_callback(name, slice_):
result = None
#search for input data...first level input
if name in rdt_in._available_fields:
result = rdt_in[name]
else:
#is it cached if it is then skip recursive evaluation
try:
result = store[name]
except KeyError:
#not first level data so continue to evaluate
n,pc = merged_pdict[name]
pv = get_value_class(pc.param_type, rdt_in.domain)
pv._pval_callback = pval_callback
result = pv[:]
store[name] = result
return result
def test_category_values(self):
num_rec = 10
dom = cm.SimpleDomainSet((num_rec,))
# CategoryType example
cat = {0:'turkey',1:'duck',2:'chicken',99:'None'}
cattype = ptypes.CategoryType(categories=cat)
catval = cm.get_value_class(cattype, domain_set=dom)
catkeys = cat.keys()
for x in xrange(num_rec):
catval[x] = [random.choice(catkeys)]
with self.assertRaises(IndexError):
catval[20]
self.assertTrue(catval[0] in cat.values())
def __getitem__(self, name):
"""
Get an item by nick name from the record dictionary.
"""
if not self._shp:
return None
if self._available_fields and name not in self._available_fields:
raise KeyError(name)
ptype = self._pdict.get_context(name).param_type
if isinstance(ptype, ParameterFunctionType):
if self._rd[name] is not None and getattr(self._rd[name],'memoized_values',None) is not None:
return self._rd[name].memoized_values[:]
try:
pfv = get_value_class(ptype, self.domain)
pfv._pval_callback = self._pval_callback
return pfv[:]
except ParameterFunctionException:
log.debug('failed to get parameter function field: %s (%s)', name, self._pdict.keys(), exc_info=True)
if self._rd[name] is not None:
return self._rd[name][:]
return None
def __getitem__(self, name):
"""
Get an item by nick name from the record dictionary.
"""
if self._available_fields and name not in self._available_fields:
raise KeyError(name)
if self._rd[name] is not None:
context = self._pdict.get_context(name)
if isinstance(context.param_type, ParameterFunctionType):
return self._rd[name].memoized_values[:]
return self._rd[name][:]
ptype = self._pdict.get_context(name).param_type
if isinstance(ptype, ParameterFunctionType):
try:
pfv = get_value_class(ptype, self.domain)
pfv._pval_callback = self._pval_callback
retval = pfv[:]
return retval
except Exception as e:
log.warning(e.message)
return None
else:
return None
def __getitem__(self, name):
"""
Get an item by nick name from the record dictionary.
"""
if self._available_fields and name not in self._available_fields:
raise KeyError(name)
if self._rd[name] is not None:
context = self._pdict.get_context(name)
if isinstance(context.param_type, ParameterFunctionType):
return self._rd[name].memoized_values[:]
return self._rd[name][:]
ptype = self._pdict.get_context(name).param_type
if isinstance(ptype, ParameterFunctionType):
try:
pfv = get_value_class(ptype, self.domain)
pfv._pval_callback = self._pval_callback
retval = pfv[:]
return retval
except Exception as e:
log.warning(e.message)
return None
else:
return None
