def granule_example(nsdict):
"""
Usage:
from examples.granule import granule_example
granule_example(locals())
tx, g, rdt, rdt2... etc, are now local variables in your shell!
"""
#Define a taxonomy and add sets. add_taxonomy_set takes one or more names and assigns them to one handle
tx = TaxyTool()
tx.add_taxonomy_set('temp', 'long_temp_name')
tx.add_taxonomy_set('cond', 'long_cond_name')
tx.add_taxonomy_set('pres', 'long_pres_name')
tx.add_taxonomy_set('rdt')
# map is {<local name>: <granule name or path>}
#Use RecordDictionaryTool to create a record dictionary. Send in the taxonomy so the Tool knows what to expect
rdt = RecordDictionaryTool(taxonomy=tx)
#Create some arrays and fill them with random values
temp_array = numpy.random.standard_normal(100)
cond_array = numpy.random.standard_normal(100)
pres_array = numpy.random.standard_normal(100)
#Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc.
rdt['temp'] = temp_array
rdt['cond'] = cond_array
rdt['pres'] = pres_array
#You can also add in another RecordDictionaryTool, providing the taxonomies are the same.
rdt2 = RecordDictionaryTool(taxonomy=tx)
rdt2['temp'] = temp_array
rdt['rdt'] = rdt2
#you can get a string representation of the RecordDictionaryTool
print rdt
print rdt.pretty_print()
#Determine the length of the RecordDictionary using the len function
print len(rdt)
#Delete an item in the RecordDictionary
del rdt['temp']
g = build_granule(data_producer_id='john', taxonomy=tx, record_dictionary=rdt)
nsdict.update(locals())
def _trigger_func(self, stream_id):
#@todo - add lots of comments in here
while True:
length = 10
#Explicitly make these numpy arrays...
c = numpy.array([random.uniform(0.0,75.0) for i in xrange(length)])
t = numpy.array([random.uniform(-1.7, 21.0) for i in xrange(length)])
p = numpy.array([random.lognormvariate(1,2) for i in xrange(length)])
lat = numpy.array([random.uniform(-90.0, 90.0) for i in xrange(length)])
lon = numpy.array([random.uniform(0.0, 360.0) for i in xrange(length)])
tvar = numpy.array([self.last_time + i for i in xrange(1,length+1)])
self.last_time = max(tvar)
rdt = RecordDictionaryTool(taxonomy=tx)
# This is an example of using groups it is not a normative statement about how to use groups
rdt0 = RecordDictionaryTool(taxonomy=tx)
rdt0['temp'] = t
rdt0['cond'] = c
rdt0['pres'] = p
#add a value sequence of raw bytes - not sure the type below is correct?
with open('/dev/urandom','r') as rand:
rdt0['raw_fixed'] = numpy.array([rand.read(32) for i in xrange(length)], dtype='a32')
#add a value sequence of raw bytes - not sure the type below is correct?
with open('/dev/urandom','r') as rand:
rdt0['raw_blob'] = numpy.array([rand.read(random.randint(1,40)) for i in xrange(length)], dtype=object)
rdt1 = RecordDictionaryTool(taxonomy=tx)
rdt1['time'] = tvar
rdt1['lat'] = lat
rdt1['lon'] = lon
rdt['group1'] = rdt1
rdt['group0'] = rdt0
log.info("logging published Record Dictionary:\n %s", rdt.pretty_print())
g = build_granule(data_producer_id='Bobs Potatoes', taxonomy=tx, record_dictionary=rdt)
log.info('Sending %d values!' % length)
self.publisher.publish(g)
time.sleep(2.0)
def _trigger_func(self, stream_id):
self.last_time = 0
#@todo - add lots of comments in here
while True:
length = 10
#Explicitly make these numpy arrays...
c = numpy.array([random.uniform(0.0,75.0) for i in xrange(length)])
t = numpy.array([random.uniform(-1.7, 21.0) for i in xrange(length)])
p = numpy.array([random.lognormvariate(1,2) for i in xrange(length)])
lat = numpy.array([random.uniform(-90.0, 90.0) for i in xrange(length)])
lon = numpy.array([random.uniform(0.0, 360.0) for i in xrange(length)])
tvar = numpy.array([self.last_time + i for i in xrange(1,length+1)])
self.last_time = max(tvar)
rdt = RecordDictionaryTool(taxonomy=self._tx)
rdt['temp'] = ExampleDataProducer_algorithm.execute(t)
rdt['cond'] = ExampleDataProducer_algorithm.execute(c)
rdt['pres'] = ExampleDataProducer_algorithm.execute(p)
rdt['time'] = tvar
rdt['lat'] = lat
rdt['lon'] = lon
log.info("logging published Record Dictionary:\n %s", rdt.pretty_print())
g = build_granule(data_producer_id=stream_id, taxonomy=self._tx, record_dictionary=rdt)
log.info('Sending %d values!' % length)
if(isinstance(g, Granule)):
self.publish(g, stream_id)
time.sleep(2.0)
class RecordDictionaryToolTestCase(unittest.TestCase):
def setUp(self):
self._tx = TaxyTool()
self._tx.add_taxonomy_set('temp', 'long_temp_name')
self._tx.add_taxonomy_set('cond', 'long_cond_name')
self._tx.add_taxonomy_set('pres', 'long_pres_name')
self._tx.add_taxonomy_set('rdt')
self._tx.add_taxonomy_set('rdt2')
# map is {<local name>: <granule name or path>}
self._rdt = RecordDictionaryTool(taxonomy=self._tx)
def test_init(self):
# initialize with a taxonomy tool
rdt = RecordDictionaryTool(taxonomy=self._tx)
self.assertIsInstance(rdt._tx, TaxyTool)
# initialize with a taxonomy object
rdt = RecordDictionaryTool(taxonomy=self._tx._t)
self.assertIsInstance(rdt._tx, TaxyTool)
# initialize with pooo
self.assertRaises(TypeError, RecordDictionaryTool, ['foo', 'barr'])
# initialize with a valid shape
rdt = RecordDictionaryTool(taxonomy=self._tx, shape=(5,2))
self.assertEquals(rdt._shp, (5,2))
rdt = RecordDictionaryTool(taxonomy=self._tx, shape=(5,))
self.assertEquals(rdt._shp, (5,))
rdt = RecordDictionaryTool(taxonomy=self._tx, shape=5)
self.assertEquals(rdt._shp, (5,))
# initialize with no length
rdt = RecordDictionaryTool(taxonomy=self._tx)
self.assertEquals(rdt._shp, None)
# initialize with pooo
self.assertRaises(TypeError, RecordDictionaryTool, self._tx, 'not an int')
def test_set_and_get(self):
#make sure you can set and get items in the granule by name in the taxonomy
temp_array = numpy.random.standard_normal(100)
cond_array = numpy.random.standard_normal(100)
pres_array = numpy.random.standard_normal(100)
self.assertRaises(KeyError, self._rdt.__setitem__, 'long_temp_name',temp_array)
self.assertRaises(KeyError, self._rdt.__setitem__, 'nonsense',temp_array)
self._rdt['temp'] = temp_array
self._rdt['cond'] = cond_array
self._rdt['pres'] = pres_array
self.assertTrue(numpy.allclose(self._rdt['temp'], temp_array))
self.assertTrue(numpy.allclose(self._rdt['cond'], cond_array))
self.assertTrue(numpy.allclose(self._rdt['pres'], pres_array))
#want to check to make sure a KeyError is raised when a non-nickname key is used, but it's not working correctly
self.assertRaises(KeyError, self._rdt.__getitem__, 'long_temp_name')
self.assertRaises(KeyError, self._rdt.__getitem__,'nonsense!')
taxy_tool_obj =self._tx
rdt = RecordDictionaryTool(taxonomy=taxy_tool_obj)
rdt['temp'] = temp_array
self._rdt['rdt'] = rdt
# Now test when the Record Dictionary Tool is created with the Taxonomy object rather than the TaxyTool
# This can fail if the == method for TaxyTool is implemented incorrectly
taxonomy_ion_obj = self._tx._t
rdt2 = RecordDictionaryTool(taxonomy=taxonomy_ion_obj)
rdt2['temp'] = temp_array
self._rdt['rdt2'] = rdt2
# Now test bad values... list not numpy array...
with self.assertRaises(TypeError) as te:
rdt2['temp'] = [1,2,3]
self.assertEquals(
te.exception.message,
'''Invalid type "<type 'list'>" in Record Dictionary Tool setitem with name "temp". Valid types are numpy.ndarray and RecordDictionaryTool'''
)
# Now test numpy scalar array...
with self.assertRaises(TypeError) as te:
rdt2['temp'] = numpy.float32(3.14159)
self.assertEquals(
te.exception.message,
'''Invalid type "<type 'numpy.float32'>" in Record Dictionary Tool setitem with name "temp". Valid types are numpy.ndarray and RecordDictionaryTool'''
)
# Now test rank zero array...
with self.assertRaises(ValueError) as te:
rdt2['temp'] = numpy.array(22.5)
self.assertEquals(
te.exception.message,
'''The rank of a value sequence array in a record dictionary must be greater than zero. Got name "temp" with rank "0"'''
)
# Test set invalid shape
pres_array = numpy.random.standard_normal(90)
with self.assertRaises(ValueError) as te:
rdt2['pres'] = pres_array
self.assertEquals(
te.exception.message,
'''Invalid array shape "(90,)" for name "pres"; Record dictionary defined shape is "(100,)"'''
)
# make a new RDT for testing higher rank objects...
taxy_tool_obj =self._tx
rdt = RecordDictionaryTool(taxonomy=taxy_tool_obj)
# Now test rank 2 array...
rdt['temp'] = numpy.array([[22.5,],])
self.assertTrue((rdt['temp'] == numpy.array([[22.5,],])).all())
# Now test rank 2 array...
rdt['cond'] = numpy.array([[28.5,],])
self.assertTrue((rdt['cond'] == numpy.array([[28.5,],])).all())
def test_iteration(self):
#Test all four iteration methods for items in the granule
temp_array = numpy.random.standard_normal(100)
cond_array = numpy.random.standard_normal(100)
pres_array = numpy.random.standard_normal(100)
self._rdt['temp'] = temp_array
self._rdt['cond'] = cond_array
self._rdt['pres'] = pres_array
for k, v in self._rdt.iteritems():
if k == 'temp':
self.assertTrue(numpy.allclose(temp_array, v))
elif k == 'cond':
self.assertTrue(numpy.allclose(cond_array, v))
elif k == 'pres':
self.assertTrue(numpy.allclose(pres_array, v))
else:
self.assertTrue(False)
for k in self._rdt.iterkeys():
self.assertTrue(k == 'temp' or k == 'cond' or k == 'pres')
for v in self._rdt.itervalues():
self.assertTrue(numpy.allclose(temp_array, v) or numpy.allclose(cond_array, v) or numpy.allclose(pres_array, v))
for k in self._rdt:
self.assertTrue(k == 'temp' or k == 'cond' or k == 'pres')
def test_update(self):
# Update this granule with the content of another. Assert that the taxonomies are the same...
pres_array = numpy.random.standard_normal(100)
self._rdt['pres'] = pres_array
rdt2 = RecordDictionaryTool(taxonomy=self._tx)
temp_array = numpy.random.standard_normal(100)
cond_array = numpy.random.standard_normal(100)
rdt2['temp'] = temp_array
rdt2['cond'] = cond_array
self._rdt.update(rdt2)
self.assertIn('pres', self._rdt)
self.assertIn('temp', self._rdt)
self.assertIn('cond', self._rdt)
self.assertTrue((self._rdt['pres'] == pres_array).all())
self.assertTrue((self._rdt['cond'] == cond_array).all())
self.assertTrue((self._rdt['temp'] == temp_array).all())
self.assertEquals(len(self._rdt), 3)
def test_len(self):
temp_array = numpy.random.standard_normal(100)
cond_array = numpy.random.standard_normal(100)
pres_array = numpy.random.standard_normal(100)
self._rdt['temp'] = temp_array
self._rdt['cond'] = cond_array
self._rdt['pres'] = pres_array
self.assertEquals(len(self._rdt), 3)
def test_repr(self):
# Come up with a reasonable string representation of the granule for debug purposes only
temp_array = numpy.random.standard_normal(100)
cond_array = numpy.random.standard_normal(100)
pres_array = numpy.random.standard_normal(100)
self._rdt['temp'] = temp_array
self._rdt['cond'] = cond_array
self._rdt['pres'] = pres_array
self.assertTrue(len(repr(self._rdt)) > 0)
def test_delete(self):
temp_array = numpy.random.standard_normal(100)
cond_array = numpy.random.standard_normal(100)
pres_array = numpy.random.standard_normal(100)
self._rdt['temp'] = temp_array
self._rdt['cond'] = cond_array
self._rdt['pres'] = pres_array
self.assertIn('pres', self._rdt)
self.assertIn('temp', self._rdt)
self.assertIn('cond', self._rdt)
del self._rdt['pres']
self.assertNotIn('pres', self._rdt)
self.assertIn('temp', self._rdt)
self.assertIn('cond', self._rdt)
def test_contains(self):
# foobar isn't even in the taxonomy!
self.assertNotIn('foobar', self._rdt)
# Temp is in the taxonomy but not the record dictionary
self.assertNotIn('temp', self._rdt)
# Now put in some data and make sure it works...
temp_array = numpy.random.standard_normal(100)
self._rdt['temp'] = temp_array
self.assertIn('temp', self._rdt)
def test_pretty_print(self):
temp_array = numpy.random.standard_normal(100)
cond_array = numpy.random.standard_normal(100)
pres_array = numpy.random.standard_normal(100)
self._rdt['temp'] = temp_array
self._rdt['cond'] = cond_array
self._rdt['pres'] = pres_array
rdt = RecordDictionaryTool(taxonomy=self._tx)
rdt['rdt'] = temp_array
self._rdt['rdt'] = rdt
self.assertGreater(len(self._rdt.pretty_print()), 0)
