def test_createDataProduct_and_DataProducer_success(self):
# setup
self.clients.resource_registry.find_resources.return_value = ([], 'do not care')
self.clients.resource_registry.find_associations.return_value = []
self.clients.resource_registry.create.return_value = ('SOME_RR_ID1', 'Version_1')
self.clients.data_acquisition_management.assign_data_product.return_value = None
self.clients.pubsub_management.create_stream.return_value = "stream_id"
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 1d spatial topology (station/trajectory)
sdom = sdom.dump()
tdom = tdom.dump()
#@TODO: DO NOT DO THIS, WHEN THIS TEST IS REWRITTEN GET RID OF THIS, IT WILL FAIL, thanks -Luke
parameter_dictionary = get_param_dict('ctd_parsed_param_dict')
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
# test call
dp_id = self.data_product_management_service.create_data_product(data_product=dp_obj,
stream_definition_id='a stream def id')
def _trigger_func(self, stream_id):
log.debug("SimpleCtdDataProducer:_trigger_func ")
parameter_dictionary = get_param_dict('ctd_parsed_param_dict')
rdt = RecordDictionaryTool(param_dictionary=parameter_dictionary)
# The base SimpleCtdPublisher provides a gevent Event that indicates when the process is being
# shut down. We can use a simple pattern here to accomplish both a safe shutdown of this loop
# when the process shuts down *AND* do the timeout between loops in a very safe/efficient fashion.
#
# By using this instead of a sleep in the loop itself, we can immediatly interrupt this loop when
# the process is being shut down instead of having to wait for the sleep to terminate.
while not self.finished.wait(timeout=2):
length = 10
#Explicitly make these numpy arrays...
c = numpy.array(
[random.uniform(0.0, 75.0) for i in xrange(length)])
t = numpy.array(
[random.uniform(-1.7, 21.0) for i in xrange(length)])
p = numpy.array(
[random.lognormvariate(1, 2) for i in xrange(length)])
lat = numpy.array(
[random.uniform(-90.0, 90.0) for i in xrange(length)])
lon = numpy.array(
[random.uniform(0.0, 360.0) for i in xrange(length)])
h = numpy.array(
[random.uniform(0.0, 360.0) for i in xrange(length)])
tvar = numpy.array(
[self.last_time + i for i in xrange(1, length + 1)])
self.last_time = max(tvar)
rdt['time'] = tvar
rdt['lat'] = lat
rdt['lon'] = lon
rdt['temp'] = t
rdt['conductivity'] = c
rdt['pressure'] = p
g = rdt.to_granule()
log.debug('SimpleCtdDataProducer: Sending %d values!' % length)
self.publisher.publish(g)
def _trigger_func(self, stream_id):
log.debug("SimpleCtdDataProducer:_trigger_func ")
parameter_dictionary = get_param_dict("ctd_parsed_param_dict")
rdt = RecordDictionaryTool(param_dictionary=parameter_dictionary)
# The base SimpleCtdPublisher provides a gevent Event that indicates when the process is being
# shut down. We can use a simple pattern here to accomplish both a safe shutdown of this loop
# when the process shuts down *AND* do the timeout between loops in a very safe/efficient fashion.
#
# By using this instead of a sleep in the loop itself, we can immediatly interrupt this loop when
# the process is being shut down instead of having to wait for the sleep to terminate.
while not self.finished.wait(timeout=2):
length = 10
# Explicitly make these numpy arrays...
c = numpy.array([random.uniform(0.0, 75.0) for i in xrange(length)])
t = numpy.array([random.uniform(-1.7, 21.0) for i in xrange(length)])
p = numpy.array([random.lognormvariate(1, 2) for i in xrange(length)])
lat = numpy.array([random.uniform(-90.0, 90.0) for i in xrange(length)])
lon = numpy.array([random.uniform(0.0, 360.0) for i in xrange(length)])
h = numpy.array([random.uniform(0.0, 360.0) for i in xrange(length)])
tvar = numpy.array([self.last_time + i for i in xrange(1, length + 1)])
self.last_time = max(tvar)
rdt["time"] = tvar
rdt["lat"] = lat
rdt["lon"] = lon
rdt["temp"] = t
rdt["conductivity"] = c
rdt["pressure"] = p
g = rdt.to_granule()
log.debug("SimpleCtdDataProducer: Sending %d values!" % length)
self.publisher.publish(g)
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
streams = {
'parsed' : 'ctd_parsed_param_dict',
'raw' : 'ctd_raw_param_dict'
}
for (stream_name, param_dict_name) in streams.iteritems():
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data')
pd = get_param_dict(param_dict_name)
stream_config = dict(stream_route=stream_route,
stream_id=stream_id,
parameter_dictionary=pd.dump())
self._stream_config[stream_name] = stream_config
def test_get_last_update(self):
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 1d spatial topology (station/trajectory)
sdom = sdom.dump()
tdom = tdom.dump()
#@TODO: DO NOT DO THIS, WHEN THIS TEST IS REWRITTEN GET RID OF THIS, IT WILL FAIL, thanks -Luke
parameter_dictionary = get_param_dict('ctd_parsed_param_dict')
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id = self.dpsc_cli.create_data_product(data_product=dp_obj, stream_definition_id=self.stream_def_id, parameter_dictionary=parameter_dictionary)
stream_ids, garbage = self.rrclient.find_objects(data_product_id, PRED.hasStream, id_only=True)
stream_id = stream_ids[0]
fake_lu = LastUpdate()
fake_lu_doc = self.db._ion_object_to_persistence_dict(fake_lu)
self.db.create_doc(fake_lu_doc, object_id=stream_id)
#------------------------------------------
# Now execute
#------------------------------------------
res = self.dpsc_cli.get_last_update(data_product_id=data_product_id)
self.assertTrue(isinstance(res[stream_id], LastUpdate), 'retrieving documents failed')
def test_createDataProduct_and_DataProducer_success(self):
# setup
self.clients.resource_registry.find_resources.return_value = (
[], 'do not care')
self.clients.resource_registry.find_associations.return_value = []
self.clients.resource_registry.create.return_value = ('SOME_RR_ID1',
'Version_1')
self.clients.data_acquisition_management.assign_data_product.return_value = None
self.clients.pubsub_management.create_stream.return_value = "stream_id"
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs,
MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial',
[0]), scrs, MutabilityEnum.IMMUTABLE
) # 1d spatial topology (station/trajectory)
sdom = sdom.dump()
tdom = tdom.dump()
#@TODO: DO NOT DO THIS, WHEN THIS TEST IS REWRITTEN GET RID OF THIS, IT WILL FAIL, thanks -Luke
parameter_dictionary = get_param_dict('ctd_parsed_param_dict')
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain=tdom,
spatial_domain=sdom)
# test call
dp_id = self.data_product_management_service.create_data_product(
data_product=dp_obj, stream_definition_id='a stream def id')
