def test_createDataProduct_and_DataProducer_success(self): # setup self.clients.resource_registry.find_resources.return_value = ([], 'do not care') self.clients.resource_registry.find_associations.return_value = [] self.clients.resource_registry.create.return_value = ('SOME_RR_ID1', 'Version_1') self.clients.data_acquisition_management.assign_data_product.return_value = None self.clients.pubsub_management.create_stream.return_value = "stream_id" # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 1d spatial topology (station/trajectory) sdom = sdom.dump() tdom = tdom.dump() #@TODO: DO NOT DO THIS, WHEN THIS TEST IS REWRITTEN GET RID OF THIS, IT WILL FAIL, thanks -Luke parameter_dictionary = get_param_dict('ctd_parsed_param_dict') parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) # test call dp_id = self.data_product_management_service.create_data_product(data_product=dp_obj, stream_definition_id='a stream def id')
def _trigger_func(self, stream_id): log.debug("SimpleCtdDataProducer:_trigger_func ") parameter_dictionary = get_param_dict('ctd_parsed_param_dict') rdt = RecordDictionaryTool(param_dictionary=parameter_dictionary) # The base SimpleCtdPublisher provides a gevent Event that indicates when the process is being # shut down. We can use a simple pattern here to accomplish both a safe shutdown of this loop # when the process shuts down *AND* do the timeout between loops in a very safe/efficient fashion. # # By using this instead of a sleep in the loop itself, we can immediatly interrupt this loop when # the process is being shut down instead of having to wait for the sleep to terminate. while not self.finished.wait(timeout=2): length = 10 #Explicitly make these numpy arrays... c = numpy.array( [random.uniform(0.0, 75.0) for i in xrange(length)]) t = numpy.array( [random.uniform(-1.7, 21.0) for i in xrange(length)]) p = numpy.array( [random.lognormvariate(1, 2) for i in xrange(length)]) lat = numpy.array( [random.uniform(-90.0, 90.0) for i in xrange(length)]) lon = numpy.array( [random.uniform(0.0, 360.0) for i in xrange(length)]) h = numpy.array( [random.uniform(0.0, 360.0) for i in xrange(length)]) tvar = numpy.array( [self.last_time + i for i in xrange(1, length + 1)]) self.last_time = max(tvar) rdt['time'] = tvar rdt['lat'] = lat rdt['lon'] = lon rdt['temp'] = t rdt['conductivity'] = c rdt['pressure'] = p g = rdt.to_granule() log.debug('SimpleCtdDataProducer: Sending %d values!' % length) self.publisher.publish(g)
def _trigger_func(self, stream_id): log.debug("SimpleCtdDataProducer:_trigger_func ") parameter_dictionary = get_param_dict("ctd_parsed_param_dict") rdt = RecordDictionaryTool(param_dictionary=parameter_dictionary) # The base SimpleCtdPublisher provides a gevent Event that indicates when the process is being # shut down. We can use a simple pattern here to accomplish both a safe shutdown of this loop # when the process shuts down *AND* do the timeout between loops in a very safe/efficient fashion. # # By using this instead of a sleep in the loop itself, we can immediatly interrupt this loop when # the process is being shut down instead of having to wait for the sleep to terminate. while not self.finished.wait(timeout=2): length = 10 # Explicitly make these numpy arrays... c = numpy.array([random.uniform(0.0, 75.0) for i in xrange(length)]) t = numpy.array([random.uniform(-1.7, 21.0) for i in xrange(length)]) p = numpy.array([random.lognormvariate(1, 2) for i in xrange(length)]) lat = numpy.array([random.uniform(-90.0, 90.0) for i in xrange(length)]) lon = numpy.array([random.uniform(0.0, 360.0) for i in xrange(length)]) h = numpy.array([random.uniform(0.0, 360.0) for i in xrange(length)]) tvar = numpy.array([self.last_time + i for i in xrange(1, length + 1)]) self.last_time = max(tvar) rdt["time"] = tvar rdt["lat"] = lat rdt["lon"] = lon rdt["temp"] = t rdt["conductivity"] = c rdt["pressure"] = p g = rdt.to_granule() log.debug("SimpleCtdDataProducer: Sending %d values!" % length) self.publisher.publish(g)
def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) # Create streams and subscriptions for each stream named in driver. self._stream_config = {} streams = { 'parsed' : 'ctd_parsed_param_dict', 'raw' : 'ctd_raw_param_dict' } for (stream_name, param_dict_name) in streams.iteritems(): stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data') pd = get_param_dict(param_dict_name) stream_config = dict(stream_route=stream_route, stream_id=stream_id, parameter_dictionary=pd.dump()) self._stream_config[stream_name] = stream_config
def test_get_last_update(self): # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 1d spatial topology (station/trajectory) sdom = sdom.dump() tdom = tdom.dump() #@TODO: DO NOT DO THIS, WHEN THIS TEST IS REWRITTEN GET RID OF THIS, IT WILL FAIL, thanks -Luke parameter_dictionary = get_param_dict('ctd_parsed_param_dict') parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) data_product_id = self.dpsc_cli.create_data_product(data_product=dp_obj, stream_definition_id=self.stream_def_id, parameter_dictionary=parameter_dictionary) stream_ids, garbage = self.rrclient.find_objects(data_product_id, PRED.hasStream, id_only=True) stream_id = stream_ids[0] fake_lu = LastUpdate() fake_lu_doc = self.db._ion_object_to_persistence_dict(fake_lu) self.db.create_doc(fake_lu_doc, object_id=stream_id) #------------------------------------------ # Now execute #------------------------------------------ res = self.dpsc_cli.get_last_update(data_product_id=data_product_id) self.assertTrue(isinstance(res[stream_id], LastUpdate), 'retrieving documents failed')
def test_createDataProduct_and_DataProducer_success(self): # setup self.clients.resource_registry.find_resources.return_value = ( [], 'do not care') self.clients.resource_registry.find_associations.return_value = [] self.clients.resource_registry.create.return_value = ('SOME_RR_ID1', 'Version_1') self.clients.data_acquisition_management.assign_data_product.return_value = None self.clients.pubsub_management.create_stream.return_value = "stream_id" # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE ) # 1d spatial topology (station/trajectory) sdom = sdom.dump() tdom = tdom.dump() #@TODO: DO NOT DO THIS, WHEN THIS TEST IS REWRITTEN GET RID OF THIS, IT WILL FAIL, thanks -Luke parameter_dictionary = get_param_dict('ctd_parsed_param_dict') parameter_dictionary = parameter_dictionary.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain=tdom, spatial_domain=sdom) # test call dp_id = self.data_product_management_service.create_data_product( data_product=dp_obj, stream_definition_id='a stream def id')