def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() encoder = IonObjectSerializer() # Create streams and subscriptions for each stream named in driver. self._stream_config = {} stream_name = 'parsed' param_dict_name = 'ctd_parsed_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name( param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition( name=stream_name, parameter_dictionary_id=pd_id) stream_def = pubsub_client.read_stream_definition(stream_def_id) stream_def_dict = encoder.serialize(stream_def) pd = stream_def.parameter_dictionary stream_id, stream_route = pubsub_client.create_stream( name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, parameter_dictionary=pd, stream_def_dict=stream_def_dict) self._stream_config[stream_name] = stream_config stream_name = 'raw' param_dict_name = 'ctd_raw_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name( param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition( name=stream_name, parameter_dictionary_id=pd_id) stream_def = pubsub_client.read_stream_definition(stream_def_id) stream_def_dict = encoder.serialize(stream_def) pd = stream_def.parameter_dictionary stream_id, stream_route = pubsub_client.create_stream( name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, parameter_dictionary=pd, stream_def_dict=stream_def_dict) self._stream_config[stream_name] = stream_config
def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() encoder = IonObjectSerializer() # Create streams and subscriptions for each stream named in driver. self._stream_config = {} stream_name = "parsed" param_dict_name = "ctd_parsed_param_dict" pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) stream_def = pubsub_client.read_stream_definition(stream_def_id) stream_def_dict = encoder.serialize(stream_def) pd = stream_def.parameter_dictionary stream_id, stream_route = pubsub_client.create_stream( name=stream_name, exchange_point="science_data", stream_definition_id=stream_def_id ) stream_config = dict( routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, parameter_dictionary=pd, stream_def_dict=stream_def_dict, ) self._stream_config[stream_name] = stream_config stream_name = "raw" param_dict_name = "ctd_raw_param_dict" pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) stream_def = pubsub_client.read_stream_definition(stream_def_id) stream_def_dict = encoder.serialize(stream_def) pd = stream_def.parameter_dictionary stream_id, stream_route = pubsub_client.create_stream( name=stream_name, exchange_point="science_data", stream_definition_id=stream_def_id ) stream_config = dict( routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, parameter_dictionary=pd, stream_def_dict=stream_def_dict, ) self._stream_config[stream_name] = stream_config
def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() # Create streams and subscriptions for each stream named in driver. self.stream_config = {} streams = self.packet_config for (stream_name, param_dict_name) in streams.iteritems(): pd_id = dataset_management.read_parameter_dictionary_by_name(DEFAULT_PARAM_DICT, id_only=True) if(not pd_id): log.error("No pd_id found for param_dict '%s'" % DEFAULT_PARAM_DICT) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(stream_route=stream_route, routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd) self.stream_config[stream_name] = stream_config
def build_stream_config(streams): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=cc.node) dataset_management = DatasetManagementServiceClient() # Create streams and subscriptions for each stream named in driver. agent_stream_config = {} for (stream_name, param_dict_name) in streams.iteritems(): pd_id = dataset_management.read_parameter_dictionary_by_name( param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition( name=stream_name, parameter_dictionary_id=pd_id) pd = pubsub_client.read_stream_definition( stream_def_id).parameter_dictionary stream_id, stream_route = pubsub_client.create_stream( name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(stream_route=stream_route, routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd) agent_stream_config[stream_name] = stream_config return agent_stream_config
def build_stream_config(streams): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=cc.node) dataset_management = DatasetManagementServiceClient() # Create streams and subscriptions for each stream named in driver. agent_stream_config = {} for (stream_name, param_dict_name) in streams.iteritems(): pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(stream_route=stream_route, routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd) agent_stream_config[stream_name] = stream_config return agent_stream_config
def _build_stream_config(self): """ """ if (not self.packet_config): return streams = self.packet_config log.debug("Streams: %s", streams) # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() # Create streams and subscriptions for each stream named in driver. self.stream_config = {} for stream_name in streams: pd_id = None try: pd_id = dataset_management.read_parameter_dictionary_by_name( stream_name, id_only=True) except: log.error("No pd_id found for param_dict '%s'" % stream_name) if (self.use_default_stream): log.error("using default pd '%s'" % DEFAULT_STREAM_NAME) pd_id = dataset_management.read_parameter_dictionary_by_name( DEFAULT_STREAM_NAME, id_only=True) if (not pd_id): raise IDKException( "Missing parameter dictionary for stream '%s'" % stream_name) log.debug("parameter dictionary id: %s" % pd_id) stream_def_id = pubsub_client.create_stream_definition( name=stream_name, parameter_dictionary_id=pd_id) #log.debug("Stream: %s (%s), stream_def_id %s" % (stream_name, type(stream_name), stream_def_id)) pd = pubsub_client.read_stream_definition( stream_def_id).parameter_dictionary #log.debug("Parameter Dictionary: %s" % pd) try: stream_id, stream_route = pubsub_client.create_stream( name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict( stream_route=stream_route, routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd) self.stream_config[stream_name] = stream_config #log.debug("Stream Config (%s): %s" % (stream_name, stream_config)) except Exception as e: log.error("stream publisher exception: %s", e)
def _build_stream_config(self): """ """ if not self.packet_config: return streams = self.packet_config log.debug("Streams: %s", streams) # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() # Create streams and subscriptions for each stream named in driver. self.stream_config = {} for stream_name in streams: pd_id = None try: pd_id = dataset_management.read_parameter_dictionary_by_name(stream_name, id_only=True) except: log.error("No pd_id found for param_dict '%s'" % stream_name) if self.use_default_stream: log.error("using default pd '%s'" % DEFAULT_STREAM_NAME) pd_id = dataset_management.read_parameter_dictionary_by_name(DEFAULT_STREAM_NAME, id_only=True) if not pd_id: raise IDKException("Missing parameter dictionary for stream '%s'" % stream_name) log.debug("parameter dictionary id: %s" % pd_id) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) # log.debug("Stream: %s (%s), stream_def_id %s" % (stream_name, type(stream_name), stream_def_id)) pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary # log.debug("Parameter Dictionary: %s" % pd) try: stream_id, stream_route = pubsub_client.create_stream( name=stream_name, exchange_point="science_data", stream_definition_id=stream_def_id ) stream_config = dict( stream_route=stream_route, routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd, ) self.stream_config[stream_name] = stream_config # log.debug("Stream Config (%s): %s" % (stream_name, stream_config)) except Exception as e: log.error("stream publisher exception: %s", e) log.debug("Stream config setup complete.")
def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() # Create streams and subscriptions for each stream named in driver. self._stream_config = {} streams = { 'parsed': 'ctd_parsed_param_dict', 'raw': 'ctd_raw_param_dict' } for (stream_name, param_dict_name) in streams.iteritems(): pd_id = dataset_management.read_parameter_dictionary_by_name( param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition( name=stream_name, parameter_dictionary_id=pd_id) pd = pubsub_client.read_stream_definition( stream_def_id).parameter_dictionary stream_id, stream_route = pubsub_client.create_stream( name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict( stream_route=stream_route, routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd) if stream_name == 'parsed': type = 'IntervalAlarmDef' kwargs = { 'name': 'test_sim_warning', 'stream_name': 'parsed', 'value_id': 'temp', 'message': 'Temperature is above test range of 5.0.', 'type': StreamAlarmType.WARNING, 'upper_bound': 5.0, 'upper_rel_op': '<' } alarm = {} alarm['type'] = type alarm['kwargs'] = kwargs alarms = [alarm] stream_config['alarms'] = alarms self._stream_config[stream_name] = stream_config
def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() # Create streams and subscriptions for each stream named in driver. self._stream_config = {} stream_name = 'parsed' param_dict_name = 'ctd_parsed_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(stream_route=stream_route, routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd) self._stream_config[stream_name] = stream_config stream_name = 'raw' param_dict_name = 'ctd_raw_param_dict' pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(stream_route=stream_route, routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd) self._stream_config[stream_name] = stream_config
def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() # Create streams and subscriptions for each stream named in driver. self._stream_config = {} streams = { 'parsed' : 'ctd_parsed_param_dict', 'raw' : 'ctd_raw_param_dict' } for (stream_name, param_dict_name) in streams.iteritems(): pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary stream_id, stream_route = pubsub_client.create_stream(name=stream_name, exchange_point='science_data', stream_definition_id=stream_def_id) stream_config = dict(stream_route=stream_route, routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd) if stream_name == 'parsed': type = 'IntervalAlarmDef' kwargs = { 'name' : 'test_sim_warning', 'stream_name' : 'parsed', 'value_id' : 'temp', 'message' : 'Temperature is above test range of 5.0.', 'type' : StreamAlarmType.WARNING, 'upper_bound' : 5.0, 'upper_rel_op' : '<' } alarm = {} alarm['type'] = type alarm['kwargs'] = kwargs alarms = [alarm] stream_config['alarms'] = alarms self._stream_config[stream_name] = stream_config
def _build_stream_config(self): """ """ # Create a pubsub client to create streams. pubsub_client = PubsubManagementServiceClient(node=self.container.node) dataset_management = DatasetManagementServiceClient() # Create streams and subscriptions for each stream named in driver. self._stream_config = {} streams = {"parsed": "ctd_parsed_param_dict", "raw": "ctd_raw_param_dict"} for (stream_name, param_dict_name) in streams.iteritems(): pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary stream_id, stream_route = pubsub_client.create_stream( name=stream_name, exchange_point="science_data", stream_definition_id=stream_def_id ) stream_config = dict( stream_route=stream_route, routing_key=stream_route.routing_key, exchange_point=stream_route.exchange_point, stream_id=stream_id, stream_definition_ref=stream_def_id, parameter_dictionary=pd, ) if stream_name == "parsed": type = "IntervalAlarmDef" kwargs = { "name": "test_sim_warning", "stream_name": "parsed", "value_id": "temp", "message": "Temperature is above test range of 5.0.", "type": StreamAlarmType.WARNING, "upper_bound": 5.0, "upper_rel_op": "<", } alarm = {} alarm["type"] = type alarm["kwargs"] = kwargs alarms = [alarm] stream_config["alarms"] = alarms self._stream_config[stream_name] = stream_config
class TestDMEnd2End(IonIntegrationTestCase): def setUp(self): # Love the non pep-8 convention self._start_container() self.container.start_rel_from_url("res/deploy/r2deploy.yml") self.process_dispatcher = ProcessDispatcherServiceClient() self.pubsub_management = PubsubManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() self.ingestion_management = IngestionManagementServiceClient() self.data_retriever = DataRetrieverServiceClient() self.pids = [] self.event = Event() self.exchange_space_name = "test_granules" self.exchange_point_name = "science_data" self.i = 0 self.purge_queues() self.queue_buffer = [] self.streams = [] self.addCleanup(self.stop_all_ingestion) def purge_queues(self): xn = self.container.ex_manager.create_xn_queue("science_granule_ingestion") xn.purge() def tearDown(self): self.purge_queues() for pid in self.pids: self.container.proc_manager.terminate_process(pid) IngestionManagementIntTest.clean_subscriptions() for queue in self.queue_buffer: if isinstance(queue, ExchangeNameQueue): queue.delete() elif isinstance(queue, str): xn = self.container.ex_manager.create_xn_queue(queue) xn.delete() # -------------------------------------------------------------------------------- # Helper/Utility methods # -------------------------------------------------------------------------------- def create_dataset(self, parameter_dict_id=""): """ Creates a time-series dataset """ tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() if not parameter_dict_id: parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name( "ctd_parsed_param_dict", id_only=True ) dataset_id = self.dataset_management.create_dataset( "test_dataset_%i" % self.i, parameter_dictionary_id=parameter_dict_id, spatial_domain=sdom, temporal_domain=tdom, ) return dataset_id def get_datastore(self, dataset_id): """ Gets an instance of the datastore This method is primarily used to defeat a bug where integration tests in multiple containers may sometimes delete a CouchDB datastore and the other containers are unaware of the new state of the datastore. """ dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def get_ingestion_config(self): """ Grab the ingestion configuration from the resource registry """ # The ingestion configuration should have been created by the bootstrap service # which is configured through r2deploy.yml ingest_configs, _ = self.resource_registry.find_resources(restype=RT.IngestionConfiguration, id_only=True) return ingest_configs[0] def launch_producer(self, stream_id=""): """ Launch the producer """ pid = self.container.spawn_process( "better_data_producer", "ion.processes.data.example_data_producer", "BetterDataProducer", {"process": {"stream_id": stream_id}}, ) self.pids.append(pid) def make_simple_dataset(self): """ Makes a stream, a stream definition and a dataset, the essentials for most of these tests """ pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True) stream_def_id = self.pubsub_management.create_stream_definition("ctd data", parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream( "ctd stream %i" % self.i, "xp1", stream_definition_id=stream_def_id ) dataset_id = self.create_dataset(pdict_id) self.get_datastore(dataset_id) self.i += 1 return stream_id, route, stream_def_id, dataset_id def publish_hifi(self, stream_id, stream_route, offset=0): """ Publish deterministic data """ pub = StandaloneStreamPublisher(stream_id, stream_route) stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id) stream_def_id = stream_def._id rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt["time"] = np.arange(10) + (offset * 10) rdt["temp"] = np.arange(10) + (offset * 10) pub.publish(rdt.to_granule()) def publish_fake_data(self, stream_id, route): """ Make four granules """ for i in xrange(4): self.publish_hifi(stream_id, route, i) def start_ingestion(self, stream_id, dataset_id): """ Starts ingestion/persistence for a given dataset """ ingest_config_id = self.get_ingestion_config() self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id ) def stop_ingestion(self, stream_id): ingest_config_id = self.get_ingestion_config() self.ingestion_management.unpersist_data_stream( stream_id=stream_id, ingestion_configuration_id=ingest_config_id ) def stop_all_ingestion(self): try: [self.stop_ingestion(sid) for sid in self.streams] except: pass def validate_granule_subscription(self, msg, route, stream_id): """ Validation for granule format """ if msg == {}: return rdt = RecordDictionaryTool.load_from_granule(msg) log.info("%s", rdt.pretty_print()) self.assertIsInstance(msg, Granule, "Message is improperly formatted. (%s)" % type(msg)) self.event.set() def wait_until_we_have_enough_granules(self, dataset_id="", data_size=40): """ Loops until there is a sufficient amount of data in the dataset """ done = False with gevent.Timeout(40): while not done: extents = self.dataset_management.dataset_extents(dataset_id, "time")[0] granule = self.data_retriever.retrieve_last_data_points(dataset_id, 1) rdt = RecordDictionaryTool.load_from_granule(granule) if rdt["time"] and rdt["time"][0] != rdt._pdict.get_context("time").fill_value and extents >= data_size: done = True else: gevent.sleep(0.2) # -------------------------------------------------------------------------------- # Test Methods # -------------------------------------------------------------------------------- @attr("SMOKE") def test_dm_end_2_end(self): # -------------------------------------------------------------------------------- # Set up a stream and have a mock instrument (producer) send data # -------------------------------------------------------------------------------- self.event.clear() # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext("binary", param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context("binary", bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext("records", param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context("records", rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary( "replay_pdict", parameter_context_ids=context_ids, temporal_context="time" ) stream_definition = self.pubsub_management.create_stream_definition( "ctd data", parameter_dictionary_id=pdict_id ) stream_id, route = self.pubsub_management.create_stream( "producer", exchange_point=self.exchange_point_name, stream_definition_id=stream_definition ) # -------------------------------------------------------------------------------- # Start persisting the data on the stream # - Get the ingestion configuration from the resource registry # - Create the dataset # - call persist_data_stream to setup the subscription for the ingestion workers # on the stream that you specify which causes the data to be persisted # -------------------------------------------------------------------------------- ingest_config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id ) # -------------------------------------------------------------------------------- # Now the granules are ingesting and persisted # -------------------------------------------------------------------------------- self.launch_producer(stream_id) self.wait_until_we_have_enough_granules(dataset_id, 40) # -------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive # -------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_id) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) self.assertTrue((rdt["time"][:10] == np.arange(10)).all(), "%s" % rdt["time"][:]) self.assertTrue((rdt["binary"][:10] == np.array(["hi"] * 10, dtype="object")).all()) # -------------------------------------------------------------------------------- # Now to try the streamed approach # -------------------------------------------------------------------------------- replay_stream_id, replay_route = self.pubsub_management.create_stream( "replay_out", exchange_point=self.exchange_point_name, stream_definition_id=stream_definition ) self.replay_id, process_id = self.data_retriever.define_replay( dataset_id=dataset_id, stream_id=replay_stream_id ) log.info("Process ID: %s", process_id) replay_client = ReplayClient(process_id) # -------------------------------------------------------------------------------- # Create the listening endpoint for the the retriever to talk to # -------------------------------------------------------------------------------- xp = self.container.ex_manager.create_xp(self.exchange_point_name) subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription) self.queue_buffer.append(self.exchange_space_name) subscriber.start() subscriber.xn.bind(replay_route.routing_key, xp) self.data_retriever.start_replay_agent(self.replay_id) self.assertTrue(replay_client.await_agent_ready(5), "The process never launched") replay_client.start_replay() self.assertTrue(self.event.wait(10)) subscriber.stop() self.data_retriever.cancel_replay_agent(self.replay_id) # -------------------------------------------------------------------------------- # Test the slicing capabilities # -------------------------------------------------------------------------------- granule = self.data_retriever.retrieve(dataset_id=dataset_id, query={"tdoa": slice(0, 5)}) rdt = RecordDictionaryTool.load_from_granule(granule) b = rdt["time"] == np.arange(5) self.assertTrue(b.all() if not isinstance(b, bool) else b) self.streams.append(stream_id) self.stop_ingestion(stream_id) @unittest.skip("Doesnt work") @attr("LOCOINT") @unittest.skipIf(os.getenv("CEI_LAUNCH_TEST", False), "Skip test while in CEI LAUNCH mode") def test_replay_pause(self): # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext("binary", param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context("binary", bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext("records", param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context("records", rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary( "replay_pdict", parameter_context_ids=context_ids, temporal_context="time" ) stream_def_id = self.pubsub_management.create_stream_definition( "replay_stream", parameter_dictionary_id=pdict_id ) replay_stream, replay_route = self.pubsub_management.create_stream( "replay", "xp1", stream_definition_id=stream_def_id ) dataset_id = self.create_dataset(pdict_id) scov = DatasetManagementService._get_coverage(dataset_id) bb = CoverageCraft(scov) bb.rdt["time"] = np.arange(100) bb.rdt["temp"] = np.random.random(100) + 30 bb.sync_with_granule() DatasetManagementService._persist_coverage( dataset_id, bb.coverage ) # This invalidates it for multi-host configurations # Set up the subscriber to verify the data subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription) xp = self.container.ex_manager.create_xp("xp1") self.queue_buffer.append(self.exchange_space_name) subscriber.start() subscriber.xn.bind(replay_route.routing_key, xp) # Set up the replay agent and the client wrapper # 1) Define the Replay (dataset and stream to publish on) self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream) # 2) Make a client to the interact with the process (optionall provide it a process to bind with) replay_client = ReplayClient(process_id) # 3) Start the agent (launch the process) self.data_retriever.start_replay_agent(self.replay_id) # 4) Start replaying... replay_client.start_replay() # Wait till we get some granules self.assertTrue(self.event.wait(5)) # We got granules, pause the replay, clear the queue and allow the process to finish consuming replay_client.pause_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure there's no remaining messages being consumed self.assertFalse(self.event.wait(1)) # Resume the replay and wait until we start getting granules again replay_client.resume_replay() self.assertTrue(self.event.wait(5)) # Stop the replay, clear the queues replay_client.stop_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure that it did indeed stop self.assertFalse(self.event.wait(1)) subscriber.stop() def test_retrieve_and_transform(self): # Make a simple dataset and start ingestion, pretty standard stuff. ctd_stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() self.start_ingestion(ctd_stream_id, dataset_id) # Stream definition for the salinity data salinity_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( "ctd_parsed_param_dict", id_only=True ) sal_stream_def_id = self.pubsub_management.create_stream_definition( "sal data", parameter_dictionary_id=salinity_pdict_id ) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt["time"] = np.arange(10) rdt["temp"] = np.random.randn(10) * 10 + 30 rdt["conductivity"] = np.random.randn(10) * 2 + 10 rdt["pressure"] = np.random.randn(10) * 1 + 12 publisher = StandaloneStreamPublisher(ctd_stream_id, route) publisher.publish(rdt.to_granule()) rdt["time"] = np.arange(10, 20) publisher.publish(rdt.to_granule()) self.wait_until_we_have_enough_granules(dataset_id, 20) granule = self.data_retriever.retrieve( dataset_id, None, None, "ion.processes.data.transforms.ctd.ctd_L2_salinity", "CTDL2SalinityTransformAlgorithm", kwargs=dict(params=sal_stream_def_id), ) rdt = RecordDictionaryTool.load_from_granule(granule) for i in rdt["salinity"]: self.assertNotEquals(i, 0) self.streams.append(ctd_stream_id) self.stop_ingestion(ctd_stream_id) def test_last_granule(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() self.start_ingestion(stream_id, dataset_id) self.publish_hifi(stream_id, route, 0) self.publish_hifi(stream_id, route, 1) self.wait_until_we_have_enough_granules(dataset_id, 20) # I just need two success = False def verifier(): replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id, 10) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt["time"] == np.arange(10) + 10 if not isinstance(comp, bool): return comp.all() return False success = poll(verifier) self.assertTrue(success) success = False def verify_points(): replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id, 5) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt["time"] == np.arange(15, 20) if not isinstance(comp, bool): return comp.all() return False success = poll(verify_points) self.assertTrue(success) self.streams.append(stream_id) self.stop_ingestion(stream_id) def test_replay_with_parameters(self): # -------------------------------------------------------------------------------- # Create the configurations and the dataset # -------------------------------------------------------------------------------- # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext("binary", param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context("binary", bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext("records", param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context("records", rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary( "replay_pdict", parameter_context_ids=context_ids, temporal_context="time" ) stream_def_id = self.pubsub_management.create_stream_definition( "replay_stream", parameter_dictionary_id=pdict_id ) stream_id, route = self.pubsub_management.create_stream( "replay_with_params", exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id ) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id ) # -------------------------------------------------------------------------------- # Coerce the datastore into existence (beats race condition) # -------------------------------------------------------------------------------- self.get_datastore(dataset_id) self.launch_producer(stream_id) self.wait_until_we_have_enough_granules(dataset_id, 40) query = { "start_time": 0 - 2208988800, "end_time": 20 - 2208988800, "stride_time": 2, "parameters": ["time", "temp"], } retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id, query=query) rdt = RecordDictionaryTool.load_from_granule(retrieved_data) comp = np.arange(0, 20, 2) == rdt["time"] self.assertTrue(comp.all(), "%s" % rdt.pretty_print()) self.assertEquals(set(rdt.iterkeys()), set(["time", "temp"])) extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=["time", "temp"]) self.assertTrue(extents["time"] >= 20) self.assertTrue(extents["temp"] >= 20) self.streams.append(stream_id) self.stop_ingestion(stream_id) def test_repersist_data(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() self.start_ingestion(stream_id, dataset_id) self.publish_hifi(stream_id, route, 0) self.publish_hifi(stream_id, route, 1) self.wait_until_we_have_enough_granules(dataset_id, 20) config_id = self.get_ingestion_config() self.ingestion_management.unpersist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id) self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id ) self.publish_hifi(stream_id, route, 2) self.publish_hifi(stream_id, route, 3) self.wait_until_we_have_enough_granules(dataset_id, 40) success = False with gevent.timeout.Timeout(5): while not success: replay_granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt["time"] == np.arange(0, 40) if not isinstance(comp, bool): success = comp.all() gevent.sleep(1) self.assertTrue(success) self.streams.append(stream_id) self.stop_ingestion(stream_id) @attr("LOCOINT") @unittest.skipIf( os.getenv("CEI_LAUNCH_TEST", False), "Host requires file-system access to coverage files, CEI mode does not support.", ) def test_correct_time(self): # There are 2208988800 seconds between Jan 1 1900 and Jan 1 1970, i.e. # the conversion factor between unix and NTP time unix_now = np.floor(time.time()) ntp_now = unix_now + 2208988800 unix_ago = unix_now - 20 ntp_ago = unix_ago + 2208988800 stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() coverage = DatasetManagementService._get_coverage(dataset_id) coverage.insert_timesteps(20) coverage.set_parameter_values("time", np.arange(ntp_ago, ntp_now)) temporal_bounds = self.dataset_management.dataset_temporal_bounds(dataset_id) self.assertTrue(np.abs(temporal_bounds[0] - unix_ago) < 2) self.assertTrue(np.abs(temporal_bounds[1] - unix_now) < 2) @attr("LOCOINT") @unittest.skipIf( os.getenv("CEI_LAUNCH_TEST", False), "Host requires file-system access to coverage files, CEI mode does not support.", ) def test_empty_coverage_time(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() coverage = DatasetManagementService._get_coverage(dataset_id) temporal_bounds = self.dataset_management.dataset_temporal_bounds(dataset_id) self.assertEquals([coverage.get_parameter_context("time").fill_value] * 2, temporal_bounds) @attr("LOCOINT") @unittest.skipIf( os.getenv("CEI_LAUNCH_TEST", False), "Host requires file-system access to coverage files, CEI mode does not support.", ) def test_out_of_band_retrieve(self): # Setup the environemnt stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() self.start_ingestion(stream_id, dataset_id) # Fill the dataset self.publish_fake_data(stream_id, route) self.wait_until_we_have_enough_granules(dataset_id, 40) # Retrieve the data granule = DataRetrieverService.retrieve_oob(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) self.assertTrue((rdt["time"] == np.arange(40)).all()) @attr("LOCOINT") @unittest.skipIf( os.getenv("CEI_LAUNCH_TEST", False), "Host requires file-system access to coverage files, CEI mode does not support.", ) def test_retrieve_cache(self): DataRetrieverService._refresh_interval = 1 datasets = [self.make_simple_dataset() for i in xrange(10)] for stream_id, route, stream_def_id, dataset_id in datasets: coverage = DatasetManagementService._get_coverage(dataset_id) coverage.insert_timesteps(10) coverage.set_parameter_values("time", np.arange(10)) coverage.set_parameter_values("temp", np.arange(10)) # Verify cache hit and refresh dataset_ids = [i[3] for i in datasets] self.assertTrue(dataset_ids[0] not in DataRetrieverService._retrieve_cache) DataRetrieverService._get_coverage(dataset_ids[0]) # Hit the chache cov, age = DataRetrieverService._retrieve_cache[dataset_ids[0]] # Verify that it was hit and it's now in there self.assertTrue(dataset_ids[0] in DataRetrieverService._retrieve_cache) gevent.sleep(DataRetrieverService._refresh_interval + 0.2) DataRetrieverService._get_coverage(dataset_ids[0]) # Hit the chache cov, age2 = DataRetrieverService._retrieve_cache[dataset_ids[0]] self.assertTrue(age2 != age) for dataset_id in dataset_ids: DataRetrieverService._get_coverage(dataset_id) self.assertTrue(dataset_ids[0] not in DataRetrieverService._retrieve_cache) stream_id, route, stream_def, dataset_id = datasets[0] self.start_ingestion(stream_id, dataset_id) DataRetrieverService._get_coverage(dataset_id) self.assertTrue(dataset_id in DataRetrieverService._retrieve_cache) DataRetrieverService._refresh_interval = 100 self.publish_hifi(stream_id, route, 1) self.wait_until_we_have_enough_granules(dataset_id, data_size=20) event = gevent.event.Event() with gevent.Timeout(20): while not event.wait(0.1): if dataset_id not in DataRetrieverService._retrieve_cache: event.set() self.assertTrue(event.is_set()) @unittest.skip("Outdated due to ingestion retry") @attr("LOCOINT") @unittest.skipIf( os.getenv("CEI_LAUNCH_TEST", False), "Host requires file-system access to coverage files, CEI mode does not support.", ) def test_ingestion_failover(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() self.start_ingestion(stream_id, dataset_id) event = Event() def cb(*args, **kwargs): event.set() sub = EventSubscriber(event_type="ExceptionEvent", callback=cb, origin="stream_exception") sub.start() self.publish_fake_data(stream_id, route) self.wait_until_we_have_enough_granules(dataset_id, 40) file_path = DatasetManagementService._get_coverage_path(dataset_id) master_file = os.path.join(file_path, "%s_master.hdf5" % dataset_id) with open(master_file, "w") as f: f.write("this will crash HDF") self.publish_hifi(stream_id, route, 5) self.assertTrue(event.wait(10)) sub.stop()
class PubsubManagementIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.pubsub_management = PubsubManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() self.data_product_management = DataProductManagementServiceClient() self.pdicts = {} self.queue_cleanup = list() self.exchange_cleanup = list() self.context_ids = set() def tearDown(self): for queue in self.queue_cleanup: xn = self.container.ex_manager.create_xn_queue(queue) xn.delete() for exchange in self.exchange_cleanup: xp = self.container.ex_manager.create_xp(exchange) xp.delete() self.cleanup_contexts() def test_stream_def_crud(self): # Test Creation pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict') stream_definition_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict.identifier) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_definition_id) # Make sure there is an assoc self.assertTrue(self.resource_registry.find_associations(subject=stream_definition_id, predicate=PRED.hasParameterDictionary, object=pdict.identifier, id_only=True)) # Test Reading stream_definition = self.pubsub_management.read_stream_definition(stream_definition_id) self.assertTrue(PubsubManagementService._compare_pdicts(pdict.dump(), stream_definition.parameter_dictionary)) # Test comparisons in_stream_definition_id = self.pubsub_management.create_stream_definition('L0 products', parameter_dictionary_id=pdict.identifier, available_fields=['time','temp','conductivity','pressure']) self.addCleanup(self.pubsub_management.delete_stream_definition, in_stream_definition_id) out_stream_definition_id = in_stream_definition_id self.assertTrue(self.pubsub_management.compare_stream_definition(in_stream_definition_id, out_stream_definition_id)) self.assertTrue(self.pubsub_management.compatible_stream_definitions(in_stream_definition_id, out_stream_definition_id)) out_stream_definition_id = self.pubsub_management.create_stream_definition('L2 Products', parameter_dictionary_id=pdict.identifier, available_fields=['time','salinity','density']) self.addCleanup(self.pubsub_management.delete_stream_definition, out_stream_definition_id) self.assertFalse(self.pubsub_management.compare_stream_definition(in_stream_definition_id, out_stream_definition_id)) self.assertTrue(self.pubsub_management.compatible_stream_definitions(in_stream_definition_id, out_stream_definition_id)) @unittest.skip('Needs to be refactored for cleanup') def test_validate_stream_defs(self): self.addCleanup(self.cleanup_contexts) #test no input incoming_pdict_id = self._get_pdict(['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0']) outgoing_pdict_id = self._get_pdict(['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) available_fields_in = [] available_fields_out = [] incoming_stream_def_id = self.pubsub_management.create_stream_definition('in_sd_0', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) self.addCleanup(self.pubsub_management.delete_stream_definition, incoming_stream_def_id) outgoing_stream_def_id = self.pubsub_management.create_stream_definition('out_sd_0', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) self.addCleanup(self.pubsub_management.delete_stream_definition, outgoing_stream_def_id) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertFalse(result) #test input with no output incoming_pdict_id = self._get_pdict(['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0']) outgoing_pdict_id = self._get_pdict(['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) available_fields_in = ['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'] available_fields_out = [] incoming_stream_def_id = self.pubsub_management.create_stream_definition('in_sd_1', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) self.addCleanup(self.pubsub_management.delete_stream_definition, incoming_stream_def_id) outgoing_stream_def_id = self.pubsub_management.create_stream_definition('out_sd_1', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) self.addCleanup(self.pubsub_management.delete_stream_definition, outgoing_stream_def_id) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertTrue(result) #test available field missing parameter context definition -- missing PRESWAT_L0 incoming_pdict_id = self._get_pdict(['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0']) outgoing_pdict_id = self._get_pdict(['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) available_fields_in = ['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'] available_fields_out = ['DENSITY'] incoming_stream_def_id = self.pubsub_management.create_stream_definition('in_sd_2', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) self.addCleanup(self.pubsub_management.delete_stream_definition, incoming_stream_def_id) outgoing_stream_def_id = self.pubsub_management.create_stream_definition('out_sd_2', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) self.addCleanup(self.pubsub_management.delete_stream_definition, outgoing_stream_def_id) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertFalse(result) #test l1 from l0 incoming_pdict_id = self._get_pdict(['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0']) outgoing_pdict_id = self._get_pdict(['TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) available_fields_in = ['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'] available_fields_out = ['TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1'] incoming_stream_def_id = self.pubsub_management.create_stream_definition('in_sd_3', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) self.addCleanup(self.pubsub_management.delete_stream_definition, incoming_stream_def_id) outgoing_stream_def_id = self.pubsub_management.create_stream_definition('out_sd_3', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) self.addCleanup(self.pubsub_management.delete_stream_definition, outgoing_stream_def_id) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertTrue(result) #test l2 from l0 incoming_pdict_id = self._get_pdict(['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0']) outgoing_pdict_id = self._get_pdict(['TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1', 'DENSITY', 'PRACSAL']) available_fields_in = ['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'] available_fields_out = ['DENSITY', 'PRACSAL'] incoming_stream_def_id = self.pubsub_management.create_stream_definition('in_sd_4', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) self.addCleanup(self.pubsub_management.delete_stream_definition, incoming_stream_def_id) outgoing_stream_def_id = self.pubsub_management.create_stream_definition('out_sd_4', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) self.addCleanup(self.pubsub_management.delete_stream_definition, outgoing_stream_def_id) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertTrue(result) #test Ln from L0 incoming_pdict_id = self._get_pdict(['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0']) outgoing_pdict_id = self._get_pdict(['DENSITY','PRACSAL','TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) available_fields_in = ['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'] available_fields_out = ['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1'] incoming_stream_def_id = self.pubsub_management.create_stream_definition('in_sd_5', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) self.addCleanup(self.pubsub_management.delete_stream_definition, incoming_stream_def_id) outgoing_stream_def_id = self.pubsub_management.create_stream_definition('out_sd_5', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) self.addCleanup(self.pubsub_management.delete_stream_definition, outgoing_stream_def_id) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertTrue(result) #test L2 from L1 incoming_pdict_id = self._get_pdict(['TIME', 'LAT', 'LON', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) outgoing_pdict_id = self._get_pdict(['DENSITY','PRACSAL','TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) available_fields_in = ['TIME', 'LAT', 'LON', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1'] available_fields_out = ['DENSITY', 'PRACSAL'] incoming_stream_def_id = self.pubsub_management.create_stream_definition('in_sd_6', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) self.addCleanup(self.pubsub_management.delete_stream_definition, incoming_stream_def_id) outgoing_stream_def_id = self.pubsub_management.create_stream_definition('out_sd_6', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) self.addCleanup(self.pubsub_management.delete_stream_definition, outgoing_stream_def_id) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertTrue(result) #test L1 from L0 missing L0 incoming_pdict_id = self._get_pdict(['TIME', 'LAT', 'LON']) outgoing_pdict_id = self._get_pdict(['TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) available_fields_in = ['TIME', 'LAT', 'LON'] available_fields_out = ['DENSITY', 'PRACSAL'] incoming_stream_def_id = self.pubsub_management.create_stream_definition('in_sd_7', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) self.addCleanup(self.pubsub_management.delete_stream_definition, incoming_stream_def_id) outgoing_stream_def_id = self.pubsub_management.create_stream_definition('out_sd_7', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) self.addCleanup(self.pubsub_management.delete_stream_definition, outgoing_stream_def_id) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertFalse(result) #test L2 from L0 missing L0 incoming_pdict_id = self._get_pdict(['TIME', 'LAT', 'LON']) outgoing_pdict_id = self._get_pdict(['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) available_fields_in = ['TIME', 'LAT', 'LON'] available_fields_out = ['DENSITY', 'PRACSAL'] incoming_stream_def_id = self.pubsub_management.create_stream_definition('in_sd_8', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) self.addCleanup(self.pubsub_management.delete_stream_definition, incoming_stream_def_id) outgoing_stream_def_id = self.pubsub_management.create_stream_definition('out_sd_8', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) self.addCleanup(self.pubsub_management.delete_stream_definition, outgoing_stream_def_id) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertFalse(result) #test L2 from L0 missing L1 incoming_pdict_id = self._get_pdict(['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0']) outgoing_pdict_id = self._get_pdict(['DENSITY', 'PRACSAL']) available_fields_in = ['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0'] available_fields_out = ['DENSITY', 'PRACSAL'] incoming_stream_def_id = self.pubsub_management.create_stream_definition('in_sd_9', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) self.addCleanup(self.pubsub_management.delete_stream_definition, incoming_stream_def_id) outgoing_stream_def_id = self.pubsub_management.create_stream_definition('out_sd_9', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) self.addCleanup(self.pubsub_management.delete_stream_definition, outgoing_stream_def_id) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertFalse(result) def publish_on_stream(self, stream_id, msg): stream = self.pubsub_management.read_stream(stream_id) stream_route = stream.stream_route publisher = StandaloneStreamPublisher(stream_id=stream_id, stream_route=stream_route) publisher.publish(msg) def test_stream_crud(self): stream_def_id = self.pubsub_management.create_stream_definition('test_definition', stream_type='stream') self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) topic_id = self.pubsub_management.create_topic(name='test_topic', exchange_point='test_exchange') self.addCleanup(self.pubsub_management.delete_topic, topic_id) self.exchange_cleanup.append('test_exchange') topic2_id = self.pubsub_management.create_topic(name='another_topic', exchange_point='outside') self.addCleanup(self.pubsub_management.delete_topic, topic2_id) stream_id, route = self.pubsub_management.create_stream(name='test_stream', topic_ids=[topic_id, topic2_id], exchange_point='test_exchange', stream_definition_id=stream_def_id) topics, assocs = self.resource_registry.find_objects(subject=stream_id, predicate=PRED.hasTopic, id_only=True) self.assertEquals(topics,[topic_id]) defs, assocs = self.resource_registry.find_objects(subject=stream_id, predicate=PRED.hasStreamDefinition, id_only=True) self.assertTrue(len(defs)) stream = self.pubsub_management.read_stream(stream_id) self.assertEquals(stream.name,'test_stream') self.pubsub_management.delete_stream(stream_id) with self.assertRaises(NotFound): self.pubsub_management.read_stream(stream_id) defs, assocs = self.resource_registry.find_objects(subject=stream_id, predicate=PRED.hasStreamDefinition, id_only=True) self.assertFalse(len(defs)) topics, assocs = self.resource_registry.find_objects(subject=stream_id, predicate=PRED.hasTopic, id_only=True) self.assertFalse(len(topics)) def test_data_product_subscription(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) stream_def_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) tdom, sdom = time_series_domain() dp = DataProduct(name='ctd parsed') dp.spatial_domain = sdom.dump() dp.temporal_domain = tdom.dump() data_product_id = self.data_product_management.create_data_product(data_product=dp, stream_definition_id=stream_def_id) self.addCleanup(self.data_product_management.delete_data_product, data_product_id) subscription_id = self.pubsub_management.create_subscription('validator', data_product_ids=[data_product_id]) self.addCleanup(self.pubsub_management.delete_subscription, subscription_id) validated = Event() def validation(msg, route, stream_id): validated.set() stream_ids, _ = self.resource_registry.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True) dp_stream_id = stream_ids.pop() validator = StandaloneStreamSubscriber('validator', callback=validation) validator.start() self.addCleanup(validator.stop) self.pubsub_management.activate_subscription(subscription_id) self.addCleanup(self.pubsub_management.deactivate_subscription, subscription_id) route = self.pubsub_management.read_stream_route(dp_stream_id) publisher = StandaloneStreamPublisher(dp_stream_id, route) publisher.publish('hi') self.assertTrue(validated.wait(10)) def test_subscription_crud(self): stream_def_id = self.pubsub_management.create_stream_definition('test_definition', stream_type='stream') stream_id, route = self.pubsub_management.create_stream(name='test_stream', exchange_point='test_exchange', stream_definition_id=stream_def_id) subscription_id = self.pubsub_management.create_subscription(name='test subscription', stream_ids=[stream_id], exchange_name='test_queue') self.exchange_cleanup.append('test_exchange') subs, assocs = self.resource_registry.find_objects(subject=subscription_id,predicate=PRED.hasStream,id_only=True) self.assertEquals(subs,[stream_id]) res, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='test_queue', id_only=True) self.assertEquals(len(res),1) subs, assocs = self.resource_registry.find_subjects(object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertEquals(subs[0], res[0]) subscription = self.pubsub_management.read_subscription(subscription_id) self.assertEquals(subscription.exchange_name, 'test_queue') self.pubsub_management.delete_subscription(subscription_id) subs, assocs = self.resource_registry.find_objects(subject=subscription_id,predicate=PRED.hasStream,id_only=True) self.assertFalse(len(subs)) subs, assocs = self.resource_registry.find_subjects(object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertFalse(len(subs)) self.pubsub_management.delete_stream(stream_id) self.pubsub_management.delete_stream_definition(stream_def_id) def test_move_before_activate(self): stream_id, route = self.pubsub_management.create_stream(name='test_stream', exchange_point='test_xp') #-------------------------------------------------------------------------------- # Test moving before activate #-------------------------------------------------------------------------------- subscription_id = self.pubsub_management.create_subscription('first_queue', stream_ids=[stream_id]) xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='first_queue', id_only=True) subjects, _ = self.resource_registry.find_subjects(object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertEquals(xn_ids[0], subjects[0]) self.pubsub_management.move_subscription(subscription_id, exchange_name='second_queue') xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='second_queue', id_only=True) subjects, _ = self.resource_registry.find_subjects(object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertEquals(len(subjects),1) self.assertEquals(subjects[0], xn_ids[0]) self.pubsub_management.delete_subscription(subscription_id) self.pubsub_management.delete_stream(stream_id) def test_move_activated_subscription(self): stream_id, route = self.pubsub_management.create_stream(name='test_stream', exchange_point='test_xp') #-------------------------------------------------------------------------------- # Test moving after activate #-------------------------------------------------------------------------------- subscription_id = self.pubsub_management.create_subscription('first_queue', stream_ids=[stream_id]) self.pubsub_management.activate_subscription(subscription_id) xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='first_queue', id_only=True) subjects, _ = self.resource_registry.find_subjects(object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertEquals(xn_ids[0], subjects[0]) self.verified = Event() def verify(m,r,s): self.assertEquals(m,'verified') self.verified.set() subscriber = StandaloneStreamSubscriber('second_queue', verify) subscriber.start() self.pubsub_management.move_subscription(subscription_id, exchange_name='second_queue') xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='second_queue', id_only=True) subjects, _ = self.resource_registry.find_subjects(object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertEquals(len(subjects),1) self.assertEquals(subjects[0], xn_ids[0]) publisher = StandaloneStreamPublisher(stream_id, route) publisher.publish('verified') self.assertTrue(self.verified.wait(2)) self.pubsub_management.deactivate_subscription(subscription_id) self.pubsub_management.delete_subscription(subscription_id) self.pubsub_management.delete_stream(stream_id) def test_queue_cleanup(self): stream_id, route = self.pubsub_management.create_stream('test_stream','xp1') xn_objs, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='queue1') for xn_obj in xn_objs: xn = self.container.ex_manager.create_xn_queue(xn_obj.name) xn.delete() subscription_id = self.pubsub_management.create_subscription('queue1',stream_ids=[stream_id]) xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='queue1') self.assertEquals(len(xn_ids),1) self.pubsub_management.delete_subscription(subscription_id) xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='queue1') self.assertEquals(len(xn_ids),0) def test_activation_and_deactivation(self): stream_id, route = self.pubsub_management.create_stream('stream1','xp1') subscription_id = self.pubsub_management.create_subscription('sub1', stream_ids=[stream_id]) self.check1 = Event() def verifier(m,r,s): self.check1.set() subscriber = StandaloneStreamSubscriber('sub1',verifier) subscriber.start() publisher = StandaloneStreamPublisher(stream_id, route) publisher.publish('should not receive') self.assertFalse(self.check1.wait(0.25)) self.pubsub_management.activate_subscription(subscription_id) publisher.publish('should receive') self.assertTrue(self.check1.wait(2)) self.check1.clear() self.assertFalse(self.check1.is_set()) self.pubsub_management.deactivate_subscription(subscription_id) publisher.publish('should not receive') self.assertFalse(self.check1.wait(0.5)) self.pubsub_management.activate_subscription(subscription_id) publisher.publish('should receive') self.assertTrue(self.check1.wait(2)) subscriber.stop() self.pubsub_management.deactivate_subscription(subscription_id) self.pubsub_management.delete_subscription(subscription_id) self.pubsub_management.delete_stream(stream_id) def test_topic_crud(self): topic_id = self.pubsub_management.create_topic(name='test_topic', exchange_point='test_xp') self.exchange_cleanup.append('test_xp') topic = self.pubsub_management.read_topic(topic_id) self.assertEquals(topic.name,'test_topic') self.assertEquals(topic.exchange_point, 'test_xp') self.pubsub_management.delete_topic(topic_id) with self.assertRaises(NotFound): self.pubsub_management.read_topic(topic_id) def test_full_pubsub(self): self.sub1_sat = Event() self.sub2_sat = Event() def subscriber1(m,r,s): self.sub1_sat.set() def subscriber2(m,r,s): self.sub2_sat.set() sub1 = StandaloneStreamSubscriber('sub1', subscriber1) sub1.start() self.addCleanup(sub1.stop) sub2 = StandaloneStreamSubscriber('sub2', subscriber2) sub2.start() self.addCleanup(sub2.stop) log_topic = self.pubsub_management.create_topic('instrument_logs', exchange_point='instruments') self.addCleanup(self.pubsub_management.delete_topic, log_topic) science_topic = self.pubsub_management.create_topic('science_data', exchange_point='instruments') self.addCleanup(self.pubsub_management.delete_topic, science_topic) events_topic = self.pubsub_management.create_topic('notifications', exchange_point='events') self.addCleanup(self.pubsub_management.delete_topic, events_topic) log_stream, route = self.pubsub_management.create_stream('instrument1-logs', topic_ids=[log_topic], exchange_point='instruments') self.addCleanup(self.pubsub_management.delete_stream, log_stream) ctd_stream, route = self.pubsub_management.create_stream('instrument1-ctd', topic_ids=[science_topic], exchange_point='instruments') self.addCleanup(self.pubsub_management.delete_stream, ctd_stream) event_stream, route = self.pubsub_management.create_stream('notifications', topic_ids=[events_topic], exchange_point='events') self.addCleanup(self.pubsub_management.delete_stream, event_stream) raw_stream, route = self.pubsub_management.create_stream('temp', exchange_point='global.data') self.addCleanup(self.pubsub_management.delete_stream, raw_stream) subscription1 = self.pubsub_management.create_subscription('subscription1', stream_ids=[log_stream,event_stream], exchange_name='sub1') self.addCleanup(self.pubsub_management.delete_subscription, subscription1) subscription2 = self.pubsub_management.create_subscription('subscription2', exchange_points=['global.data'], stream_ids=[ctd_stream], exchange_name='sub2') self.addCleanup(self.pubsub_management.delete_subscription, subscription2) self.pubsub_management.activate_subscription(subscription1) self.addCleanup(self.pubsub_management.deactivate_subscription, subscription1) self.pubsub_management.activate_subscription(subscription2) self.addCleanup(self.pubsub_management.deactivate_subscription, subscription2) self.publish_on_stream(log_stream, 1) self.assertTrue(self.sub1_sat.wait(4)) self.assertFalse(self.sub2_sat.is_set()) self.publish_on_stream(raw_stream,1) self.assertTrue(self.sub1_sat.wait(4)) def test_topic_craziness(self): self.msg_queue = Queue() def subscriber1(m,r,s): self.msg_queue.put(m) sub1 = StandaloneStreamSubscriber('sub1', subscriber1) sub1.start() self.addCleanup(sub1.stop) topic1 = self.pubsub_management.create_topic('topic1', exchange_point='xp1') self.addCleanup(self.pubsub_management.delete_topic, topic1) topic2 = self.pubsub_management.create_topic('topic2', exchange_point='xp1', parent_topic_id=topic1) self.addCleanup(self.pubsub_management.delete_topic, topic2) topic3 = self.pubsub_management.create_topic('topic3', exchange_point='xp1', parent_topic_id=topic1) self.addCleanup(self.pubsub_management.delete_topic, topic3) topic4 = self.pubsub_management.create_topic('topic4', exchange_point='xp1', parent_topic_id=topic2) self.addCleanup(self.pubsub_management.delete_topic, topic4) topic5 = self.pubsub_management.create_topic('topic5', exchange_point='xp1', parent_topic_id=topic2) self.addCleanup(self.pubsub_management.delete_topic, topic5) topic6 = self.pubsub_management.create_topic('topic6', exchange_point='xp1', parent_topic_id=topic3) self.addCleanup(self.pubsub_management.delete_topic, topic6) topic7 = self.pubsub_management.create_topic('topic7', exchange_point='xp1', parent_topic_id=topic3) self.addCleanup(self.pubsub_management.delete_topic, topic7) # Tree 2 topic8 = self.pubsub_management.create_topic('topic8', exchange_point='xp2') self.addCleanup(self.pubsub_management.delete_topic, topic8) topic9 = self.pubsub_management.create_topic('topic9', exchange_point='xp2', parent_topic_id=topic8) self.addCleanup(self.pubsub_management.delete_topic, topic9) topic10 = self.pubsub_management.create_topic('topic10', exchange_point='xp2', parent_topic_id=topic9) self.addCleanup(self.pubsub_management.delete_topic, topic10) topic11 = self.pubsub_management.create_topic('topic11', exchange_point='xp2', parent_topic_id=topic9) self.addCleanup(self.pubsub_management.delete_topic, topic11) topic12 = self.pubsub_management.create_topic('topic12', exchange_point='xp2', parent_topic_id=topic11) self.addCleanup(self.pubsub_management.delete_topic, topic12) topic13 = self.pubsub_management.create_topic('topic13', exchange_point='xp2', parent_topic_id=topic11) self.addCleanup(self.pubsub_management.delete_topic, topic13) self.exchange_cleanup.extend(['xp1','xp2']) stream1_id, route = self.pubsub_management.create_stream('stream1', topic_ids=[topic7, topic4, topic5], exchange_point='xp1') self.addCleanup(self.pubsub_management.delete_stream, stream1_id) stream2_id, route = self.pubsub_management.create_stream('stream2', topic_ids=[topic8], exchange_point='xp2') self.addCleanup(self.pubsub_management.delete_stream, stream2_id) stream3_id, route = self.pubsub_management.create_stream('stream3', topic_ids=[topic10,topic13], exchange_point='xp2') self.addCleanup(self.pubsub_management.delete_stream, stream3_id) stream4_id, route = self.pubsub_management.create_stream('stream4', topic_ids=[topic9], exchange_point='xp2') self.addCleanup(self.pubsub_management.delete_stream, stream4_id) stream5_id, route = self.pubsub_management.create_stream('stream5', topic_ids=[topic11], exchange_point='xp2') self.addCleanup(self.pubsub_management.delete_stream, stream5_id) subscription1 = self.pubsub_management.create_subscription('sub1', topic_ids=[topic1]) self.addCleanup(self.pubsub_management.delete_subscription, subscription1) subscription2 = self.pubsub_management.create_subscription('sub2', topic_ids=[topic8], exchange_name='sub1') self.addCleanup(self.pubsub_management.delete_subscription, subscription2) subscription3 = self.pubsub_management.create_subscription('sub3', topic_ids=[topic9], exchange_name='sub1') self.addCleanup(self.pubsub_management.delete_subscription, subscription3) subscription4 = self.pubsub_management.create_subscription('sub4', topic_ids=[topic10,topic13, topic11], exchange_name='sub1') self.addCleanup(self.pubsub_management.delete_subscription, subscription4) #-------------------------------------------------------------------------------- self.pubsub_management.activate_subscription(subscription1) self.publish_on_stream(stream1_id,1) self.assertEquals(self.msg_queue.get(timeout=10), 1) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.1) self.pubsub_management.deactivate_subscription(subscription1) #-------------------------------------------------------------------------------- self.pubsub_management.activate_subscription(subscription2) self.publish_on_stream(stream2_id,2) self.assertEquals(self.msg_queue.get(timeout=10), 2) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.1) self.pubsub_management.deactivate_subscription(subscription2) #-------------------------------------------------------------------------------- self.pubsub_management.activate_subscription(subscription3) self.publish_on_stream(stream2_id, 3) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.3) self.publish_on_stream(stream3_id, 4) self.assertEquals(self.msg_queue.get(timeout=10),4) self.pubsub_management.deactivate_subscription(subscription3) #-------------------------------------------------------------------------------- self.pubsub_management.activate_subscription(subscription4) self.publish_on_stream(stream4_id, 5) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.3) self.publish_on_stream(stream5_id, 6) self.assertEquals(self.msg_queue.get(timeout=10),6) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.3) self.pubsub_management.deactivate_subscription(subscription4) #-------------------------------------------------------------------------------- def cleanup_contexts(self): for context_id in self.context_ids: self.dataset_management.delete_parameter_context(context_id) def add_context_to_cleanup(self, context_id): self.context_ids.add(context_id) def _get_pdict(self, filter_values): t_ctxt = ParameterContext('TIME', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1900' t_ctxt_id = self.dataset_management.create_parameter_context(name='TIME', parameter_context=t_ctxt.dump(), parameter_type='quantity<int64>', units=t_ctxt.uom) self.add_context_to_cleanup(t_ctxt_id) lat_ctxt = ParameterContext('LAT', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt_id = self.dataset_management.create_parameter_context(name='LAT', parameter_context=lat_ctxt.dump(), parameter_type='quantity<float32>', units=lat_ctxt.uom) self.add_context_to_cleanup(lat_ctxt_id) lon_ctxt = ParameterContext('LON', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt_id = self.dataset_management.create_parameter_context(name='LON', parameter_context=lon_ctxt.dump(), parameter_type='quantity<float32>', units=lon_ctxt.uom) self.add_context_to_cleanup(lon_ctxt_id) # Independent Parameters # Temperature - values expected to be the decimal results of conversion from hex temp_ctxt = ParameterContext('TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) temp_ctxt.uom = 'deg_C' temp_ctxt_id = self.dataset_management.create_parameter_context(name='TEMPWAT_L0', parameter_context=temp_ctxt.dump(), parameter_type='quantity<float32>', units=temp_ctxt.uom) self.add_context_to_cleanup(temp_ctxt_id) # Conductivity - values expected to be the decimal results of conversion from hex cond_ctxt = ParameterContext('CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) cond_ctxt.uom = 'S m-1' cond_ctxt_id = self.dataset_management.create_parameter_context(name='CONDWAT_L0', parameter_context=cond_ctxt.dump(), parameter_type='quantity<float32>', units=cond_ctxt.uom) self.add_context_to_cleanup(cond_ctxt_id) # Pressure - values expected to be the decimal results of conversion from hex press_ctxt = ParameterContext('PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) press_ctxt.uom = 'dbar' press_ctxt_id = self.dataset_management.create_parameter_context(name='PRESWAT_L0', parameter_context=press_ctxt.dump(), parameter_type='quantity<float32>', units=press_ctxt.uom) self.add_context_to_cleanup(press_ctxt_id) # Dependent Parameters # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10 tl1_func = '(T / 10000) - 10' tl1_pmap = {'T': 'TEMPWAT_L0'} expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T'], param_map=tl1_pmap) tempL1_ctxt = ParameterContext('TEMPWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) tempL1_ctxt.uom = 'deg_C' tempL1_ctxt_id = self.dataset_management.create_parameter_context(name=tempL1_ctxt.name, parameter_context=tempL1_ctxt.dump(), parameter_type='pfunc', units=tempL1_ctxt.uom) self.add_context_to_cleanup(tempL1_ctxt_id) # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5 cl1_func = '(C / 100000) - 0.5' cl1_pmap = {'C': 'CONDWAT_L0'} expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C'], param_map=cl1_pmap) condL1_ctxt = ParameterContext('CONDWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) condL1_ctxt.uom = 'S m-1' condL1_ctxt_id = self.dataset_management.create_parameter_context(name=condL1_ctxt.name, parameter_context=condL1_ctxt.dump(), parameter_type='pfunc', units=condL1_ctxt.uom) self.add_context_to_cleanup(condL1_ctxt_id) # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721 # PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range) pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)' pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721} expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range'], param_map=pl1_pmap) presL1_ctxt = ParameterContext('PRESWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) presL1_ctxt.uom = 'S m-1' presL1_ctxt_id = self.dataset_management.create_parameter_context(name=presL1_ctxt.name, parameter_context=presL1_ctxt.dump(), parameter_type='pfunc', units=presL1_ctxt.uom) self.add_context_to_cleanup(presL1_ctxt_id) # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1) owner = 'gsw' sal_func = 'SP_from_C' sal_arglist = ['C', 't', 'p'] sal_pmap = {'C': NumexprFunction('CONDWAT_L1*10', 'C*10', ['C'], param_map={'C': 'CONDWAT_L1'}), 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'} sal_kwargmap = None expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist, sal_kwargmap, sal_pmap) sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL) sal_ctxt.uom = 'g kg-1' sal_ctxt_id = self.dataset_management.create_parameter_context(name=sal_ctxt.name, parameter_context=sal_ctxt.dump(), parameter_type='pfunc', units=sal_ctxt.uom) self.add_context_to_cleanup(sal_ctxt_id) # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1) # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1) owner = 'gsw' abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON','LAT']) cons_temp_expr = PythonFunction('cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1']) dens_expr = PythonFunction('DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1']) dens_ctxt = ParameterContext('DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL) dens_ctxt.uom = 'kg m-3' dens_ctxt_id = self.dataset_management.create_parameter_context(name=dens_ctxt.name, parameter_context=dens_ctxt.dump(), parameter_type='pfunc', units=dens_ctxt.uom) self.add_context_to_cleanup(dens_ctxt_id) ids = [t_ctxt_id, lat_ctxt_id, lon_ctxt_id, temp_ctxt_id, cond_ctxt_id, press_ctxt_id, tempL1_ctxt_id, condL1_ctxt_id, presL1_ctxt_id, sal_ctxt_id, dens_ctxt_id] contexts = [t_ctxt, lat_ctxt, lon_ctxt, temp_ctxt, cond_ctxt, press_ctxt, tempL1_ctxt, condL1_ctxt, presL1_ctxt, sal_ctxt, dens_ctxt] context_ids = [ids[i] for i,ctxt in enumerate(contexts) if ctxt.name in filter_values] pdict_name = '_'.join([ctxt.name for ctxt in contexts if ctxt.name in filter_values]) try: self.pdicts[pdict_name] return self.pdicts[pdict_name] except KeyError: pdict_id = self.dataset_management.create_parameter_dictionary(pdict_name, parameter_context_ids=context_ids, temporal_context='time') self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id) self.pdicts[pdict_name] = pdict_id return pdict_id
class RecordDictionaryIntegrationTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dataset_management = DatasetManagementServiceClient() self.pubsub_management = PubsubManagementServiceClient() self.rdt = None self.data_producer_id = None self.provider_metadata_update = None self.event = Event() def verify_incoming(self, m,r,s): rdt = RecordDictionaryTool.load_from_granule(m) for k,v in rdt.iteritems(): np.testing.assert_array_equal(v, self.rdt[k]) self.assertEquals(m.data_producer_id, self.data_producer_id) self.assertEquals(m.provider_metadata_update, self.provider_metadata_update) self.assertNotEqual(m.creation_timestamp, None) self.event.set() def test_serialize_compatability(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition('ctd extended', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) stream_id, route = self.pubsub_management.create_stream('ctd1', 'xp1', stream_definition_id=stream_def_id) self.addCleanup(self.pubsub_management.delete_stream, stream_id) sub_id = self.pubsub_management.create_subscription('sub1', stream_ids=[stream_id]) self.addCleanup(self.pubsub_management.delete_subscription, sub_id) self.pubsub_management.activate_subscription(sub_id) self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id) verified = Event() def verifier(msg, route, stream_id): for k,v in msg.record_dictionary.iteritems(): if v is not None: self.assertIsInstance(v, np.ndarray) rdt = RecordDictionaryTool.load_from_granule(msg) for k,v in rdt.iteritems(): self.assertIsInstance(rdt[k], np.ndarray) self.assertIsInstance(v, np.ndarray) verified.set() subscriber = StandaloneStreamSubscriber('sub1', callback=verifier) subscriber.start() self.addCleanup(subscriber.stop) publisher = StandaloneStreamPublisher(stream_id,route) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) ph.fill_rdt(rdt,10) publisher.publish(rdt.to_granule()) self.assertTrue(verified.wait(60)) def test_granule(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) stream_def_id = self.pubsub_management.create_stream_definition('ctd', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':"GA03FLMA-RI001-13-CTDMOG999"}) pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict') self.addCleanup(self.pubsub_management.delete_stream_definition,stream_def_id) stream_id, route = self.pubsub_management.create_stream('ctd_stream', 'xp1', stream_definition_id=stream_def_id) self.addCleanup(self.pubsub_management.delete_stream,stream_id) publisher = StandaloneStreamPublisher(stream_id, route) subscriber = StandaloneStreamSubscriber('sub', self.verify_incoming) subscriber.start() self.addCleanup(subscriber.stop) subscription_id = self.pubsub_management.create_subscription('sub', stream_ids=[stream_id]) self.pubsub_management.activate_subscription(subscription_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) rdt['temp'] = np.random.randn(10) * 10 + 30 rdt['pressure'] = [20] * 10 self.assertEquals(set(pdict.keys()), set(rdt.fields)) self.assertEquals(pdict.temporal_parameter_name, rdt.temporal_parameter) self.assertEquals(rdt._stream_config['reference_designator'],"GA03FLMA-RI001-13-CTDMOG999") self.rdt = rdt self.data_producer_id = 'data_producer' self.provider_metadata_update = {1:1} publisher.publish(rdt.to_granule(data_producer_id='data_producer', provider_metadata_update={1:1})) self.assertTrue(self.event.wait(10)) self.pubsub_management.deactivate_subscription(subscription_id) self.pubsub_management.delete_subscription(subscription_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.array([None,None,None]) self.assertTrue(rdt['time'] is None) rdt['time'] = np.array([None, 1, 2]) self.assertEquals(rdt['time'][0], rdt.fill_value('time')) stream_def_obj = self.pubsub_management.read_stream_definition(stream_def_id) rdt = RecordDictionaryTool(stream_definition=stream_def_obj) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) granule = rdt.to_granule() rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_equal(rdt['time'], np.arange(20)) np.testing.assert_array_equal(rdt['temp'], np.arange(20)) def test_filter(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) filtered_stream_def_id = self.pubsub_management.create_stream_definition('filtered', parameter_dictionary_id=pdict_id, available_fields=['time', 'temp']) self.addCleanup(self.pubsub_management.delete_stream_definition, filtered_stream_def_id) rdt = RecordDictionaryTool(stream_definition_id=filtered_stream_def_id) self.assertEquals(rdt._available_fields,['time','temp']) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) with self.assertRaises(KeyError): rdt['pressure'] = np.arange(20) granule = rdt.to_granule() rdt2 = RecordDictionaryTool.load_from_granule(granule) self.assertEquals(rdt._available_fields, rdt2._available_fields) self.assertEquals(rdt.fields, rdt2.fields) for k,v in rdt.iteritems(): self.assertTrue(np.array_equal(rdt[k], rdt2[k])) def test_rdt_param_funcs(self): param_funcs = { 'identity' : { 'function_type' : PFT.PYTHON, 'owner' : 'ion_functions.data.interpolation', 'function' : 'identity', 'args':['x'] }, 'ctd_tempwat' : { 'function_type' : PFT.PYTHON, 'owner' : 'ion_functions.data.ctd_functions', 'function' : 'ctd_sbe37im_tempwat', 'args' : ['t0'] }, 'ctd_preswat' : { 'function_type' : PFT.PYTHON, 'owner' : 'ion_functions.data.ctd_functions', 'function' : 'ctd_sbe37im_preswat', 'args' : ["p0", "p_range_psia"] }, 'ctd_condwat' : { 'function_type' : PFT.PYTHON, 'owner' : 'ion_functions.data.ctd_functions', 'function' : 'ctd_sbe37im_condwat', 'args' : ['c0'] }, 'ctd_pracsal' : { 'function_type' : PFT.PYTHON, 'owner' : 'ion_functions.data.ctd_functions', 'function' : 'ctd_pracsal', 'args' : ['c', 't', 'p'] }, 'ctd_density' : { 'function_type' : PFT.PYTHON, 'owner' : 'ion_functions.data.ctd_functions', 'function' : 'ctd_density', 'args' : ['SP','t','p','lat','lon'] } } pfunc_ids = {} for name, param_def in param_funcs.iteritems(): paramfunc = ParameterFunction(name, **param_def) pf_id = self.dataset_management.create_parameter_function(paramfunc) pfunc_ids[name] = pf_id params = { 'time' : { 'parameter_type' : 'quantity', 'value_encoding' : 'float64', 'units' : 'seconds since 1900-01-01' }, 'temperature_counts' : { 'parameter_type' : 'quantity', 'value_encoding' : 'float32', 'units' : '1' }, 'pressure_counts' : { 'parameter_type' : 'quantity', 'value_encoding' : 'float32', 'units' : '1' }, 'conductivity_counts' : { 'parameter_type' : 'quantity', 'value_encoding' : 'float32', 'units' : '1' }, 'temperature' : { 'parameter_type' : 'function', 'parameter_function_id' : pfunc_ids['ctd_tempwat'], 'parameter_function_map' : { 't0' : 'temperature_counts'}, 'value_encoding' : 'float32', 'units' : 'deg_C' }, 'pressure' : { 'parameter_type' : 'function', 'parameter_function_id' : pfunc_ids['ctd_preswat'], 'parameter_function_map' : {'p0' : 'pressure_counts', 'p_range_psia' : 679.34040721}, 'value_encoding' : 'float32', 'units' : 'dbar' }, 'conductivity' : { 'parameter_type' : 'function', 'parameter_function_id' : pfunc_ids['ctd_condwat'], 'parameter_function_map' : {'c0' : 'conductivity_counts'}, 'value_encoding' : 'float32', 'units' : 'Sm-1' }, 'salinity' : { 'parameter_type' : 'function', 'parameter_function_id' : pfunc_ids['ctd_pracsal'], 'parameter_function_map' : {'c' : 'conductivity', 't' : 'temperature', 'p' : 'pressure'}, 'value_encoding' : 'float32', 'units' : '1' }, 'density' : { 'parameter_type' : 'function', 'parameter_function_id' : pfunc_ids['ctd_density'], 'parameter_function_map' : { 'SP' : 'salinity', 't' : 'temperature', 'p' : 'pressure', 'lat' : 'lat', 'lon' : 'lon' }, 'value_encoding' : 'float32', 'units' : 'kg m-1' }, 'lat' : { 'parameter_type' : 'sparse', 'value_encoding' : 'float32', 'units' : 'degrees_north' }, 'lon' : { 'parameter_type' : 'sparse', 'value_encoding' : 'float32', 'units' : 'degrees_east' } } param_dict = {} for name, param in params.iteritems(): pcontext = ParameterContext(name, **param) param_id = self.dataset_management.create_parameter(pcontext) param_dict[name] = param_id pdict_id = self.dataset_management.create_parameter_dictionary('ctd_test', param_dict.values(), 'time') stream_def_id = self.pubsub_management.create_stream_definition('ctd_test', parameter_dictionary_id=pdict_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = [0] rdt['temperature_counts'] = [280000] rdt['conductivity_counts'] = [100000] rdt['pressure_counts'] = [2789] rdt['lat'] = [45] rdt['lon'] = [-71] np.testing.assert_allclose(rdt['density'], np.array([1001.00543606])) def test_rdt_lookup(self): rdt = self.create_lookup_rdt() self.assertTrue('offset_a' in rdt.lookup_values()) self.assertFalse('offset_b' in rdt.lookup_values()) rdt['time'] = [0] rdt['temp'] = [10.0] rdt['offset_a'] = [2.0] self.assertEquals(rdt['offset_b'], None) self.assertEquals(rdt.lookup_values(), ['offset_a']) np.testing.assert_array_almost_equal(rdt['calibrated'], np.array([12.0])) svm = StoredValueManager(self.container) svm.stored_value_cas('coefficient_document', {'offset_b':2.0}) svm.stored_value_cas("GA03FLMA-RI001-13-CTDMOG999_OFFSETC", {'offset_c':3.0}) rdt.fetch_lookup_values() np.testing.assert_array_equal(rdt['offset_b'], np.array([2.0])) np.testing.assert_array_equal(rdt['calibrated_b'], np.array([14.0])) np.testing.assert_array_equal(rdt['offset_c'], np.array([3.0])) def create_rdt(self): contexts, pfuncs = self.create_pfuncs() context_ids = list(contexts.itervalues()) pdict_id = self.dataset_management.create_parameter_dictionary(name='functional_pdict', parameter_context_ids=context_ids, temporal_context='test_TIME') self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id) stream_def_id = self.pubsub_management.create_stream_definition('functional', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) return rdt def create_lookup_rdt(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_lookups() stream_def_id = self.pubsub_management.create_stream_definition('lookup', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':"GA03FLMA-RI001-13-CTDMOG999"}) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) return rdt def create_pfuncs(self): contexts = {} funcs = {} t_ctxt = ParameterContext(name='TIME', parameter_type='quantity', value_encoding='float64', units='seconds since 1900-01-01') t_ctxt_id = self.dataset_management.create_parameter(t_ctxt) contexts['TIME'] = t_ctxt_id lat_ctxt = ParameterContext(name='LAT', parameter_type="sparse", value_encoding='float32', units='degrees_north') lat_ctxt_id = self.dataset_management.create_parameter(lat_ctxt) contexts['LAT'] = lat_ctxt_id lon_ctxt = ParameterContext(name='LON', parameter_type='sparse', value_encoding='float32', units='degrees_east') lon_ctxt_id = self.dataset_management.create_parameter(lon_ctxt) contexts['LON'] = lon_ctxt_id # Independent Parameters # Temperature - values expected to be the decimal results of conversion from hex temp_ctxt = ParameterContext(name='TEMPWAT_L0', parameter_type='quantity', value_encoding='float32', units='deg_C') temp_ctxt_id = self.dataset_management.create_parameter(temp_ctxt) contexts['TEMPWAT_L0'] = temp_ctxt_id # Conductivity - values expected to be the decimal results of conversion from hex cond_ctxt = ParameterContext(name='CONDWAT_L0', parameter_type='quantity', value_encoding='float32', units='S m-1') cond_ctxt_id = self.dataset_management.create_parameter(cond_ctxt) contexts['CONDWAT_L0'] = cond_ctxt_id # Pressure - values expected to be the decimal results of conversion from hex press_ctxt = ParameterContext(name='PRESWAT_L0', parameter_type='quantity', value_encoding='float32', units='dbar') press_ctxt_id = self.dataset_management.create_parameter(press_ctxt) contexts['PRESWAT_L0'] = press_ctxt_id # Dependent Parameters # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10 tl1_func = '(T / 10000) - 10' expr = ParameterFunction(name='TEMPWAT_L1', function_type=PFT.NUMEXPR, function=tl1_func, args=['T']) expr_id = self.dataset_management.create_parameter_function(expr) funcs['TEMPWAT_L1'] = expr_id tl1_pmap = {'T': 'TEMPWAT_L0'} tempL1_ctxt = ParameterContext(name='TEMPWAT_L1', parameter_type='function', parameter_function_id=expr_id, parameter_function_map=tl1_pmap, value_encoding='float32', units='deg_C') tempL1_ctxt_id = self.dataset_management.create_parameter(tempL1_ctxt) contexts['TEMPWAT_L1'] = tempL1_ctxt_id # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5 cl1_func = '(C / 100000) - 0.5' expr = ParameterFunction(name='CONDWAT_L1', function_type=PFT.NUMEXPR, function=cl1_func, args=['C']) expr_id = self.dataset_management.create_parameter_function(expr) funcs['CONDWAT_L1'] = expr_id cl1_pmap = {'C': 'CONDWAT_L0'} condL1_ctxt = ParameterContext(name='CONDWAT_L1', parameter_type='function', parameter_function_id=expr_id, parameter_function_map=cl1_pmap, value_encoding='float32', units='S m-1') condL1_ctxt_id = self.dataset_management.create_parameter(condL1_ctxt) contexts['CONDWAT_L1'] = condL1_ctxt_id # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721 # PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range) pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)' expr = ParameterFunction(name='PRESWAT_L1',function=pl1_func,function_type=PFT.NUMEXPR,args=['P','p_range']) expr_id = self.dataset_management.create_parameter_function(expr) funcs['PRESWAT_L1'] = expr_id pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721} presL1_ctxt = ParameterContext(name='PRESWAT_L1', parameter_type='function', parameter_function_id=expr_id, parameter_function_map=pl1_pmap, value_encoding='float32', units='S m-1') presL1_ctxt_id = self.dataset_management.create_parameter(presL1_ctxt) contexts['PRESWAT_L1'] = presL1_ctxt_id # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1) owner = 'gsw' sal_func = 'SP_from_C' sal_arglist = ['C', 't', 'p'] expr = ParameterFunction(name='PRACSAL',function_type=PFT.PYTHON,function=sal_func,owner=owner,args=sal_arglist) expr_id = self.dataset_management.create_parameter_function(expr) funcs['PRACSAL'] = expr_id c10_f = ParameterFunction(name='c10', function_type=PFT.NUMEXPR, function='C*10', args=['C']) expr_id = self.dataset_management.create_parameter_function(c10_f) c10 = ParameterContext(name='c10', parameter_type='function', parameter_function_id=expr_id, parameter_function_map={'C':'CONDWAT_L1'}, value_encoding='float32', units='1') c10_id = self.dataset_management.create_parameter(c10) contexts['c10'] = c10_id # A magic function that may or may not exist actually forms the line below at runtime. sal_pmap = {'C': 'c10', 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'} sal_ctxt = ParameterContext(name='PRACSAL', parameter_type='function', parameter_function_id=expr_id, parameter_function_map=sal_pmap, value_encoding='float32', units='g kg-1') sal_ctxt_id = self.dataset_management.create_parameter(sal_ctxt) contexts['PRACSAL'] = sal_ctxt_id # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1) # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1) owner = 'gsw' abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON','LAT']) cons_temp_expr = PythonFunction('cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1']) dens_expr = PythonFunction('DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1']) dens_ctxt = CoverageParameterContext('DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL) dens_ctxt.uom = 'kg m-3' dens_ctxt_id = self.dataset_management.create_parameter_context(name='DENSITY', parameter_context=dens_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, dens_ctxt_id) contexts['DENSITY'] = dens_ctxt_id return contexts, funcs
class RecordDictionaryIntegrationTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dataset_management = DatasetManagementServiceClient() self.pubsub_management = PubsubManagementServiceClient() self.rdt = None self.data_producer_id = None self.provider_metadata_update = None self.event = Event() def verify_incoming(self, m,r,s): rdt = RecordDictionaryTool.load_from_granule(m) self.assertEquals(rdt, self.rdt) self.assertEquals(m.data_producer_id, self.data_producer_id) self.assertEquals(m.provider_metadata_update, self.provider_metadata_update) self.assertNotEqual(m.creation_timestamp, None) self.event.set() def test_serialize_compatability(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition('ctd extended', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) stream_id, route = self.pubsub_management.create_stream('ctd1', 'xp1', stream_definition_id=stream_def_id) self.addCleanup(self.pubsub_management.delete_stream, stream_id) sub_id = self.pubsub_management.create_subscription('sub1', stream_ids=[stream_id]) self.addCleanup(self.pubsub_management.delete_subscription, sub_id) self.pubsub_management.activate_subscription(sub_id) self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id) verified = Event() def verifier(msg, route, stream_id): for k,v in msg.record_dictionary.iteritems(): if v is not None: self.assertIsInstance(v, np.ndarray) rdt = RecordDictionaryTool.load_from_granule(msg) for field in rdt.fields: self.assertIsInstance(rdt[field], np.ndarray) verified.set() subscriber = StandaloneStreamSubscriber('sub1', callback=verifier) subscriber.start() self.addCleanup(subscriber.stop) publisher = StandaloneStreamPublisher(stream_id,route) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) ph.fill_rdt(rdt,10) publisher.publish(rdt.to_granule()) self.assertTrue(verified.wait(10)) def test_granule(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) stream_def_id = self.pubsub_management.create_stream_definition('ctd', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':"GA03FLMA-RI001-13-CTDMOG999"}) pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict') self.addCleanup(self.pubsub_management.delete_stream_definition,stream_def_id) stream_id, route = self.pubsub_management.create_stream('ctd_stream', 'xp1', stream_definition_id=stream_def_id) self.addCleanup(self.pubsub_management.delete_stream,stream_id) publisher = StandaloneStreamPublisher(stream_id, route) subscriber = StandaloneStreamSubscriber('sub', self.verify_incoming) subscriber.start() self.addCleanup(subscriber.stop) subscription_id = self.pubsub_management.create_subscription('sub', stream_ids=[stream_id]) self.pubsub_management.activate_subscription(subscription_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) rdt['temp'] = np.random.randn(10) * 10 + 30 rdt['pressure'] = [20] * 10 self.assertEquals(set(pdict.keys()), set(rdt.fields)) self.assertEquals(pdict.temporal_parameter_name, rdt.temporal_parameter) self.assertEquals(rdt._stream_config['reference_designator'],"GA03FLMA-RI001-13-CTDMOG999") self.rdt = rdt self.data_producer_id = 'data_producer' self.provider_metadata_update = {1:1} publisher.publish(rdt.to_granule(data_producer_id='data_producer', provider_metadata_update={1:1})) self.assertTrue(self.event.wait(10)) self.pubsub_management.deactivate_subscription(subscription_id) self.pubsub_management.delete_subscription(subscription_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.array([None,None,None]) self.assertTrue(rdt['time'] is None) rdt['time'] = np.array([None, 1, 2]) self.assertEquals(rdt['time'][0], rdt.fill_value('time')) stream_def_obj = self.pubsub_management.read_stream_definition(stream_def_id) rdt = RecordDictionaryTool(stream_definition=stream_def_obj) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) granule = rdt.to_granule() rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_equal(rdt['time'], np.arange(20)) np.testing.assert_array_equal(rdt['temp'], np.arange(20)) def test_filter(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) filtered_stream_def_id = self.pubsub_management.create_stream_definition('filtered', parameter_dictionary_id=pdict_id, available_fields=['time', 'temp']) self.addCleanup(self.pubsub_management.delete_stream_definition, filtered_stream_def_id) rdt = RecordDictionaryTool(stream_definition_id=filtered_stream_def_id) self.assertEquals(rdt._available_fields,['time','temp']) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) with self.assertRaises(KeyError): rdt['pressure'] = np.arange(20) granule = rdt.to_granule(connection_id='c1', connection_index='0') rdt2 = RecordDictionaryTool.load_from_granule(granule) self.assertEquals(rdt._available_fields, rdt2._available_fields) self.assertEquals(rdt.fields, rdt2.fields) self.assertEquals(rdt2.connection_id,'c1') self.assertEquals(rdt2.connection_index,'0') for k,v in rdt.iteritems(): self.assertTrue(np.array_equal(rdt[k], rdt2[k])) def test_rdt_param_funcs(self): rdt = self.create_rdt() rdt['TIME'] = [0] rdt['TEMPWAT_L0'] = [280000] rdt['CONDWAT_L0'] = [100000] rdt['PRESWAT_L0'] = [2789] rdt['LAT'] = [45] rdt['LON'] = [-71] np.testing.assert_array_almost_equal(rdt['DENSITY'], np.array([1001.76506258], dtype='float32')) def test_rdt_lookup(self): rdt = self.create_lookup_rdt() self.assertTrue('offset_a' in rdt.lookup_values()) self.assertFalse('offset_b' in rdt.lookup_values()) rdt['time'] = [0] rdt['temp'] = [10.0] rdt['offset_a'] = [2.0] self.assertEquals(rdt['offset_b'], None) self.assertEquals(rdt.lookup_values(), ['offset_a']) np.testing.assert_array_almost_equal(rdt['calibrated'], np.array([12.0])) svm = StoredValueManager(self.container) svm.stored_value_cas('coefficient_document', {'offset_b':2.0}) svm.stored_value_cas("GA03FLMA-RI001-13-CTDMOG999_OFFSETC", {'offset_c':3.0}) rdt.fetch_lookup_values() np.testing.assert_array_equal(rdt['offset_b'], np.array([2.0])) np.testing.assert_array_equal(rdt['calibrated_b'], np.array([14.0])) np.testing.assert_array_equal(rdt['offset_c'], np.array([3.0])) def create_rdt(self): contexts, pfuncs = self.create_pfuncs() context_ids = [_id for ct,_id in contexts.itervalues()] pdict_id = self.dataset_management.create_parameter_dictionary(name='functional_pdict', parameter_context_ids=context_ids, temporal_context='test_TIME') self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id) stream_def_id = self.pubsub_management.create_stream_definition('functional', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) return rdt def create_lookup_rdt(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_lookups() stream_def_id = self.pubsub_management.create_stream_definition('lookup', parameter_dictionary_id=pdict_id, stream_configuration={'reference_designator':"GA03FLMA-RI001-13-CTDMOG999"}) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) return rdt def create_pfuncs(self): contexts = {} funcs = {} t_ctxt = ParameterContext('TIME', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 1900-01-01' t_ctxt_id = self.dataset_management.create_parameter_context(name='test_TIME', parameter_context=t_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, t_ctxt_id) contexts['TIME'] = (t_ctxt, t_ctxt_id) lat_ctxt = ParameterContext('LAT', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt_id = self.dataset_management.create_parameter_context(name='test_LAT', parameter_context=lat_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, lat_ctxt_id) contexts['LAT'] = lat_ctxt, lat_ctxt_id lon_ctxt = ParameterContext('LON', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt_id = self.dataset_management.create_parameter_context(name='test_LON', parameter_context=lon_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, lon_ctxt_id) contexts['LON'] = lon_ctxt, lon_ctxt_id # Independent Parameters # Temperature - values expected to be the decimal results of conversion from hex temp_ctxt = ParameterContext('TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) temp_ctxt.uom = 'deg_C' temp_ctxt_id = self.dataset_management.create_parameter_context(name='test_TEMPWAT_L0', parameter_context=temp_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, temp_ctxt_id) contexts['TEMPWAT_L0'] = temp_ctxt, temp_ctxt_id # Conductivity - values expected to be the decimal results of conversion from hex cond_ctxt = ParameterContext('CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) cond_ctxt.uom = 'S m-1' cond_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L0', parameter_context=cond_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, cond_ctxt_id) contexts['CONDWAT_L0'] = cond_ctxt, cond_ctxt_id # Pressure - values expected to be the decimal results of conversion from hex press_ctxt = ParameterContext('PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) press_ctxt.uom = 'dbar' press_ctxt_id = self.dataset_management.create_parameter_context(name='test_PRESWAT_L0', parameter_context=press_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, press_ctxt_id) contexts['PRESWAT_L0'] = press_ctxt, press_ctxt_id # Dependent Parameters # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10 tl1_func = '(T / 10000) - 10' expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T']) expr_id = self.dataset_management.create_parameter_function(name='test_TEMPWAT_L1', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['TEMPWAT_L1'] = expr, expr_id tl1_pmap = {'T': 'TEMPWAT_L0'} expr.param_map = tl1_pmap tempL1_ctxt = ParameterContext('TEMPWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) tempL1_ctxt.uom = 'deg_C' tempL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_TEMPWAT_L1', parameter_context=tempL1_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, tempL1_ctxt_id) contexts['TEMPWAT_L1'] = tempL1_ctxt, tempL1_ctxt_id # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5 cl1_func = '(C / 100000) - 0.5' expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C']) expr_id = self.dataset_management.create_parameter_function(name='test_CONDWAT_L1', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['CONDWAT_L1'] = expr, expr_id cl1_pmap = {'C': 'CONDWAT_L0'} expr.param_map = cl1_pmap condL1_ctxt = ParameterContext('CONDWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) condL1_ctxt.uom = 'S m-1' condL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L1', parameter_context=condL1_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, condL1_ctxt_id) contexts['CONDWAT_L1'] = condL1_ctxt, condL1_ctxt_id # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721 # PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range) pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)' expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range']) expr_id = self.dataset_management.create_parameter_function(name='test_PRESWAT_L1', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['PRESWAT_L1'] = expr, expr_id pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721} expr.param_map = pl1_pmap presL1_ctxt = ParameterContext('PRESWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) presL1_ctxt.uom = 'S m-1' presL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L1', parameter_context=presL1_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, presL1_ctxt_id) contexts['PRESWAT_L1'] = presL1_ctxt, presL1_ctxt_id # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1) owner = 'gsw' sal_func = 'SP_from_C' sal_arglist = ['C', 't', 'p'] expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist) expr_id = self.dataset_management.create_parameter_function(name='test_PRACSAL', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['PRACSAL'] = expr, expr_id # A magic function that may or may not exist actually forms the line below at runtime. sal_pmap = {'C': NumexprFunction('CONDWAT_L1*10', 'C*10', ['C'], param_map={'C': 'CONDWAT_L1'}), 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'} expr.param_map = sal_pmap sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL) sal_ctxt.uom = 'g kg-1' sal_ctxt_id = self.dataset_management.create_parameter_context(name='test_PRACSAL', parameter_context=sal_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, sal_ctxt_id) contexts['PRACSAL'] = sal_ctxt, sal_ctxt_id # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1) # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1) owner = 'gsw' abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON','LAT']) cons_temp_expr = PythonFunction('cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1']) dens_expr = PythonFunction('DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1']) dens_ctxt = ParameterContext('DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL) dens_ctxt.uom = 'kg m-3' dens_ctxt_id = self.dataset_management.create_parameter_context(name='test_DENSITY', parameter_context=dens_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, dens_ctxt_id) contexts['DENSITY'] = dens_ctxt, dens_ctxt_id return contexts, funcs
def read_stream_def(stream_def_id): pubsub_cli = PubsubManagementServiceClient() stream_def_obj = pubsub_cli.read_stream_definition(stream_def_id) return stream_def_obj
class TestDMEnd2End(IonIntegrationTestCase): def setUp(self): # Love the non pep-8 convention self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.process_dispatcher = ProcessDispatcherServiceClient() self.pubsub_management = PubsubManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() self.ingestion_management = IngestionManagementServiceClient() self.data_retriever = DataRetrieverServiceClient() self.pids = [] self.event = Event() self.exchange_space_name = 'test_granules' self.exchange_point_name = 'science_data' self.purge_queues() self.queue_buffer = [] def purge_queues(self): xn = self.container.ex_manager.create_xn_queue('science_granule_ingestion') xn.purge() def tearDown(self): self.purge_queues() for pid in self.pids: self.container.proc_manager.terminate_process(pid) IngestionManagementIntTest.clean_subscriptions() for queue in self.queue_buffer: if isinstance(queue, ExchangeNameQueue): queue.delete() elif isinstance(queue, str): xn = self.container.ex_manager.create_xn_queue(queue) xn.delete() def launch_producer(self, stream_id=''): #-------------------------------------------------------------------------------- # Launch the producer #-------------------------------------------------------------------------------- pid = self.container.spawn_process('better_data_producer', 'ion.processes.data.example_data_producer', 'BetterDataProducer', {'process':{'stream_id':stream_id}}) self.pids.append(pid) def get_ingestion_config(self): #-------------------------------------------------------------------------------- # Grab the ingestion configuration from the resource registry #-------------------------------------------------------------------------------- # The ingestion configuration should have been created by the bootstrap service # which is configured through r2deploy.yml ingest_configs, _ = self.resource_registry.find_resources(restype=RT.IngestionConfiguration,id_only=True) return ingest_configs[0] def publish_hifi(self,stream_id,stream_route,offset=0): pub = StandaloneStreamPublisher(stream_id, stream_route) stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id) stream_def_id = stream_def._id rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) + (offset * 10) rdt['temp'] = np.arange(10) + (offset * 10) pub.publish(rdt.to_granule()) def publish_fake_data(self,stream_id, route): for i in xrange(4): self.publish_hifi(stream_id,route,i) def get_datastore(self, dataset_id): dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def validate_granule_subscription(self, msg, route, stream_id): if msg == {}: return rdt = RecordDictionaryTool.load_from_granule(msg) log.info('%s', rdt.pretty_print()) self.assertIsInstance(msg,Granule,'Message is improperly formatted. (%s)' % type(msg)) self.event.set() def make_file_data(self): from interface.objects import File import uuid data = 'hello world\n' rand = str(uuid.uuid4())[:8] meta = File(name='/examples/' + rand + '.txt', group_id='example1') return {'body': data, 'meta':meta} def publish_file(self, stream_id, stream_route): publisher = StandaloneStreamPublisher(stream_id,stream_route) publisher.publish(self.make_file_data()) def wait_until_we_have_enough_granules(self, dataset_id='',granules=4): datastore = self.get_datastore(dataset_id) dataset = self.dataset_management.read_dataset(dataset_id) with gevent.timeout.Timeout(40): success = False while not success: success = len(datastore.query_view(dataset.view_name)) >= granules gevent.sleep(0.1) log.info(datastore.query_view(dataset.view_name)) def wait_until_we_have_enough_files(self): datastore = self.container.datastore_manager.get_datastore('filesystem', DataStore.DS_PROFILE.FILESYSTEM) now = time.time() timeout = now + 10 done = False while not done: if now >= timeout: raise Timeout('Files are not populating in time.') if len(datastore.query_view('catalog/file_by_owner')) >= 1: done = True now = time.time() def create_dataset(self, parameter_dict_id=''): tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() if not parameter_dict_id: parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) dataset_id = self.dataset_management.create_dataset('test_dataset', parameter_dictionary_id=parameter_dict_id, spatial_domain=sdom, temporal_domain=tdom) return dataset_id @unittest.skip('Doesnt work') @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_replay_pause(self): # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id) replay_stream, replay_route = self.pubsub_management.create_stream('replay', 'xp1', stream_definition_id=stream_def_id) dataset_id = self.create_dataset(pdict_id) scov = DatasetManagementService._get_coverage(dataset_id) bb = CoverageCraft(scov) bb.rdt['time'] = np.arange(100) bb.rdt['temp'] = np.random.random(100) + 30 bb.sync_with_granule() DatasetManagementService._persist_coverage(dataset_id, bb.coverage) # This invalidates it for multi-host configurations # Set up the subscriber to verify the data subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription) xp = self.container.ex_manager.create_xp('xp1') self.queue_buffer.append(self.exchange_space_name) subscriber.start() subscriber.xn.bind(replay_route.routing_key, xp) # Set up the replay agent and the client wrapper # 1) Define the Replay (dataset and stream to publish on) self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream) # 2) Make a client to the interact with the process (optionall provide it a process to bind with) replay_client = ReplayClient(process_id) # 3) Start the agent (launch the process) self.data_retriever.start_replay_agent(self.replay_id) # 4) Start replaying... replay_client.start_replay() # Wait till we get some granules self.assertTrue(self.event.wait(5)) # We got granules, pause the replay, clear the queue and allow the process to finish consuming replay_client.pause_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure there's no remaining messages being consumed self.assertFalse(self.event.wait(1)) # Resume the replay and wait until we start getting granules again replay_client.resume_replay() self.assertTrue(self.event.wait(5)) # Stop the replay, clear the queues replay_client.stop_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure that it did indeed stop self.assertFalse(self.event.wait(1)) subscriber.stop() @attr('SMOKE') def test_dm_end_2_end(self): #-------------------------------------------------------------------------------- # Set up a stream and have a mock instrument (producer) send data #-------------------------------------------------------------------------------- self.event.clear() # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_definition = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream('producer', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition) #-------------------------------------------------------------------------------- # Start persisting the data on the stream # - Get the ingestion configuration from the resource registry # - Create the dataset # - call persist_data_stream to setup the subscription for the ingestion workers # on the stream that you specify which causes the data to be persisted #-------------------------------------------------------------------------------- ingest_config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id) #-------------------------------------------------------------------------------- # Now the granules are ingesting and persisted #-------------------------------------------------------------------------------- self.launch_producer(stream_id) self.wait_until_we_have_enough_granules(dataset_id,4) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_id) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) self.assertTrue((rdt['time'][:10] == np.arange(10)).all(),'%s' % rdt['time'][:]) self.assertTrue((rdt['binary'][:10] == np.array(['hi']*10, dtype='object')).all()) #-------------------------------------------------------------------------------- # Now to try the streamed approach #-------------------------------------------------------------------------------- replay_stream_id, replay_route = self.pubsub_management.create_stream('replay_out', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition) self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream_id) log.info('Process ID: %s', process_id) replay_client = ReplayClient(process_id) #-------------------------------------------------------------------------------- # Create the listening endpoint for the the retriever to talk to #-------------------------------------------------------------------------------- xp = self.container.ex_manager.create_xp(self.exchange_point_name) subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription) self.queue_buffer.append(self.exchange_space_name) subscriber.start() subscriber.xn.bind(replay_route.routing_key, xp) self.data_retriever.start_replay_agent(self.replay_id) self.assertTrue(replay_client.await_agent_ready(5), 'The process never launched') replay_client.start_replay() self.assertTrue(self.event.wait(10)) subscriber.stop() self.data_retriever.cancel_replay_agent(self.replay_id) #-------------------------------------------------------------------------------- # Test the slicing capabilities #-------------------------------------------------------------------------------- granule = self.data_retriever.retrieve(dataset_id=dataset_id, query={'tdoa':slice(0,5)}) rdt = RecordDictionaryTool.load_from_granule(granule) b = rdt['time'] == np.arange(5) self.assertTrue(b.all() if not isinstance(b,bool) else b) def test_retrieve_and_transform(self): # Stream definition for the CTD data pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) stream_def_id = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id) ctd_stream_id, route = self.pubsub_management.create_stream('ctd stream', 'xp1', stream_definition_id=stream_def_id) # Stream definition for the salinity data salinity_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) sal_stream_def_id = self.pubsub_management.create_stream_definition('sal data', parameter_dictionary_id=salinity_pdict_id) ingest_config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) #-------------------------------------------------------------------------------- # Again with this ridiculous problem #-------------------------------------------------------------------------------- self.get_datastore(dataset_id) self.ingestion_management.persist_data_stream(stream_id=ctd_stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) rdt['temp'] = np.random.randn(10) * 10 + 30 rdt['conductivity'] = np.random.randn(10) * 2 + 10 publisher = StandaloneStreamPublisher(ctd_stream_id, route) publisher.publish(rdt.to_granule()) rdt['time'] = np.arange(10,20) publisher.publish(rdt.to_granule()) self.wait_until_we_have_enough_granules(dataset_id, 2) granule = self.data_retriever.retrieve(dataset_id, None, None, 'ion.processes.data.transforms.ctd.ctd_L2_salinity', 'CTDL2SalinityTransformAlgorithm', kwargs=dict(params=sal_stream_def_id)) rdt = RecordDictionaryTool.load_from_granule(granule) for i in rdt['salinity']: self.assertNotEquals(i,0) def test_last_granule(self): #-------------------------------------------------------------------------------- # Create the necessary configurations for the test #-------------------------------------------------------------------------------- pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) stream_def_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream('last_granule', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id) #-------------------------------------------------------------------------------- # Create the datastore first, #-------------------------------------------------------------------------------- self.get_datastore(dataset_id) self.publish_hifi(stream_id,route, 0) self.publish_hifi(stream_id,route, 1) self.wait_until_we_have_enough_granules(dataset_id,2) # I just need two success = False def verifier(): replay_granule = self.data_retriever.retrieve_last_granule(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(10) + 10 if not isinstance(comp,bool): return comp.all() return False success = poll(verifier) self.assertTrue(success) success = False def verify_points(): replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id,5) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(15,20) if not isinstance(comp,bool): return comp.all() return False success = poll(verify_points) self.assertTrue(success) def test_replay_with_parameters(self): #-------------------------------------------------------------------------------- # Create the configurations and the dataset #-------------------------------------------------------------------------------- # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream('replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id) #-------------------------------------------------------------------------------- # Coerce the datastore into existence (beats race condition) #-------------------------------------------------------------------------------- self.get_datastore(dataset_id) self.launch_producer(stream_id) self.wait_until_we_have_enough_granules(dataset_id,4) query = { 'start_time': 0, 'end_time': 20, 'stride_time' : 2, 'parameters': ['time','temp'] } retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id,query=query) rdt = RecordDictionaryTool.load_from_granule(retrieved_data) comp = np.arange(0,20,2) == rdt['time'] self.assertTrue(comp.all(),'%s' % rdt.pretty_print()) self.assertEquals(set(rdt.iterkeys()), set(['time','temp'])) extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=['time','temp']) self.assertTrue(extents['time']>=20) self.assertTrue(extents['temp']>=20) def test_repersist_data(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) stream_def_id = self.pubsub_management.create_stream_definition(name='ctd', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream(name='repersist', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id) self.get_datastore(dataset_id) self.publish_hifi(stream_id,route,0) self.publish_hifi(stream_id,route,1) self.wait_until_we_have_enough_granules(dataset_id,2) self.ingestion_management.unpersist_data_stream(stream_id=stream_id,ingestion_configuration_id=config_id) self.ingestion_management.persist_data_stream(stream_id=stream_id,ingestion_configuration_id=config_id,dataset_id=dataset_id) self.publish_hifi(stream_id,route,2) self.publish_hifi(stream_id,route,3) self.wait_until_we_have_enough_granules(dataset_id,4) success = False with gevent.timeout.Timeout(5): while not success: replay_granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(0,40) if not isinstance(comp,bool): success = comp.all() gevent.sleep(1) self.assertTrue(success)
def pdict_from_stream_def(stream_def_id): pubsub_cli = PubsubManagementServiceClient() stream_def_obj = pubsub_cli.read_stream_definition(stream_def_id) return stream_def_obj.parameter_dictionary
class PubsubManagementIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.pubsub_management = PubsubManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() self.pdicts = {} self.queue_cleanup = list() self.exchange_cleanup = list() def tearDown(self): for queue in self.queue_cleanup: xn = self.container.ex_manager.create_xn_queue(queue) xn.delete() for exchange in self.exchange_cleanup: xp = self.container.ex_manager.create_xp(exchange) xp.delete() def test_stream_def_crud(self): # Test Creation pdict = DatasetManagementService.get_parameter_dictionary_by_name( 'ctd_parsed_param_dict') stream_definition_id = self.pubsub_management.create_stream_definition( 'ctd parsed', parameter_dictionary_id=pdict.identifier) # Make sure there is an assoc self.assertTrue( self.resource_registry.find_associations( subject=stream_definition_id, predicate=PRED.hasParameterDictionary, object=pdict.identifier, id_only=True)) # Test Reading stream_definition = self.pubsub_management.read_stream_definition( stream_definition_id) self.assertTrue( PubsubManagementService._compare_pdicts( pdict.dump(), stream_definition.parameter_dictionary)) # Test Deleting self.pubsub_management.delete_stream_definition(stream_definition_id) self.assertFalse( self.resource_registry.find_associations( subject=stream_definition_id, predicate=PRED.hasParameterDictionary, object=pdict.identifier, id_only=True)) # Test comparisons in_stream_definition_id = self.pubsub_management.create_stream_definition( 'L0 products', parameter_dictionary_id=pdict.identifier, available_fields=['time', 'temp', 'conductivity', 'pressure']) self.addCleanup(self.pubsub_management.delete_stream_definition, in_stream_definition_id) out_stream_definition_id = in_stream_definition_id self.assertTrue( self.pubsub_management.compare_stream_definition( in_stream_definition_id, out_stream_definition_id)) self.assertTrue( self.pubsub_management.compatible_stream_definitions( in_stream_definition_id, out_stream_definition_id)) out_stream_definition_id = self.pubsub_management.create_stream_definition( 'L2 Products', parameter_dictionary_id=pdict.identifier, available_fields=['time', 'salinity', 'density']) self.addCleanup(self.pubsub_management.delete_stream_definition, out_stream_definition_id) self.assertFalse( self.pubsub_management.compare_stream_definition( in_stream_definition_id, out_stream_definition_id)) self.assertTrue( self.pubsub_management.compatible_stream_definitions( in_stream_definition_id, out_stream_definition_id)) def test_validate_stream_defs(self): #test no input incoming_pdict_id = self._get_pdict( ['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0']) outgoing_pdict_id = self._get_pdict( ['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) available_fields_in = [] available_fields_out = [] incoming_stream_def_id = self.pubsub_management.create_stream_definition( 'in_sd_0', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( 'out_sd_0', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) result = self.pubsub_management.validate_stream_defs( incoming_stream_def_id, outgoing_stream_def_id) self.assertFalse(result) #test input with no output incoming_pdict_id = self._get_pdict( ['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0']) outgoing_pdict_id = self._get_pdict( ['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) available_fields_in = [ 'TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0' ] available_fields_out = [] incoming_stream_def_id = self.pubsub_management.create_stream_definition( 'in_sd_1', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( 'out_sd_1', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) result = self.pubsub_management.validate_stream_defs( incoming_stream_def_id, outgoing_stream_def_id) self.assertTrue(result) #test available field missing parameter context definition -- missing PRESWAT_L0 incoming_pdict_id = self._get_pdict( ['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0']) outgoing_pdict_id = self._get_pdict( ['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) available_fields_in = [ 'TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0' ] available_fields_out = ['DENSITY'] incoming_stream_def_id = self.pubsub_management.create_stream_definition( 'in_sd_2', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( 'out_sd_2', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) result = self.pubsub_management.validate_stream_defs( incoming_stream_def_id, outgoing_stream_def_id) self.assertFalse(result) #test l1 from l0 incoming_pdict_id = self._get_pdict( ['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0']) outgoing_pdict_id = self._get_pdict( ['TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) available_fields_in = [ 'TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0' ] available_fields_out = ['TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1'] incoming_stream_def_id = self.pubsub_management.create_stream_definition( 'in_sd_3', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( 'out_sd_3', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) result = self.pubsub_management.validate_stream_defs( incoming_stream_def_id, outgoing_stream_def_id) self.assertTrue(result) #test l2 from l0 incoming_pdict_id = self._get_pdict( ['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0']) outgoing_pdict_id = self._get_pdict( ['TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1', 'DENSITY', 'PRACSAL']) available_fields_in = [ 'TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0' ] available_fields_out = ['DENSITY', 'PRACSAL'] incoming_stream_def_id = self.pubsub_management.create_stream_definition( 'in_sd_4', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( 'out_sd_4', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) result = self.pubsub_management.validate_stream_defs( incoming_stream_def_id, outgoing_stream_def_id) self.assertTrue(result) #test Ln from L0 incoming_pdict_id = self._get_pdict( ['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0']) outgoing_pdict_id = self._get_pdict( ['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) available_fields_in = [ 'TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0' ] available_fields_out = [ 'DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1' ] incoming_stream_def_id = self.pubsub_management.create_stream_definition( 'in_sd_5', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( 'out_sd_5', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) result = self.pubsub_management.validate_stream_defs( incoming_stream_def_id, outgoing_stream_def_id) self.assertTrue(result) #test L2 from L1 incoming_pdict_id = self._get_pdict( ['TIME', 'LAT', 'LON', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) outgoing_pdict_id = self._get_pdict( ['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) available_fields_in = [ 'TIME', 'LAT', 'LON', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1' ] available_fields_out = ['DENSITY', 'PRACSAL'] incoming_stream_def_id = self.pubsub_management.create_stream_definition( 'in_sd_6', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( 'out_sd_6', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) result = self.pubsub_management.validate_stream_defs( incoming_stream_def_id, outgoing_stream_def_id) self.assertTrue(result) #test L1 from L0 missing L0 incoming_pdict_id = self._get_pdict(['TIME', 'LAT', 'LON']) outgoing_pdict_id = self._get_pdict( ['TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) available_fields_in = ['TIME', 'LAT', 'LON'] available_fields_out = ['DENSITY', 'PRACSAL'] incoming_stream_def_id = self.pubsub_management.create_stream_definition( 'in_sd_7', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( 'out_sd_7', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) result = self.pubsub_management.validate_stream_defs( incoming_stream_def_id, outgoing_stream_def_id) self.assertFalse(result) #test L2 from L0 missing L0 incoming_pdict_id = self._get_pdict(['TIME', 'LAT', 'LON']) outgoing_pdict_id = self._get_pdict( ['DENSITY', 'PRACSAL', 'TEMPWAT_L1', 'CONDWAT_L1', 'PRESWAT_L1']) available_fields_in = ['TIME', 'LAT', 'LON'] available_fields_out = ['DENSITY', 'PRACSAL'] incoming_stream_def_id = self.pubsub_management.create_stream_definition( 'in_sd_8', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( 'out_sd_8', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) result = self.pubsub_management.validate_stream_defs( incoming_stream_def_id, outgoing_stream_def_id) self.assertFalse(result) #test L2 from L0 missing L1 incoming_pdict_id = self._get_pdict( ['TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0']) outgoing_pdict_id = self._get_pdict(['DENSITY', 'PRACSAL']) available_fields_in = [ 'TIME', 'LAT', 'LON', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0' ] available_fields_out = ['DENSITY', 'PRACSAL'] incoming_stream_def_id = self.pubsub_management.create_stream_definition( 'in_sd_9', parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( 'out_sd_9', parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out) result = self.pubsub_management.validate_stream_defs( incoming_stream_def_id, outgoing_stream_def_id) self.assertFalse(result) def publish_on_stream(self, stream_id, msg): stream = self.pubsub_management.read_stream(stream_id) stream_route = stream.stream_route publisher = StandaloneStreamPublisher(stream_id=stream_id, stream_route=stream_route) publisher.publish(msg) def test_stream_crud(self): stream_def_id = self.pubsub_management.create_stream_definition( 'test_definition', stream_type='stream') topic_id = self.pubsub_management.create_topic( name='test_topic', exchange_point='test_exchange') self.exchange_cleanup.append('test_exchange') topic2_id = self.pubsub_management.create_topic( name='another_topic', exchange_point='outside') stream_id, route = self.pubsub_management.create_stream( name='test_stream', topic_ids=[topic_id, topic2_id], exchange_point='test_exchange', stream_definition_id=stream_def_id) topics, assocs = self.resource_registry.find_objects( subject=stream_id, predicate=PRED.hasTopic, id_only=True) self.assertEquals(topics, [topic_id]) defs, assocs = self.resource_registry.find_objects( subject=stream_id, predicate=PRED.hasStreamDefinition, id_only=True) self.assertTrue(len(defs)) stream = self.pubsub_management.read_stream(stream_id) self.assertEquals(stream.name, 'test_stream') self.pubsub_management.delete_stream(stream_id) with self.assertRaises(NotFound): self.pubsub_management.read_stream(stream_id) defs, assocs = self.resource_registry.find_objects( subject=stream_id, predicate=PRED.hasStreamDefinition, id_only=True) self.assertFalse(len(defs)) topics, assocs = self.resource_registry.find_objects( subject=stream_id, predicate=PRED.hasTopic, id_only=True) self.assertFalse(len(topics)) self.pubsub_management.delete_topic(topic_id) self.pubsub_management.delete_topic(topic2_id) self.pubsub_management.delete_stream_definition(stream_def_id) def test_subscription_crud(self): stream_def_id = self.pubsub_management.create_stream_definition( 'test_definition', stream_type='stream') stream_id, route = self.pubsub_management.create_stream( name='test_stream', exchange_point='test_exchange', stream_definition_id=stream_def_id) subscription_id = self.pubsub_management.create_subscription( name='test subscription', stream_ids=[stream_id], exchange_name='test_queue') self.exchange_cleanup.append('test_exchange') subs, assocs = self.resource_registry.find_objects( subject=subscription_id, predicate=PRED.hasStream, id_only=True) self.assertEquals(subs, [stream_id]) res, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='test_queue', id_only=True) self.assertEquals(len(res), 1) subs, assocs = self.resource_registry.find_subjects( object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertEquals(subs[0], res[0]) subscription = self.pubsub_management.read_subscription( subscription_id) self.assertEquals(subscription.exchange_name, 'test_queue') self.pubsub_management.delete_subscription(subscription_id) subs, assocs = self.resource_registry.find_objects( subject=subscription_id, predicate=PRED.hasStream, id_only=True) self.assertFalse(len(subs)) subs, assocs = self.resource_registry.find_subjects( object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertFalse(len(subs)) self.pubsub_management.delete_stream(stream_id) self.pubsub_management.delete_stream_definition(stream_def_id) def test_move_before_activate(self): stream_id, route = self.pubsub_management.create_stream( name='test_stream', exchange_point='test_xp') #-------------------------------------------------------------------------------- # Test moving before activate #-------------------------------------------------------------------------------- subscription_id = self.pubsub_management.create_subscription( 'first_queue', stream_ids=[stream_id]) xn_ids, _ = self.resource_registry.find_resources( restype=RT.ExchangeName, name='first_queue', id_only=True) subjects, _ = self.resource_registry.find_subjects( object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertEquals(xn_ids[0], subjects[0]) self.pubsub_management.move_subscription(subscription_id, exchange_name='second_queue') xn_ids, _ = self.resource_registry.find_resources( restype=RT.ExchangeName, name='second_queue', id_only=True) subjects, _ = self.resource_registry.find_subjects( object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertEquals(len(subjects), 1) self.assertEquals(subjects[0], xn_ids[0]) self.pubsub_management.delete_subscription(subscription_id) self.pubsub_management.delete_stream(stream_id) def test_move_activated_subscription(self): stream_id, route = self.pubsub_management.create_stream( name='test_stream', exchange_point='test_xp') #-------------------------------------------------------------------------------- # Test moving after activate #-------------------------------------------------------------------------------- subscription_id = self.pubsub_management.create_subscription( 'first_queue', stream_ids=[stream_id]) self.pubsub_management.activate_subscription(subscription_id) xn_ids, _ = self.resource_registry.find_resources( restype=RT.ExchangeName, name='first_queue', id_only=True) subjects, _ = self.resource_registry.find_subjects( object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertEquals(xn_ids[0], subjects[0]) self.verified = Event() def verify(m, r, s): self.assertEquals(m, 'verified') self.verified.set() subscriber = StandaloneStreamSubscriber('second_queue', verify) subscriber.start() self.pubsub_management.move_subscription(subscription_id, exchange_name='second_queue') xn_ids, _ = self.resource_registry.find_resources( restype=RT.ExchangeName, name='second_queue', id_only=True) subjects, _ = self.resource_registry.find_subjects( object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertEquals(len(subjects), 1) self.assertEquals(subjects[0], xn_ids[0]) publisher = StandaloneStreamPublisher(stream_id, route) publisher.publish('verified') self.assertTrue(self.verified.wait(2)) self.pubsub_management.deactivate_subscription(subscription_id) self.pubsub_management.delete_subscription(subscription_id) self.pubsub_management.delete_stream(stream_id) def test_queue_cleanup(self): stream_id, route = self.pubsub_management.create_stream( 'test_stream', 'xp1') xn_objs, _ = self.resource_registry.find_resources( restype=RT.ExchangeName, name='queue1') for xn_obj in xn_objs: xn = self.container.ex_manager.create_xn_queue(xn_obj.name) xn.delete() subscription_id = self.pubsub_management.create_subscription( 'queue1', stream_ids=[stream_id]) xn_ids, _ = self.resource_registry.find_resources( restype=RT.ExchangeName, name='queue1') self.assertEquals(len(xn_ids), 1) self.pubsub_management.delete_subscription(subscription_id) xn_ids, _ = self.resource_registry.find_resources( restype=RT.ExchangeName, name='queue1') self.assertEquals(len(xn_ids), 0) def test_activation_and_deactivation(self): stream_id, route = self.pubsub_management.create_stream( 'stream1', 'xp1') subscription_id = self.pubsub_management.create_subscription( 'sub1', stream_ids=[stream_id]) self.check1 = Event() def verifier(m, r, s): self.check1.set() subscriber = StandaloneStreamSubscriber('sub1', verifier) subscriber.start() publisher = StandaloneStreamPublisher(stream_id, route) publisher.publish('should not receive') self.assertFalse(self.check1.wait(0.25)) self.pubsub_management.activate_subscription(subscription_id) publisher.publish('should receive') self.assertTrue(self.check1.wait(2)) self.check1.clear() self.assertFalse(self.check1.is_set()) self.pubsub_management.deactivate_subscription(subscription_id) publisher.publish('should not receive') self.assertFalse(self.check1.wait(0.5)) self.pubsub_management.activate_subscription(subscription_id) publisher.publish('should receive') self.assertTrue(self.check1.wait(2)) subscriber.stop() self.pubsub_management.deactivate_subscription(subscription_id) self.pubsub_management.delete_subscription(subscription_id) self.pubsub_management.delete_stream(stream_id) def test_topic_crud(self): topic_id = self.pubsub_management.create_topic( name='test_topic', exchange_point='test_xp') self.exchange_cleanup.append('test_xp') topic = self.pubsub_management.read_topic(topic_id) self.assertEquals(topic.name, 'test_topic') self.assertEquals(topic.exchange_point, 'test_xp') self.pubsub_management.delete_topic(topic_id) with self.assertRaises(NotFound): self.pubsub_management.read_topic(topic_id) def test_full_pubsub(self): self.sub1_sat = Event() self.sub2_sat = Event() def subscriber1(m, r, s): self.sub1_sat.set() def subscriber2(m, r, s): self.sub2_sat.set() sub1 = StandaloneStreamSubscriber('sub1', subscriber1) self.queue_cleanup.append(sub1.xn.queue) sub1.start() sub2 = StandaloneStreamSubscriber('sub2', subscriber2) self.queue_cleanup.append(sub2.xn.queue) sub2.start() log_topic = self.pubsub_management.create_topic( 'instrument_logs', exchange_point='instruments') science_topic = self.pubsub_management.create_topic( 'science_data', exchange_point='instruments') events_topic = self.pubsub_management.create_topic( 'notifications', exchange_point='events') log_stream, route = self.pubsub_management.create_stream( 'instrument1-logs', topic_ids=[log_topic], exchange_point='instruments') ctd_stream, route = self.pubsub_management.create_stream( 'instrument1-ctd', topic_ids=[science_topic], exchange_point='instruments') event_stream, route = self.pubsub_management.create_stream( 'notifications', topic_ids=[events_topic], exchange_point='events') raw_stream, route = self.pubsub_management.create_stream( 'temp', exchange_point='global.data') self.exchange_cleanup.extend(['instruments', 'events', 'global.data']) subscription1 = self.pubsub_management.create_subscription( 'subscription1', stream_ids=[log_stream, event_stream], exchange_name='sub1') subscription2 = self.pubsub_management.create_subscription( 'subscription2', exchange_points=['global.data'], stream_ids=[ctd_stream], exchange_name='sub2') self.pubsub_management.activate_subscription(subscription1) self.pubsub_management.activate_subscription(subscription2) self.publish_on_stream(log_stream, 1) self.assertTrue(self.sub1_sat.wait(4)) self.assertFalse(self.sub2_sat.is_set()) self.publish_on_stream(raw_stream, 1) self.assertTrue(self.sub1_sat.wait(4)) sub1.stop() sub2.stop() def test_topic_craziness(self): self.msg_queue = Queue() def subscriber1(m, r, s): self.msg_queue.put(m) sub1 = StandaloneStreamSubscriber('sub1', subscriber1) self.queue_cleanup.append(sub1.xn.queue) sub1.start() topic1 = self.pubsub_management.create_topic('topic1', exchange_point='xp1') topic2 = self.pubsub_management.create_topic('topic2', exchange_point='xp1', parent_topic_id=topic1) topic3 = self.pubsub_management.create_topic('topic3', exchange_point='xp1', parent_topic_id=topic1) topic4 = self.pubsub_management.create_topic('topic4', exchange_point='xp1', parent_topic_id=topic2) topic5 = self.pubsub_management.create_topic('topic5', exchange_point='xp1', parent_topic_id=topic2) topic6 = self.pubsub_management.create_topic('topic6', exchange_point='xp1', parent_topic_id=topic3) topic7 = self.pubsub_management.create_topic('topic7', exchange_point='xp1', parent_topic_id=topic3) # Tree 2 topic8 = self.pubsub_management.create_topic('topic8', exchange_point='xp2') topic9 = self.pubsub_management.create_topic('topic9', exchange_point='xp2', parent_topic_id=topic8) topic10 = self.pubsub_management.create_topic('topic10', exchange_point='xp2', parent_topic_id=topic9) topic11 = self.pubsub_management.create_topic('topic11', exchange_point='xp2', parent_topic_id=topic9) topic12 = self.pubsub_management.create_topic('topic12', exchange_point='xp2', parent_topic_id=topic11) topic13 = self.pubsub_management.create_topic('topic13', exchange_point='xp2', parent_topic_id=topic11) self.exchange_cleanup.extend(['xp1', 'xp2']) stream1_id, route = self.pubsub_management.create_stream( 'stream1', topic_ids=[topic7, topic4, topic5], exchange_point='xp1') stream2_id, route = self.pubsub_management.create_stream( 'stream2', topic_ids=[topic8], exchange_point='xp2') stream3_id, route = self.pubsub_management.create_stream( 'stream3', topic_ids=[topic10, topic13], exchange_point='xp2') stream4_id, route = self.pubsub_management.create_stream( 'stream4', topic_ids=[topic9], exchange_point='xp2') stream5_id, route = self.pubsub_management.create_stream( 'stream5', topic_ids=[topic11], exchange_point='xp2') subscription1 = self.pubsub_management.create_subscription( 'sub1', topic_ids=[topic1]) subscription2 = self.pubsub_management.create_subscription( 'sub2', topic_ids=[topic8], exchange_name='sub1') subscription3 = self.pubsub_management.create_subscription( 'sub3', topic_ids=[topic9], exchange_name='sub1') subscription4 = self.pubsub_management.create_subscription( 'sub4', topic_ids=[topic10, topic13, topic11], exchange_name='sub1') #-------------------------------------------------------------------------------- self.pubsub_management.activate_subscription(subscription1) self.publish_on_stream(stream1_id, 1) self.assertEquals(self.msg_queue.get(timeout=10), 1) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.1) self.pubsub_management.deactivate_subscription(subscription1) self.pubsub_management.delete_subscription(subscription1) #-------------------------------------------------------------------------------- self.pubsub_management.activate_subscription(subscription2) self.publish_on_stream(stream2_id, 2) self.assertEquals(self.msg_queue.get(timeout=10), 2) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.1) self.pubsub_management.deactivate_subscription(subscription2) self.pubsub_management.delete_subscription(subscription2) #-------------------------------------------------------------------------------- self.pubsub_management.activate_subscription(subscription3) self.publish_on_stream(stream2_id, 3) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.3) self.publish_on_stream(stream3_id, 4) self.assertEquals(self.msg_queue.get(timeout=10), 4) self.pubsub_management.deactivate_subscription(subscription3) self.pubsub_management.delete_subscription(subscription3) #-------------------------------------------------------------------------------- self.pubsub_management.activate_subscription(subscription4) self.publish_on_stream(stream4_id, 5) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.3) self.publish_on_stream(stream5_id, 6) self.assertEquals(self.msg_queue.get(timeout=10), 6) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.3) self.pubsub_management.deactivate_subscription(subscription4) self.pubsub_management.delete_subscription(subscription4) #-------------------------------------------------------------------------------- sub1.stop() self.pubsub_management.delete_topic(topic13) self.pubsub_management.delete_topic(topic12) self.pubsub_management.delete_topic(topic11) self.pubsub_management.delete_topic(topic10) self.pubsub_management.delete_topic(topic9) self.pubsub_management.delete_topic(topic8) self.pubsub_management.delete_topic(topic7) self.pubsub_management.delete_topic(topic6) self.pubsub_management.delete_topic(topic5) self.pubsub_management.delete_topic(topic4) self.pubsub_management.delete_topic(topic3) self.pubsub_management.delete_topic(topic2) self.pubsub_management.delete_topic(topic1) self.pubsub_management.delete_stream(stream1_id) self.pubsub_management.delete_stream(stream2_id) self.pubsub_management.delete_stream(stream3_id) self.pubsub_management.delete_stream(stream4_id) self.pubsub_management.delete_stream(stream5_id) def _get_pdict(self, filter_values): t_ctxt = ParameterContext( 'TIME', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1900' t_ctxt_id = self.dataset_management.create_parameter_context( name='TIME', parameter_context=t_ctxt.dump(), parameter_type='quantity<int64>', unit_of_measure=t_ctxt.uom) lat_ctxt = ParameterContext( 'LAT', param_type=ConstantType( QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt_id = self.dataset_management.create_parameter_context( name='LAT', parameter_context=lat_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=lat_ctxt.uom) lon_ctxt = ParameterContext( 'LON', param_type=ConstantType( QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt_id = self.dataset_management.create_parameter_context( name='LON', parameter_context=lon_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=lon_ctxt.uom) # Independent Parameters # Temperature - values expected to be the decimal results of conversion from hex temp_ctxt = ParameterContext( 'TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) temp_ctxt.uom = 'deg_C' temp_ctxt_id = self.dataset_management.create_parameter_context( name='TEMPWAT_L0', parameter_context=temp_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=temp_ctxt.uom) # Conductivity - values expected to be the decimal results of conversion from hex cond_ctxt = ParameterContext( 'CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) cond_ctxt.uom = 'S m-1' cond_ctxt_id = self.dataset_management.create_parameter_context( name='CONDWAT_L0', parameter_context=cond_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=cond_ctxt.uom) # Pressure - values expected to be the decimal results of conversion from hex press_ctxt = ParameterContext( 'PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) press_ctxt.uom = 'dbar' press_ctxt_id = self.dataset_management.create_parameter_context( name='PRESWAT_L0', parameter_context=press_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=press_ctxt.uom) # Dependent Parameters # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10 tl1_func = '(T / 10000) - 10' tl1_pmap = {'T': 'TEMPWAT_L0'} expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T'], param_map=tl1_pmap) tempL1_ctxt = ParameterContext( 'TEMPWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) tempL1_ctxt.uom = 'deg_C' tempL1_ctxt_id = self.dataset_management.create_parameter_context( name=tempL1_ctxt.name, parameter_context=tempL1_ctxt.dump(), parameter_type='pfunc', unit_of_measure=tempL1_ctxt.uom) # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5 cl1_func = '(C / 100000) - 0.5' cl1_pmap = {'C': 'CONDWAT_L0'} expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C'], param_map=cl1_pmap) condL1_ctxt = ParameterContext( 'CONDWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) condL1_ctxt.uom = 'S m-1' condL1_ctxt_id = self.dataset_management.create_parameter_context( name=condL1_ctxt.name, parameter_context=condL1_ctxt.dump(), parameter_type='pfunc', unit_of_measure=condL1_ctxt.uom) # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721 # PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range) pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)' pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721} expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range'], param_map=pl1_pmap) presL1_ctxt = ParameterContext( 'PRESWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) presL1_ctxt.uom = 'S m-1' presL1_ctxt_id = self.dataset_management.create_parameter_context( name=presL1_ctxt.name, parameter_context=presL1_ctxt.dump(), parameter_type='pfunc', unit_of_measure=presL1_ctxt.uom) # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1) owner = 'gsw' sal_func = 'SP_from_C' sal_arglist = ['C', 't', 'p'] sal_pmap = { 'C': NumexprFunction('CONDWAT_L1*10', 'C*10', ['C'], param_map={'C': 'CONDWAT_L1'}), 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1' } sal_kwargmap = None expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist, sal_kwargmap, sal_pmap) sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL) sal_ctxt.uom = 'g kg-1' sal_ctxt_id = self.dataset_management.create_parameter_context( name=sal_ctxt.name, parameter_context=sal_ctxt.dump(), parameter_type='pfunc', unit_of_measure=sal_ctxt.uom) # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1) # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1) owner = 'gsw' abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT']) cons_temp_expr = PythonFunction( 'cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1']) dens_expr = PythonFunction( 'DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1']) dens_ctxt = ParameterContext( 'DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL) dens_ctxt.uom = 'kg m-3' dens_ctxt_id = self.dataset_management.create_parameter_context( name=dens_ctxt.name, parameter_context=dens_ctxt.dump(), parameter_type='pfunc', unit_of_measure=dens_ctxt.uom) ids = [ t_ctxt_id, lat_ctxt_id, lon_ctxt_id, temp_ctxt_id, cond_ctxt_id, press_ctxt_id, tempL1_ctxt_id, condL1_ctxt_id, presL1_ctxt_id, sal_ctxt_id, dens_ctxt_id ] contexts = [ t_ctxt, lat_ctxt, lon_ctxt, temp_ctxt, cond_ctxt, press_ctxt, tempL1_ctxt, condL1_ctxt, presL1_ctxt, sal_ctxt, dens_ctxt ] context_ids = [ ids[i] for i, ctxt in enumerate(contexts) if ctxt.name in filter_values ] pdict_name = '_'.join( [ctxt.name for ctxt in contexts if ctxt.name in filter_values]) try: self.pdicts[pdict_name] return self.pdicts[pdict_name] except KeyError: pdict_id = self.dataset_management.create_parameter_dictionary( pdict_name, parameter_context_ids=context_ids, temporal_context='time') self.pdicts[pdict_name] = pdict_id return pdict_id
class PubsubManagementIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url("res/deploy/r2deploy.yml") self.pubsub_management = PubsubManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() self.queue_cleanup = list() self.exchange_cleanup = list() def tearDown(self): for queue in self.queue_cleanup: xn = self.container.ex_manager.create_xn_queue(queue) xn.delete() for exchange in self.exchange_cleanup: xp = self.container.ex_manager.create_xp(exchange) xp.delete() def test_stream_def_crud(self): # Test Creation pdict = DatasetManagementService.get_parameter_dictionary_by_name("ctd_parsed_param_dict") stream_definition_id = self.pubsub_management.create_stream_definition( "ctd parsed", parameter_dictionary_id=pdict.identifier ) # Make sure there is an assoc self.assertTrue( self.resource_registry.find_associations( subject=stream_definition_id, predicate=PRED.hasParameterDictionary, object=pdict.identifier, id_only=True, ) ) # Test Reading stream_definition = self.pubsub_management.read_stream_definition(stream_definition_id) self.assertTrue(PubsubManagementService._compare_pdicts(pdict.dump(), stream_definition.parameter_dictionary)) # Test Deleting self.pubsub_management.delete_stream_definition(stream_definition_id) self.assertFalse( self.resource_registry.find_associations( subject=stream_definition_id, predicate=PRED.hasParameterDictionary, object=pdict.identifier, id_only=True, ) ) # Test comparisons in_stream_definition_id = self.pubsub_management.create_stream_definition( "L0 products", parameter_dictionary=pdict.identifier, available_fields=["time", "temp", "conductivity", "pressure"], ) self.addCleanup(self.pubsub_management.delete_stream_definition, in_stream_definition_id) out_stream_definition_id = in_stream_definition_id self.assertTrue( self.pubsub_management.compare_stream_definition(in_stream_definition_id, out_stream_definition_id) ) self.assertTrue( self.pubsub_management.compatible_stream_definitions(in_stream_definition_id, out_stream_definition_id) ) out_stream_definition_id = self.pubsub_management.create_stream_definition( "L2 Products", parameter_dictionary=pdict.identifier, available_fields=["time", "salinity", "density"] ) self.addCleanup(self.pubsub_management.delete_stream_definition, out_stream_definition_id) self.assertFalse( self.pubsub_management.compare_stream_definition(in_stream_definition_id, out_stream_definition_id) ) self.assertTrue( self.pubsub_management.compatible_stream_definitions(in_stream_definition_id, out_stream_definition_id) ) def test_validate_stream_defs(self): # test no input incoming_pdict_id = self._get_pdict(["time", "lat", "lon", "TEMPWAT_L0", "CONDWAT_L0", "PRESWAT_L0"]) outgoing_pdict_id = self._get_pdict(["DENSITY", "PRACSAL", "TEMPWAT_L1", "CONDWAT_L1", "PRESWAT_L1"]) available_fields_in = [] available_fields_out = [] incoming_stream_def_id = self.pubsub_management.create_stream_definition( "in_sd_0", parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in ) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( "out_sd_0", parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out ) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertFalse(result) # test input with no output incoming_pdict_id = self._get_pdict(["time", "lat", "lon", "TEMPWAT_L0", "CONDWAT_L0", "PRESWAT_L0"]) outgoing_pdict_id = self._get_pdict(["DENSITY", "PRACSAL", "TEMPWAT_L1", "CONDWAT_L1", "PRESWAT_L1"]) available_fields_in = ["time", "lat", "lon", "TEMPWAT_L0", "CONDWAT_L0", "PRESWAT_L0"] available_fields_out = [] incoming_stream_def_id = self.pubsub_management.create_stream_definition( "in_sd_1", parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in ) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( "out_sd_1", parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out ) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertTrue(result) # test available field missing parameter context definition -- missing PRESWAT_L0 incoming_pdict_id = self._get_pdict(["time", "lat", "lon", "TEMPWAT_L0", "CONDWAT_L0"]) outgoing_pdict_id = self._get_pdict(["DENSITY", "PRACSAL", "TEMPWAT_L1", "CONDWAT_L1", "PRESWAT_L1"]) available_fields_in = ["time", "lat", "lon", "TEMPWAT_L0", "CONDWAT_L0", "PRESWAT_L0"] available_fields_out = ["DENSITY"] incoming_stream_def_id = self.pubsub_management.create_stream_definition( "in_sd_2", parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in ) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( "out_sd_2", parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out ) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertFalse(result) # test l1 from l0 incoming_pdict_id = self._get_pdict(["time", "lat", "lon", "TEMPWAT_L0", "CONDWAT_L0", "PRESWAT_L0"]) outgoing_pdict_id = self._get_pdict(["TEMPWAT_L1", "CONDWAT_L1", "PRESWAT_L1"]) available_fields_in = ["time", "lat", "lon", "TEMPWAT_L0", "CONDWAT_L0", "PRESWAT_L0"] available_fields_out = ["TEMPWAT_L1", "CONDWAT_L1", "PRESWAT_L1"] incoming_stream_def_id = self.pubsub_management.create_stream_definition( "in_sd_3", parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in ) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( "out_sd_3", parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out ) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertTrue(result) # test l2 from l0 incoming_pdict_id = self._get_pdict(["time", "lat", "lon", "TEMPWAT_L0", "CONDWAT_L0", "PRESWAT_L0"]) outgoing_pdict_id = self._get_pdict(["TEMPWAT_L1", "CONDWAT_L1", "PRESWAT_L1", "DENSITY", "PRACSAL"]) available_fields_in = ["time", "lat", "lon", "TEMPWAT_L0", "CONDWAT_L0", "PRESWAT_L0"] available_fields_out = ["DENSITY", "PRACSAL"] incoming_stream_def_id = self.pubsub_management.create_stream_definition( "in_sd_4", parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in ) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( "out_sd_4", parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out ) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertTrue(result) # test Ln from L0 incoming_pdict_id = self._get_pdict(["time", "lat", "lon", "TEMPWAT_L0", "CONDWAT_L0", "PRESWAT_L0"]) outgoing_pdict_id = self._get_pdict(["DENSITY", "PRACSAL", "TEMPWAT_L1", "CONDWAT_L1", "PRESWAT_L1"]) available_fields_in = ["time", "lat", "lon", "TEMPWAT_L0", "CONDWAT_L0", "PRESWAT_L0"] available_fields_out = ["DENSITY", "PRACSAL", "TEMPWAT_L1", "CONDWAT_L1", "PRESWAT_L1"] incoming_stream_def_id = self.pubsub_management.create_stream_definition( "in_sd_5", parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in ) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( "out_sd_5", parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out ) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertTrue(result) # test L2 from L1 incoming_pdict_id = self._get_pdict(["time", "lat", "lon", "TEMPWAT_L1", "CONDWAT_L1", "PRESWAT_L1"]) outgoing_pdict_id = self._get_pdict(["DENSITY", "PRACSAL", "TEMPWAT_L1", "CONDWAT_L1", "PRESWAT_L1"]) available_fields_in = ["time", "lat", "lon", "TEMPWAT_L1", "CONDWAT_L1", "PRESWAT_L1"] available_fields_out = ["DENSITY", "PRACSAL"] incoming_stream_def_id = self.pubsub_management.create_stream_definition( "in_sd_6", parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in ) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( "out_sd_6", parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out ) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertTrue(result) # test L1 from L0 missing L0 incoming_pdict_id = self._get_pdict(["time", "lat", "lon"]) outgoing_pdict_id = self._get_pdict(["TEMPWAT_L1", "CONDWAT_L1", "PRESWAT_L1"]) available_fields_in = ["time", "lat", "lon"] available_fields_out = ["DENSITY", "PRACSAL"] incoming_stream_def_id = self.pubsub_management.create_stream_definition( "in_sd_7", parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in ) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( "out_sd_7", parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out ) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertFalse(result) # test L2 from L0 missing L0 incoming_pdict_id = self._get_pdict(["time", "lat", "lon"]) outgoing_pdict_id = self._get_pdict(["DENSITY", "PRACSAL", "TEMPWAT_L1", "CONDWAT_L1", "PRESWAT_L1"]) available_fields_in = ["time", "lat", "lon"] available_fields_out = ["DENSITY", "PRACSAL"] incoming_stream_def_id = self.pubsub_management.create_stream_definition( "in_sd_8", parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in ) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( "out_sd_8", parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out ) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertFalse(result) # test L2 from L0 missing L1 incoming_pdict_id = self._get_pdict(["time", "lat", "lon", "TEMPWAT_L0", "CONDWAT_L0", "PRESWAT_L0"]) outgoing_pdict_id = self._get_pdict(["DENSITY", "PRACSAL"]) available_fields_in = ["time", "lat", "lon", "TEMPWAT_L0", "CONDWAT_L0", "PRESWAT_L0"] available_fields_out = ["DENSITY", "PRACSAL"] incoming_stream_def_id = self.pubsub_management.create_stream_definition( "in_sd_9", parameter_dictionary_id=incoming_pdict_id, available_fields=available_fields_in ) outgoing_stream_def_id = self.pubsub_management.create_stream_definition( "out_sd_9", parameter_dictionary_id=outgoing_pdict_id, available_fields=available_fields_out ) result = self.pubsub_management.validate_stream_defs(incoming_stream_def_id, outgoing_stream_def_id) self.assertFalse(result) def publish_on_stream(self, stream_id, msg): stream = self.pubsub_management.read_stream(stream_id) stream_route = stream.stream_route publisher = StandaloneStreamPublisher(stream_id=stream_id, stream_route=stream_route) publisher.publish(msg) def test_stream_crud(self): stream_def_id = self.pubsub_management.create_stream_definition("test_definition", stream_type="stream") topic_id = self.pubsub_management.create_topic(name="test_topic", exchange_point="test_exchange") self.exchange_cleanup.append("test_exchange") topic2_id = self.pubsub_management.create_topic(name="another_topic", exchange_point="outside") stream_id, route = self.pubsub_management.create_stream( name="test_stream", topic_ids=[topic_id, topic2_id], exchange_point="test_exchange", stream_definition_id=stream_def_id, ) topics, assocs = self.resource_registry.find_objects(subject=stream_id, predicate=PRED.hasTopic, id_only=True) self.assertEquals(topics, [topic_id]) defs, assocs = self.resource_registry.find_objects( subject=stream_id, predicate=PRED.hasStreamDefinition, id_only=True ) self.assertTrue(len(defs)) stream = self.pubsub_management.read_stream(stream_id) self.assertEquals(stream.name, "test_stream") self.pubsub_management.delete_stream(stream_id) with self.assertRaises(NotFound): self.pubsub_management.read_stream(stream_id) defs, assocs = self.resource_registry.find_objects( subject=stream_id, predicate=PRED.hasStreamDefinition, id_only=True ) self.assertFalse(len(defs)) topics, assocs = self.resource_registry.find_objects(subject=stream_id, predicate=PRED.hasTopic, id_only=True) self.assertFalse(len(topics)) self.pubsub_management.delete_topic(topic_id) self.pubsub_management.delete_topic(topic2_id) self.pubsub_management.delete_stream_definition(stream_def_id) def test_subscription_crud(self): stream_def_id = self.pubsub_management.create_stream_definition("test_definition", stream_type="stream") stream_id, route = self.pubsub_management.create_stream( name="test_stream", exchange_point="test_exchange", stream_definition_id=stream_def_id ) subscription_id = self.pubsub_management.create_subscription( name="test subscription", stream_ids=[stream_id], exchange_name="test_queue" ) self.exchange_cleanup.append("test_exchange") subs, assocs = self.resource_registry.find_objects( subject=subscription_id, predicate=PRED.hasStream, id_only=True ) self.assertEquals(subs, [stream_id]) res, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name="test_queue", id_only=True) self.assertEquals(len(res), 1) subs, assocs = self.resource_registry.find_subjects( object=subscription_id, predicate=PRED.hasSubscription, id_only=True ) self.assertEquals(subs[0], res[0]) subscription = self.pubsub_management.read_subscription(subscription_id) self.assertEquals(subscription.exchange_name, "test_queue") self.pubsub_management.delete_subscription(subscription_id) subs, assocs = self.resource_registry.find_objects( subject=subscription_id, predicate=PRED.hasStream, id_only=True ) self.assertFalse(len(subs)) subs, assocs = self.resource_registry.find_subjects( object=subscription_id, predicate=PRED.hasSubscription, id_only=True ) self.assertFalse(len(subs)) self.pubsub_management.delete_stream(stream_id) self.pubsub_management.delete_stream_definition(stream_def_id) def test_move_before_activate(self): stream_id, route = self.pubsub_management.create_stream(name="test_stream", exchange_point="test_xp") # -------------------------------------------------------------------------------- # Test moving before activate # -------------------------------------------------------------------------------- subscription_id = self.pubsub_management.create_subscription("first_queue", stream_ids=[stream_id]) xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name="first_queue", id_only=True) subjects, _ = self.resource_registry.find_subjects( object=subscription_id, predicate=PRED.hasSubscription, id_only=True ) self.assertEquals(xn_ids[0], subjects[0]) self.pubsub_management.move_subscription(subscription_id, exchange_name="second_queue") xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name="second_queue", id_only=True) subjects, _ = self.resource_registry.find_subjects( object=subscription_id, predicate=PRED.hasSubscription, id_only=True ) self.assertEquals(len(subjects), 1) self.assertEquals(subjects[0], xn_ids[0]) self.pubsub_management.delete_subscription(subscription_id) self.pubsub_management.delete_stream(stream_id) def test_move_activated_subscription(self): stream_id, route = self.pubsub_management.create_stream(name="test_stream", exchange_point="test_xp") # -------------------------------------------------------------------------------- # Test moving after activate # -------------------------------------------------------------------------------- subscription_id = self.pubsub_management.create_subscription("first_queue", stream_ids=[stream_id]) self.pubsub_management.activate_subscription(subscription_id) xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name="first_queue", id_only=True) subjects, _ = self.resource_registry.find_subjects( object=subscription_id, predicate=PRED.hasSubscription, id_only=True ) self.assertEquals(xn_ids[0], subjects[0]) self.verified = Event() def verify(m, r, s): self.assertEquals(m, "verified") self.verified.set() subscriber = StandaloneStreamSubscriber("second_queue", verify) subscriber.start() self.pubsub_management.move_subscription(subscription_id, exchange_name="second_queue") xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name="second_queue", id_only=True) subjects, _ = self.resource_registry.find_subjects( object=subscription_id, predicate=PRED.hasSubscription, id_only=True ) self.assertEquals(len(subjects), 1) self.assertEquals(subjects[0], xn_ids[0]) publisher = StandaloneStreamPublisher(stream_id, route) publisher.publish("verified") self.assertTrue(self.verified.wait(2)) self.pubsub_management.deactivate_subscription(subscription_id) self.pubsub_management.delete_subscription(subscription_id) self.pubsub_management.delete_stream(stream_id) def test_queue_cleanup(self): stream_id, route = self.pubsub_management.create_stream("test_stream", "xp1") xn_objs, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name="queue1") for xn_obj in xn_objs: xn = self.container.ex_manager.create_xn_queue(xn_obj.name) xn.delete() subscription_id = self.pubsub_management.create_subscription("queue1", stream_ids=[stream_id]) xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name="queue1") self.assertEquals(len(xn_ids), 1) self.pubsub_management.delete_subscription(subscription_id) xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name="queue1") self.assertEquals(len(xn_ids), 0) def test_activation_and_deactivation(self): stream_id, route = self.pubsub_management.create_stream("stream1", "xp1") subscription_id = self.pubsub_management.create_subscription("sub1", stream_ids=[stream_id]) self.check1 = Event() def verifier(m, r, s): self.check1.set() subscriber = StandaloneStreamSubscriber("sub1", verifier) subscriber.start() publisher = StandaloneStreamPublisher(stream_id, route) publisher.publish("should not receive") self.assertFalse(self.check1.wait(0.25)) self.pubsub_management.activate_subscription(subscription_id) publisher.publish("should receive") self.assertTrue(self.check1.wait(2)) self.check1.clear() self.assertFalse(self.check1.is_set()) self.pubsub_management.deactivate_subscription(subscription_id) publisher.publish("should not receive") self.assertFalse(self.check1.wait(0.5)) self.pubsub_management.activate_subscription(subscription_id) publisher.publish("should receive") self.assertTrue(self.check1.wait(2)) subscriber.stop() self.pubsub_management.deactivate_subscription(subscription_id) self.pubsub_management.delete_subscription(subscription_id) self.pubsub_management.delete_stream(stream_id) def test_topic_crud(self): topic_id = self.pubsub_management.create_topic(name="test_topic", exchange_point="test_xp") self.exchange_cleanup.append("test_xp") topic = self.pubsub_management.read_topic(topic_id) self.assertEquals(topic.name, "test_topic") self.assertEquals(topic.exchange_point, "test_xp") self.pubsub_management.delete_topic(topic_id) with self.assertRaises(NotFound): self.pubsub_management.read_topic(topic_id) def test_full_pubsub(self): self.sub1_sat = Event() self.sub2_sat = Event() def subscriber1(m, r, s): self.sub1_sat.set() def subscriber2(m, r, s): self.sub2_sat.set() sub1 = StandaloneStreamSubscriber("sub1", subscriber1) self.queue_cleanup.append(sub1.xn.queue) sub1.start() sub2 = StandaloneStreamSubscriber("sub2", subscriber2) self.queue_cleanup.append(sub2.xn.queue) sub2.start() log_topic = self.pubsub_management.create_topic("instrument_logs", exchange_point="instruments") science_topic = self.pubsub_management.create_topic("science_data", exchange_point="instruments") events_topic = self.pubsub_management.create_topic("notifications", exchange_point="events") log_stream, route = self.pubsub_management.create_stream( "instrument1-logs", topic_ids=[log_topic], exchange_point="instruments" ) ctd_stream, route = self.pubsub_management.create_stream( "instrument1-ctd", topic_ids=[science_topic], exchange_point="instruments" ) event_stream, route = self.pubsub_management.create_stream( "notifications", topic_ids=[events_topic], exchange_point="events" ) raw_stream, route = self.pubsub_management.create_stream("temp", exchange_point="global.data") self.exchange_cleanup.extend(["instruments", "events", "global.data"]) subscription1 = self.pubsub_management.create_subscription( "subscription1", stream_ids=[log_stream, event_stream], exchange_name="sub1" ) subscription2 = self.pubsub_management.create_subscription( "subscription2", exchange_points=["global.data"], stream_ids=[ctd_stream], exchange_name="sub2" ) self.pubsub_management.activate_subscription(subscription1) self.pubsub_management.activate_subscription(subscription2) self.publish_on_stream(log_stream, 1) self.assertTrue(self.sub1_sat.wait(4)) self.assertFalse(self.sub2_sat.is_set()) self.publish_on_stream(raw_stream, 1) self.assertTrue(self.sub1_sat.wait(4)) sub1.stop() sub2.stop() def test_topic_craziness(self): self.msg_queue = Queue() def subscriber1(m, r, s): self.msg_queue.put(m) sub1 = StandaloneStreamSubscriber("sub1", subscriber1) self.queue_cleanup.append(sub1.xn.queue) sub1.start() topic1 = self.pubsub_management.create_topic("topic1", exchange_point="xp1") topic2 = self.pubsub_management.create_topic("topic2", exchange_point="xp1", parent_topic_id=topic1) topic3 = self.pubsub_management.create_topic("topic3", exchange_point="xp1", parent_topic_id=topic1) topic4 = self.pubsub_management.create_topic("topic4", exchange_point="xp1", parent_topic_id=topic2) topic5 = self.pubsub_management.create_topic("topic5", exchange_point="xp1", parent_topic_id=topic2) topic6 = self.pubsub_management.create_topic("topic6", exchange_point="xp1", parent_topic_id=topic3) topic7 = self.pubsub_management.create_topic("topic7", exchange_point="xp1", parent_topic_id=topic3) # Tree 2 topic8 = self.pubsub_management.create_topic("topic8", exchange_point="xp2") topic9 = self.pubsub_management.create_topic("topic9", exchange_point="xp2", parent_topic_id=topic8) topic10 = self.pubsub_management.create_topic("topic10", exchange_point="xp2", parent_topic_id=topic9) topic11 = self.pubsub_management.create_topic("topic11", exchange_point="xp2", parent_topic_id=topic9) topic12 = self.pubsub_management.create_topic("topic12", exchange_point="xp2", parent_topic_id=topic11) topic13 = self.pubsub_management.create_topic("topic13", exchange_point="xp2", parent_topic_id=topic11) self.exchange_cleanup.extend(["xp1", "xp2"]) stream1_id, route = self.pubsub_management.create_stream( "stream1", topic_ids=[topic7, topic4, topic5], exchange_point="xp1" ) stream2_id, route = self.pubsub_management.create_stream("stream2", topic_ids=[topic8], exchange_point="xp2") stream3_id, route = self.pubsub_management.create_stream( "stream3", topic_ids=[topic10, topic13], exchange_point="xp2" ) stream4_id, route = self.pubsub_management.create_stream("stream4", topic_ids=[topic9], exchange_point="xp2") stream5_id, route = self.pubsub_management.create_stream("stream5", topic_ids=[topic11], exchange_point="xp2") subscription1 = self.pubsub_management.create_subscription("sub1", topic_ids=[topic1]) subscription2 = self.pubsub_management.create_subscription("sub2", topic_ids=[topic8], exchange_name="sub1") subscription3 = self.pubsub_management.create_subscription("sub3", topic_ids=[topic9], exchange_name="sub1") subscription4 = self.pubsub_management.create_subscription( "sub4", topic_ids=[topic10, topic13, topic11], exchange_name="sub1" ) # -------------------------------------------------------------------------------- self.pubsub_management.activate_subscription(subscription1) self.publish_on_stream(stream1_id, 1) self.assertEquals(self.msg_queue.get(timeout=10), 1) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.1) self.pubsub_management.deactivate_subscription(subscription1) self.pubsub_management.delete_subscription(subscription1) # -------------------------------------------------------------------------------- self.pubsub_management.activate_subscription(subscription2) self.publish_on_stream(stream2_id, 2) self.assertEquals(self.msg_queue.get(timeout=10), 2) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.1) self.pubsub_management.deactivate_subscription(subscription2) self.pubsub_management.delete_subscription(subscription2) # -------------------------------------------------------------------------------- self.pubsub_management.activate_subscription(subscription3) self.publish_on_stream(stream2_id, 3) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.3) self.publish_on_stream(stream3_id, 4) self.assertEquals(self.msg_queue.get(timeout=10), 4) self.pubsub_management.deactivate_subscription(subscription3) self.pubsub_management.delete_subscription(subscription3) # -------------------------------------------------------------------------------- self.pubsub_management.activate_subscription(subscription4) self.publish_on_stream(stream4_id, 5) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.3) self.publish_on_stream(stream5_id, 6) self.assertEquals(self.msg_queue.get(timeout=10), 6) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.3) self.pubsub_management.deactivate_subscription(subscription4) self.pubsub_management.delete_subscription(subscription4) # -------------------------------------------------------------------------------- sub1.stop() self.pubsub_management.delete_topic(topic13) self.pubsub_management.delete_topic(topic12) self.pubsub_management.delete_topic(topic11) self.pubsub_management.delete_topic(topic10) self.pubsub_management.delete_topic(topic9) self.pubsub_management.delete_topic(topic8) self.pubsub_management.delete_topic(topic7) self.pubsub_management.delete_topic(topic6) self.pubsub_management.delete_topic(topic5) self.pubsub_management.delete_topic(topic4) self.pubsub_management.delete_topic(topic3) self.pubsub_management.delete_topic(topic2) self.pubsub_management.delete_topic(topic1) self.pubsub_management.delete_stream(stream1_id) self.pubsub_management.delete_stream(stream2_id) self.pubsub_management.delete_stream(stream3_id) self.pubsub_management.delete_stream(stream4_id) self.pubsub_management.delete_stream(stream5_id) def _get_pdict(self, filter_values): t_ctxt = ParameterContext("time", param_type=QuantityType(value_encoding=np.dtype("int64"))) t_ctxt.uom = "seconds since 01-01-1900" t_ctxt.fill_value = -9999 t_ctxt_id = self.dataset_management.create_parameter_context( name="time", parameter_context=t_ctxt.dump(), parameter_type="quantity<int64>", unit_of_measure=t_ctxt.uom ) lat_ctxt = ParameterContext("lat", param_type=ConstantType(QuantityType(value_encoding=np.dtype("float32")))) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = "degree_north" lat_ctxt.fill_value = -9999 lat_ctxt_id = self.dataset_management.create_parameter_context( name="lat", parameter_context=lat_ctxt.dump(), parameter_type="quantity<float32>", unit_of_measure=lat_ctxt.uom, ) lon_ctxt = ParameterContext("lon", param_type=ConstantType(QuantityType(value_encoding=np.dtype("float32")))) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = "degree_east" lon_ctxt.fill_value = -9999 lon_ctxt_id = self.dataset_management.create_parameter_context( name="lon", parameter_context=lon_ctxt.dump(), parameter_type="quantity<float32>", unit_of_measure=lon_ctxt.uom, ) temp_ctxt = ParameterContext("TEMPWAT_L0", param_type=QuantityType(value_encoding=np.dtype("float32"))) temp_ctxt.uom = "deg_C" temp_ctxt.fill_value = -9999 temp_ctxt_id = self.dataset_management.create_parameter_context( name="TEMPWAT_L0", parameter_context=temp_ctxt.dump(), parameter_type="quantity<float32>", unit_of_measure=temp_ctxt.uom, ) # Conductivity - values expected to be the decimal results of conversion from hex cond_ctxt = ParameterContext("CONDWAT_L0", param_type=QuantityType(value_encoding=np.dtype("float32"))) cond_ctxt.uom = "S m-1" cond_ctxt.fill_value = -9999 cond_ctxt_id = self.dataset_management.create_parameter_context( name="CONDWAT_L0", parameter_context=cond_ctxt.dump(), parameter_type="quantity<float32>", unit_of_measure=cond_ctxt.uom, ) # Pressure - values expected to be the decimal results of conversion from hex press_ctxt = ParameterContext("PRESWAT_L0", param_type=QuantityType(value_encoding=np.dtype("float32"))) press_ctxt.uom = "dbar" press_ctxt.fill_value = -9999 press_ctxt_id = self.dataset_management.create_parameter_context( name="PRESWAT_L0", parameter_context=press_ctxt.dump(), parameter_type="quantity<float32>", unit_of_measure=press_ctxt.uom, ) # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10 tl1_func = "(TEMPWAT_L0 / 10000) - 10" tl1_pmap = {"TEMPWAT_L0": "TEMPWAT_L0"} func = NumexprFunction("TEMPWAT_L1", tl1_func, tl1_pmap) tempL1_ctxt = ParameterContext( "TEMPWAT_L1", param_type=ParameterFunctionType(function=func), variability=VariabilityEnum.TEMPORAL ) tempL1_ctxt.uom = "deg_C" tempL1_ctxt_id = self.dataset_management.create_parameter_context( name=tempL1_ctxt.name, parameter_context=tempL1_ctxt.dump(), parameter_type="pfunc", unit_of_measure=tempL1_ctxt.uom, ) # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5 cl1_func = "(CONDWAT_L0 / 100000) - 0.5" cl1_pmap = {"CONDWAT_L0": "CONDWAT_L0"} func = NumexprFunction("CONDWAT_L1", cl1_func, cl1_pmap) condL1_ctxt = ParameterContext( "CONDWAT_L1", param_type=ParameterFunctionType(function=func), variability=VariabilityEnum.TEMPORAL ) condL1_ctxt.uom = "S m-1" condL1_ctxt_id = self.dataset_management.create_parameter_context( name=condL1_ctxt.name, parameter_context=condL1_ctxt.dump(), parameter_type="pfunc", unit_of_measure=condL1_ctxt.uom, ) # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721 # PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range) pl1_func = "(PRESWAT_L0 * 679.34040721 / (0.85 * 65536)) - (0.05 * 679.34040721)" pl1_pmap = {"PRESWAT_L0": "PRESWAT_L0"} func = NumexprFunction("PRESWAT_L1", pl1_func, pl1_pmap) presL1_ctxt = ParameterContext( "PRESWAT_L1", param_type=ParameterFunctionType(function=func), variability=VariabilityEnum.TEMPORAL ) presL1_ctxt.uom = "S m-1" presL1_ctxt_id = self.dataset_management.create_parameter_context( name=presL1_ctxt.name, parameter_context=presL1_ctxt.dump(), parameter_type="pfunc", unit_of_measure=presL1_ctxt.uom, ) # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1) owner = "gsw" sal_func = "SP_from_C" sal_arglist = [NumexprFunction("CONDWAT_L1*10", "C*10", {"C": "CONDWAT_L1"}), "TEMPWAT_L1", "PRESWAT_L1"] sal_kwargmap = None func = PythonFunction("PRACSAL", owner, sal_func, sal_arglist, sal_kwargmap) sal_ctxt = ParameterContext( "PRACSAL", param_type=ParameterFunctionType(func), variability=VariabilityEnum.TEMPORAL ) sal_ctxt.uom = "g kg-1" sal_ctxt_id = self.dataset_management.create_parameter_context( name=sal_ctxt.name, parameter_context=sal_ctxt.dump(), parameter_type="pfunc", unit_of_measure=sal_ctxt.uom ) # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1) # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1) owner = "gsw" abs_sal_func = PythonFunction("abs_sal", owner, "SA_from_SP", ["PRACSAL", "PRESWAT_L1", "lon", "lat"], None) # abs_sal_func = PythonFunction('abs_sal', owner, 'SA_from_SP', ['lon','lat'], None) cons_temp_func = PythonFunction( "cons_temp", owner, "CT_from_t", [abs_sal_func, "TEMPWAT_L1", "PRESWAT_L1"], None ) dens_func = PythonFunction("DENSITY", owner, "rho", [abs_sal_func, cons_temp_func, "PRESWAT_L1"], None) dens_ctxt = ParameterContext( "DENSITY", param_type=ParameterFunctionType(dens_func), variability=VariabilityEnum.TEMPORAL ) dens_ctxt.uom = "kg m-3" dens_ctxt_id = self.dataset_management.create_parameter_context( name=dens_ctxt.name, parameter_context=dens_ctxt.dump(), parameter_type="pfunc", unit_of_measure=dens_ctxt.uom, ) ids = [ t_ctxt_id, lat_ctxt_id, lon_ctxt_id, temp_ctxt_id, cond_ctxt_id, press_ctxt_id, tempL1_ctxt_id, condL1_ctxt_id, presL1_ctxt_id, sal_ctxt_id, dens_ctxt_id, ] contexts = [ t_ctxt, lat_ctxt, lon_ctxt, temp_ctxt, cond_ctxt, press_ctxt, tempL1_ctxt, condL1_ctxt, presL1_ctxt, sal_ctxt, dens_ctxt, ] context_ids = [ids[i] for i, ctxt in enumerate(contexts) if ctxt.name in filter_values] pdict_name = "_".join([ctxt.name for ctxt in contexts if ctxt.name in filter_values]) pdict_id = self.dataset_management.create_parameter_dictionary( pdict_name, parameter_context_ids=context_ids, temporal_context="time" ) return pdict_id
class TransformPrototypeIntTest(IonIntegrationTestCase): def setUp(self): super(TransformPrototypeIntTest, self).setUp() self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrc = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() self.pubsub_management = PubsubManagementServiceClient() self.ssclient = SchedulerServiceClient() self.event_publisher = EventPublisher() self.exchange_names = [] self.exchange_points = [] def tearDown(self): for xn in self.exchange_names: xni = self.container.ex_manager.create_xn_queue(xn) xni.delete() for xp in self.exchange_points: xpi = self.container.ex_manager.create_xp(xp) xpi.delete() def now_utc(self): return time.mktime(datetime.datetime.utcnow().timetuple()) def _create_interval_timer_with_end_time(self,timer_interval= None, end_time = None ): ''' A convenience method to set up an interval timer with an end time ''' self.timer_received_time = 0 self.timer_interval = timer_interval start_time = self.now_utc() if not end_time: end_time = start_time + 2 * timer_interval + 1 log.debug("got the end time here!! %s" % end_time) # Set up the interval timer. The scheduler will publish event with origin set as "Interval Timer" sid = self.ssclient.create_interval_timer(start_time="now" , interval=self.timer_interval, end_time=end_time, event_origin="Interval Timer", event_subtype="") def cleanup_timer(scheduler, schedule_id): """ Do a friendly cancel of the scheduled event. If it fails, it's ok. """ try: scheduler.cancel_timer(schedule_id) except: log.warn("Couldn't cancel") self.addCleanup(cleanup_timer, self.ssclient, sid) return sid @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_event_processing(self): ''' Test that events are processed by the transforms according to a provided algorithm ''' #------------------------------------------------------------------------------------- # Set up the scheduler for an interval timer with an end time #------------------------------------------------------------------------------------- id = self._create_interval_timer_with_end_time(timer_interval=2) self.assertIsNotNone(id) #------------------------------------------------------------------------------------- # Create an event alert transform.... # The configuration for the Event Alert Transform... set up the event types to listen to #------------------------------------------------------------------------------------- configuration = { 'process':{ 'event_type': 'ResourceEvent', 'timer_origin': 'Interval Timer', 'instrument_origin': 'My_favorite_instrument' } } #------------------------------------------------------------------------------------- # Create the process #------------------------------------------------------------------------------------- pid = TransformPrototypeIntTest.create_process( name= 'event_alert_transform', module='ion.processes.data.transforms.event_alert_transform', class_name='EventAlertTransform', configuration= configuration) self.assertIsNotNone(pid) #------------------------------------------------------------------------------------- # Publish events and make assertions about alerts #------------------------------------------------------------------------------------- queue = gevent.queue.Queue() def event_received(message, headers): queue.put(message) event_subscriber = EventSubscriber( origin="EventAlertTransform", event_type="DeviceEvent", callback=event_received) event_subscriber.start() self.addCleanup(event_subscriber.stop) # publish event twice for i in xrange(5): self.event_publisher.publish_event( event_type = 'ExampleDetectableEvent', origin = "My_favorite_instrument", voltage = 5, telemetry = 10, temperature = 20) gevent.sleep(0.1) self.assertTrue(queue.empty()) #publish event the third time but after a time interval larger than 2 seconds gevent.sleep(5) #------------------------------------------------------------------------------------- # Make assertions about the alert event published by the EventAlertTransform #------------------------------------------------------------------------------------- event = queue.get(timeout=10) log.debug("Alarm event received from the EventAertTransform %s" % event) self.assertEquals(event.type_, "DeviceEvent") self.assertEquals(event.origin, "EventAlertTransform") #------------------------------------------------------------------------------------------------ # Now clear the event queue being populated by alarm events and publish normally once again #------------------------------------------------------------------------------------------------ queue.queue.clear() for i in xrange(5): self.event_publisher.publish_event( event_type = 'ExampleDetectableEvent', origin = "My_favorite_instrument", voltage = 5, telemetry = 10, temperature = 20) gevent.sleep(0.1) self.assertTrue(queue.empty()) log.debug("This completes the requirement that the EventAlertTransform publishes \ an alarm event when it does not hear from the instrument for some time.") @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_stream_processing(self): #-------------------------------------------------------------------------------- #Test that streams are processed by the transforms according to a provided algorithm #-------------------------------------------------------------------------------- #todo: In this simple implementation, we are checking if the stream has the word, PUBLISH, #todo(contd) and if the word VALUE=<number> exists and that number is less than something #todo later on we are going to use complex algorithms to make this prototype powerful #------------------------------------------------------------------------------------- # Start a subscriber to listen for an alert event from the Stream Alert Transform #------------------------------------------------------------------------------------- queue = gevent.queue.Queue() def event_received(message, headers): queue.put(message) event_subscriber = EventSubscriber( origin="StreamAlertTransform", event_type="DeviceEvent", callback=event_received) event_subscriber.start() self.addCleanup(event_subscriber.stop) #------------------------------------------------------------------------------------- # The configuration for the Stream Alert Transform... set up the event types to listen to #------------------------------------------------------------------------------------- config = { 'process':{ 'queue_name': 'a_queue', 'value': 10, 'event_type':'DeviceEvent' } } #------------------------------------------------------------------------------------- # Create the process #------------------------------------------------------------------------------------- pid = TransformPrototypeIntTest.create_process( name= 'transform_data_process', module='ion.processes.data.transforms.event_alert_transform', class_name='StreamAlertTransform', configuration= config) self.assertIsNotNone(pid) #------------------------------------------------------------------------------------- # Publish streams and make assertions about alerts #------------------------------------------------------------------------------------- exchange_name = 'a_queue' exchange_point = 'test_exchange' routing_key = 'stream_id.stream' stream_route = StreamRoute(exchange_point, routing_key) xn = self.container.ex_manager.create_xn_queue(exchange_name) xp = self.container.ex_manager.create_xp(exchange_point) xn.bind('stream_id.stream', xp) pub = StandaloneStreamPublisher('stream_id', stream_route) message = "A dummy example message containing the word PUBLISH, and with VALUE = 5 . This message" +\ " will trigger an alert event from the StreamAlertTransform because the value provided is "\ "less than 10 that was passed in through the config." pub.publish(message) event = queue.get(timeout=10) self.assertEquals(event.type_, "DeviceEvent") self.assertEquals(event.origin, "StreamAlertTransform") # self.purge_queues(exchange_name) # def purge_queues(self, exchange_name): # xn = self.container.ex_manager.create_xn_queue(exchange_name) # xn.purge() @staticmethod def create_process(name= '', module = '', class_name = '', configuration = None): ''' A helper method to create a process ''' producer_definition = ProcessDefinition(name=name) producer_definition.executable = { 'module':module, 'class': class_name } process_dispatcher = ProcessDispatcherServiceClient() procdef_id = process_dispatcher.create_process_definition(process_definition=producer_definition) pid = process_dispatcher.schedule_process(process_definition_id= procdef_id, configuration=configuration) return pid def test_demo_stream_granules_processing(self): """ Test that the Demo Stream Alert Transform is functioning. The transform coordinates with the scheduler. It is configured to listen to a source that publishes granules. It publishes a DeviceStatusEvent if it receives a granule with bad data or a DeviceCommsEvent if no granule has arrived between two timer events. The transform is configured at launch using a config dictionary. """ #------------------------------------------------------------------------------------- # Start a subscriber to listen for an alert event from the Stream Alert Transform #------------------------------------------------------------------------------------- queue_bad_data = gevent.queue.Queue() queue_no_data = gevent.queue.Queue() def bad_data(message, headers): if message.type_ == "DeviceStatusEvent": queue_bad_data.put(message) def no_data(message, headers): queue_no_data.put(message) event_subscriber_bad_data = EventSubscriber( origin="DemoStreamAlertTransform", event_type="DeviceStatusEvent", callback=bad_data) event_subscriber_no_data = EventSubscriber( origin="DemoStreamAlertTransform", event_type="DeviceCommsEvent", callback=no_data) event_subscriber_bad_data.start() event_subscriber_no_data.start() self.addCleanup(event_subscriber_bad_data.stop) self.addCleanup(event_subscriber_no_data.stop) #------------------------------------------------------------------------------------- # The configuration for the Stream Alert Transform... set up the event types to listen to #------------------------------------------------------------------------------------- self.valid_values = [-100, 100] self.timer_interval = 5 self.queue_name = 'a_queue' config = { 'process':{ 'timer_interval': self.timer_interval, 'queue_name': self.queue_name, 'variable_name': 'input_voltage', 'time_field_name': 'preferred_timestamp', 'valid_values': self.valid_values, 'timer_origin': 'Interval Timer' } } #------------------------------------------------------------------------------------- # Create the process #------------------------------------------------------------------------------------- pid = TransformPrototypeIntTest.create_process( name= 'DemoStreamAlertTransform', module='ion.processes.data.transforms.event_alert_transform', class_name='DemoStreamAlertTransform', configuration= config) self.assertIsNotNone(pid) #------------------------------------------------------------------------------------- # Publish streams and make assertions about alerts #------------------------------------------------------------------------------------- pdict_id = self.dataset_management.read_parameter_dictionary_by_name(name= 'platform_eng_parsed', id_only=True) stream_def_id = self.pubsub_management.create_stream_definition('demo_stream', parameter_dictionary_id=pdict_id) stream_id, stream_route = self.pubsub_management.create_stream( name='test_demo_alert', exchange_point='exch_point_1', stream_definition_id=stream_def_id) sub_1 = self.pubsub_management.create_subscription(name='sub_1', stream_ids=[stream_id], exchange_points=['exch_point_1'], exchange_name=self.queue_name) self.pubsub_management.activate_subscription(sub_1) self.exchange_names.append('sub_1') self.exchange_points.append('exch_point_1') #------------------------------------------------------------------------------------- # publish a *GOOD* granule #------------------------------------------------------------------------------------- self.length = 2 val = numpy.array([random.uniform(0,50) for l in xrange(self.length)]) self._publish_granules(stream_id= stream_id, stream_route= stream_route, number=1, values=val, length=self.length) self.assertTrue(queue_bad_data.empty()) #------------------------------------------------------------------------------------- # publish a few *BAD* granules #------------------------------------------------------------------------------------- self.length = 2 self.number = 2 val = numpy.array([random.uniform(110,200) for l in xrange(self.length)]) self._publish_granules(stream_id= stream_id, stream_route= stream_route, number= self.number, values=val, length=self.length) for i in xrange(self.length * self.number): event = queue_bad_data.get(timeout=10) self.assertEquals(event.type_, "DeviceStatusEvent") self.assertEquals(event.origin, "DemoStreamAlertTransform") self.assertEquals(event.state, DeviceStatusType.OUT_OF_RANGE) self.assertEquals(event.valid_values, self.valid_values) self.assertEquals(event.sub_type, 'input_voltage') self.assertIsNotNone(event.value) self.assertIsNotNone(event.time_stamp) # To ensure that only the bad values generated the alert events. Queue should be empty now self.assertEquals(queue_bad_data.qsize(), 0) #------------------------------------------------------------------------------------- # Do not publish any granules for some time. This should generate a DeviceCommsEvent for the communication status #------------------------------------------------------------------------------------- event = queue_no_data.get(timeout=15) self.assertEquals(event.type_, "DeviceCommsEvent") self.assertEquals(event.origin, "DemoStreamAlertTransform") self.assertEquals(event.state, DeviceCommsType.DATA_DELIVERY_INTERRUPTION) self.assertEquals(event.sub_type, 'input_voltage') #------------------------------------------------------------------------------------- # Empty the queues and repeat tests #------------------------------------------------------------------------------------- queue_bad_data.queue.clear() queue_no_data.queue.clear() #------------------------------------------------------------------------------------- # publish a *GOOD* granule again #------------------------------------------------------------------------------------- self.length = 2 val = numpy.array([random.uniform(0,50) for l in xrange(self.length)]) self._publish_granules(stream_id= stream_id, stream_route= stream_route, number=1, values=val, length=self.length) self.assertTrue(queue_bad_data.empty()) #------------------------------------------------------------------------------------- # Again do not publish any granules for some time. This should generate a DeviceCommsEvent for the communication status #------------------------------------------------------------------------------------- event = queue_no_data.get(timeout=20) self.assertEquals(event.type_, "DeviceCommsEvent") self.assertEquals(event.origin, "DemoStreamAlertTransform") self.assertEquals(event.state, DeviceCommsType.DATA_DELIVERY_INTERRUPTION) self.assertEquals(event.sub_type, 'input_voltage') def _publish_granules(self, stream_id=None, stream_route=None, values = None,number=None, length=None): pub = StandaloneStreamPublisher(stream_id, stream_route) stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id) stream_def_id = stream_def._id rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) for i in xrange(number): rdt['input_voltage'] = values rdt['preferred_timestamp'] = numpy.array([random.uniform(0,1000) for l in xrange(length)]) g = rdt.to_granule() pub.publish(g)
class TestDMEnd2End(IonIntegrationTestCase): def setUp(self): # Love the non pep-8 convention self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.process_dispatcher = ProcessDispatcherServiceClient() self.pubsub_management = PubsubManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() self.ingestion_management = IngestionManagementServiceClient() self.data_retriever = DataRetrieverServiceClient() self.pids = [] self.event = Event() self.exchange_space_name = 'test_granules' self.exchange_point_name = 'science_data' self.i = 0 self.purge_queues() self.queue_buffer = [] self.streams = [] self.addCleanup(self.stop_all_ingestion) def purge_queues(self): xn = self.container.ex_manager.create_xn_queue( 'science_granule_ingestion') xn.purge() def tearDown(self): self.purge_queues() for pid in self.pids: self.container.proc_manager.terminate_process(pid) IngestionManagementIntTest.clean_subscriptions() for queue in self.queue_buffer: if isinstance(queue, ExchangeNameQueue): queue.delete() elif isinstance(queue, str): xn = self.container.ex_manager.create_xn_queue(queue) xn.delete() #-------------------------------------------------------------------------------- # Helper/Utility methods #-------------------------------------------------------------------------------- def create_dataset(self, parameter_dict_id=''): ''' Creates a time-series dataset ''' tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() if not parameter_dict_id: parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) dataset_id = self.dataset_management.create_dataset( 'test_dataset_%i' % self.i, parameter_dictionary_id=parameter_dict_id, spatial_domain=sdom, temporal_domain=tdom) return dataset_id def get_datastore(self, dataset_id): ''' Gets an instance of the datastore This method is primarily used to defeat a bug where integration tests in multiple containers may sometimes delete a CouchDB datastore and the other containers are unaware of the new state of the datastore. ''' dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore( datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def get_ingestion_config(self): ''' Grab the ingestion configuration from the resource registry ''' # The ingestion configuration should have been created by the bootstrap service # which is configured through r2deploy.yml ingest_configs, _ = self.resource_registry.find_resources( restype=RT.IngestionConfiguration, id_only=True) return ingest_configs[0] def launch_producer(self, stream_id=''): ''' Launch the producer ''' pid = self.container.spawn_process( 'better_data_producer', 'ion.processes.data.example_data_producer', 'BetterDataProducer', {'process': { 'stream_id': stream_id }}) self.pids.append(pid) def make_simple_dataset(self): ''' Makes a stream, a stream definition and a dataset, the essentials for most of these tests ''' pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) stream_def_id = self.pubsub_management.create_stream_definition( 'ctd data', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream( 'ctd stream %i' % self.i, 'xp1', stream_definition_id=stream_def_id) dataset_id = self.create_dataset(pdict_id) self.get_datastore(dataset_id) self.i += 1 return stream_id, route, stream_def_id, dataset_id def publish_hifi(self, stream_id, stream_route, offset=0): ''' Publish deterministic data ''' pub = StandaloneStreamPublisher(stream_id, stream_route) stream_def = self.pubsub_management.read_stream_definition( stream_id=stream_id) stream_def_id = stream_def._id rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) + (offset * 10) rdt['temp'] = np.arange(10) + (offset * 10) pub.publish(rdt.to_granule()) def publish_fake_data(self, stream_id, route): ''' Make four granules ''' for i in xrange(4): self.publish_hifi(stream_id, route, i) def start_ingestion(self, stream_id, dataset_id): ''' Starts ingestion/persistence for a given dataset ''' ingest_config_id = self.get_ingestion_config() self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id) def stop_ingestion(self, stream_id): ingest_config_id = self.get_ingestion_config() self.ingestion_management.unpersist_data_stream( stream_id=stream_id, ingestion_configuration_id=ingest_config_id) def stop_all_ingestion(self): try: [self.stop_ingestion(sid) for sid in self.streams] except: pass def validate_granule_subscription(self, msg, route, stream_id): ''' Validation for granule format ''' if msg == {}: return rdt = RecordDictionaryTool.load_from_granule(msg) log.info('%s', rdt.pretty_print()) self.assertIsInstance( msg, Granule, 'Message is improperly formatted. (%s)' % type(msg)) self.event.set() def wait_until_we_have_enough_granules(self, dataset_id='', data_size=40): ''' Loops until there is a sufficient amount of data in the dataset ''' done = False with gevent.Timeout(40): while not done: extents = self.dataset_management.dataset_extents( dataset_id, 'time')[0] granule = self.data_retriever.retrieve_last_data_points( dataset_id, 1) rdt = RecordDictionaryTool.load_from_granule(granule) if rdt['time'] and rdt['time'][0] != rdt._pdict.get_context( 'time').fill_value and extents >= data_size: done = True else: gevent.sleep(0.2) #-------------------------------------------------------------------------------- # Test Methods #-------------------------------------------------------------------------------- @attr('SMOKE') def test_dm_end_2_end(self): #-------------------------------------------------------------------------------- # Set up a stream and have a mock instrument (producer) send data #-------------------------------------------------------------------------------- self.event.clear() # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) context_ids = self.dataset_management.read_parameter_contexts( pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append( self.dataset_management.create_parameter_context( 'binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append( self.dataset_management.create_parameter_context( 'records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary( 'replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_definition = self.pubsub_management.create_stream_definition( 'ctd data', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream( 'producer', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition) #-------------------------------------------------------------------------------- # Start persisting the data on the stream # - Get the ingestion configuration from the resource registry # - Create the dataset # - call persist_data_stream to setup the subscription for the ingestion workers # on the stream that you specify which causes the data to be persisted #-------------------------------------------------------------------------------- ingest_config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id) #-------------------------------------------------------------------------------- # Now the granules are ingesting and persisted #-------------------------------------------------------------------------------- self.launch_producer(stream_id) self.wait_until_we_have_enough_granules(dataset_id, 40) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_id) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) self.assertTrue((rdt['time'][:10] == np.arange(10)).all(), '%s' % rdt['time'][:]) self.assertTrue((rdt['binary'][:10] == np.array(['hi'] * 10, dtype='object')).all()) #-------------------------------------------------------------------------------- # Now to try the streamed approach #-------------------------------------------------------------------------------- replay_stream_id, replay_route = self.pubsub_management.create_stream( 'replay_out', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition) self.replay_id, process_id = self.data_retriever.define_replay( dataset_id=dataset_id, stream_id=replay_stream_id) log.info('Process ID: %s', process_id) replay_client = ReplayClient(process_id) #-------------------------------------------------------------------------------- # Create the listening endpoint for the the retriever to talk to #-------------------------------------------------------------------------------- xp = self.container.ex_manager.create_xp(self.exchange_point_name) subscriber = StandaloneStreamSubscriber( self.exchange_space_name, self.validate_granule_subscription) self.queue_buffer.append(self.exchange_space_name) subscriber.start() subscriber.xn.bind(replay_route.routing_key, xp) self.data_retriever.start_replay_agent(self.replay_id) self.assertTrue(replay_client.await_agent_ready(5), 'The process never launched') replay_client.start_replay() self.assertTrue(self.event.wait(10)) subscriber.stop() self.data_retriever.cancel_replay_agent(self.replay_id) #-------------------------------------------------------------------------------- # Test the slicing capabilities #-------------------------------------------------------------------------------- granule = self.data_retriever.retrieve(dataset_id=dataset_id, query={'tdoa': slice(0, 5)}) rdt = RecordDictionaryTool.load_from_granule(granule) b = rdt['time'] == np.arange(5) self.assertTrue(b.all() if not isinstance(b, bool) else b) self.streams.append(stream_id) self.stop_ingestion(stream_id) @unittest.skip('Doesnt work') @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_replay_pause(self): # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) context_ids = self.dataset_management.read_parameter_contexts( pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append( self.dataset_management.create_parameter_context( 'binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append( self.dataset_management.create_parameter_context( 'records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary( 'replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition( 'replay_stream', parameter_dictionary_id=pdict_id) replay_stream, replay_route = self.pubsub_management.create_stream( 'replay', 'xp1', stream_definition_id=stream_def_id) dataset_id = self.create_dataset(pdict_id) scov = DatasetManagementService._get_coverage(dataset_id) bb = CoverageCraft(scov) bb.rdt['time'] = np.arange(100) bb.rdt['temp'] = np.random.random(100) + 30 bb.sync_with_granule() DatasetManagementService._persist_coverage( dataset_id, bb.coverage) # This invalidates it for multi-host configurations # Set up the subscriber to verify the data subscriber = StandaloneStreamSubscriber( self.exchange_space_name, self.validate_granule_subscription) xp = self.container.ex_manager.create_xp('xp1') self.queue_buffer.append(self.exchange_space_name) subscriber.start() subscriber.xn.bind(replay_route.routing_key, xp) # Set up the replay agent and the client wrapper # 1) Define the Replay (dataset and stream to publish on) self.replay_id, process_id = self.data_retriever.define_replay( dataset_id=dataset_id, stream_id=replay_stream) # 2) Make a client to the interact with the process (optionall provide it a process to bind with) replay_client = ReplayClient(process_id) # 3) Start the agent (launch the process) self.data_retriever.start_replay_agent(self.replay_id) # 4) Start replaying... replay_client.start_replay() # Wait till we get some granules self.assertTrue(self.event.wait(5)) # We got granules, pause the replay, clear the queue and allow the process to finish consuming replay_client.pause_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure there's no remaining messages being consumed self.assertFalse(self.event.wait(1)) # Resume the replay and wait until we start getting granules again replay_client.resume_replay() self.assertTrue(self.event.wait(5)) # Stop the replay, clear the queues replay_client.stop_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure that it did indeed stop self.assertFalse(self.event.wait(1)) subscriber.stop() def test_retrieve_and_transform(self): # Make a simple dataset and start ingestion, pretty standard stuff. ctd_stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset( ) self.start_ingestion(ctd_stream_id, dataset_id) # Stream definition for the salinity data salinity_pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) sal_stream_def_id = self.pubsub_management.create_stream_definition( 'sal data', parameter_dictionary_id=salinity_pdict_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) rdt['temp'] = np.random.randn(10) * 10 + 30 rdt['conductivity'] = np.random.randn(10) * 2 + 10 rdt['pressure'] = np.random.randn(10) * 1 + 12 publisher = StandaloneStreamPublisher(ctd_stream_id, route) publisher.publish(rdt.to_granule()) rdt['time'] = np.arange(10, 20) publisher.publish(rdt.to_granule()) self.wait_until_we_have_enough_granules(dataset_id, 20) granule = self.data_retriever.retrieve( dataset_id, None, None, 'ion.processes.data.transforms.ctd.ctd_L2_salinity', 'CTDL2SalinityTransformAlgorithm', kwargs=dict(params=sal_stream_def_id)) rdt = RecordDictionaryTool.load_from_granule(granule) for i in rdt['salinity']: self.assertNotEquals(i, 0) self.streams.append(ctd_stream_id) self.stop_ingestion(ctd_stream_id) def test_last_granule(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset( ) self.start_ingestion(stream_id, dataset_id) self.publish_hifi(stream_id, route, 0) self.publish_hifi(stream_id, route, 1) self.wait_until_we_have_enough_granules(dataset_id, 20) # I just need two success = False def verifier(): replay_granule = self.data_retriever.retrieve_last_data_points( dataset_id, 10) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(10) + 10 if not isinstance(comp, bool): return comp.all() return False success = poll(verifier) self.assertTrue(success) success = False def verify_points(): replay_granule = self.data_retriever.retrieve_last_data_points( dataset_id, 5) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(15, 20) if not isinstance(comp, bool): return comp.all() return False success = poll(verify_points) self.assertTrue(success) self.streams.append(stream_id) self.stop_ingestion(stream_id) def test_replay_with_parameters(self): #-------------------------------------------------------------------------------- # Create the configurations and the dataset #-------------------------------------------------------------------------------- # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) context_ids = self.dataset_management.read_parameter_contexts( pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append( self.dataset_management.create_parameter_context( 'binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append( self.dataset_management.create_parameter_context( 'records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary( 'replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition( 'replay_stream', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream( 'replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id) dataset_modified = Event() def cb(*args, **kwargs): dataset_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id) es.start() self.addCleanup(es.stop) self.publish_fake_data(stream_id, route) self.assertTrue(dataset_modified.wait(30)) query = { 'start_time': 0 - 2208988800, 'end_time': 20 - 2208988800, 'stride_time': 2, 'parameters': ['time', 'temp'] } retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id, query=query) rdt = RecordDictionaryTool.load_from_granule(retrieved_data) comp = np.arange(0, 20, 2) == rdt['time'] self.assertTrue(comp.all(), '%s' % rdt.pretty_print()) self.assertEquals(set(rdt.iterkeys()), set(['time', 'temp'])) extents = self.dataset_management.dataset_extents( dataset_id=dataset_id, parameters=['time', 'temp']) self.assertTrue(extents['time'] >= 20) self.assertTrue(extents['temp'] >= 20) self.streams.append(stream_id) self.stop_ingestion(stream_id) def test_repersist_data(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset( ) self.start_ingestion(stream_id, dataset_id) self.publish_hifi(stream_id, route, 0) self.publish_hifi(stream_id, route, 1) self.wait_until_we_have_enough_granules(dataset_id, 20) config_id = self.get_ingestion_config() self.ingestion_management.unpersist_data_stream( stream_id=stream_id, ingestion_configuration_id=config_id) self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id) self.publish_hifi(stream_id, route, 2) self.publish_hifi(stream_id, route, 3) self.wait_until_we_have_enough_granules(dataset_id, 40) success = False with gevent.timeout.Timeout(5): while not success: replay_granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(0, 40) if not isinstance(comp, bool): success = comp.all() gevent.sleep(1) self.assertTrue(success) self.streams.append(stream_id) self.stop_ingestion(stream_id) @attr('LOCOINT') @unittest.skipIf(os.getenv( 'CEI_LAUNCH_TEST', False ), 'Host requires file-system access to coverage files, CEI mode does not support.' ) def test_correct_time(self): # There are 2208988800 seconds between Jan 1 1900 and Jan 1 1970, i.e. # the conversion factor between unix and NTP time unix_now = np.floor(time.time()) ntp_now = unix_now + 2208988800 unix_ago = unix_now - 20 ntp_ago = unix_ago + 2208988800 stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset( ) coverage = DatasetManagementService._get_coverage(dataset_id) coverage.insert_timesteps(20) coverage.set_parameter_values('time', np.arange(ntp_ago, ntp_now)) temporal_bounds = self.dataset_management.dataset_temporal_bounds( dataset_id) self.assertTrue(np.abs(temporal_bounds[0] - unix_ago) < 2) self.assertTrue(np.abs(temporal_bounds[1] - unix_now) < 2) @attr('LOCOINT') @unittest.skipIf(os.getenv( 'CEI_LAUNCH_TEST', False ), 'Host requires file-system access to coverage files, CEI mode does not support.' ) def test_empty_coverage_time(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset( ) coverage = DatasetManagementService._get_coverage(dataset_id) temporal_bounds = self.dataset_management.dataset_temporal_bounds( dataset_id) self.assertEquals([coverage.get_parameter_context('time').fill_value] * 2, temporal_bounds) @attr('LOCOINT') @unittest.skipIf(os.getenv( 'CEI_LAUNCH_TEST', False ), 'Host requires file-system access to coverage files, CEI mode does not support.' ) def test_out_of_band_retrieve(self): # Setup the environemnt stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset( ) self.start_ingestion(stream_id, dataset_id) # Fill the dataset self.publish_fake_data(stream_id, route) self.wait_until_we_have_enough_granules(dataset_id, 40) # Retrieve the data granule = DataRetrieverService.retrieve_oob(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) self.assertTrue((rdt['time'] == np.arange(40)).all()) @attr('LOCOINT') @unittest.skipIf(os.getenv( 'CEI_LAUNCH_TEST', False ), 'Host requires file-system access to coverage files, CEI mode does not support.' ) def test_retrieve_cache(self): DataRetrieverService._refresh_interval = 1 datasets = [self.make_simple_dataset() for i in xrange(10)] for stream_id, route, stream_def_id, dataset_id in datasets: coverage = DatasetManagementService._get_coverage(dataset_id) coverage.insert_timesteps(10) coverage.set_parameter_values('time', np.arange(10)) coverage.set_parameter_values('temp', np.arange(10)) # Verify cache hit and refresh dataset_ids = [i[3] for i in datasets] self.assertTrue( dataset_ids[0] not in DataRetrieverService._retrieve_cache) DataRetrieverService._get_coverage(dataset_ids[0]) # Hit the chache cov, age = DataRetrieverService._retrieve_cache[dataset_ids[0]] # Verify that it was hit and it's now in there self.assertTrue(dataset_ids[0] in DataRetrieverService._retrieve_cache) gevent.sleep(DataRetrieverService._refresh_interval + 0.2) DataRetrieverService._get_coverage(dataset_ids[0]) # Hit the chache cov, age2 = DataRetrieverService._retrieve_cache[dataset_ids[0]] self.assertTrue(age2 != age) for dataset_id in dataset_ids: DataRetrieverService._get_coverage(dataset_id) self.assertTrue( dataset_ids[0] not in DataRetrieverService._retrieve_cache) stream_id, route, stream_def, dataset_id = datasets[0] self.start_ingestion(stream_id, dataset_id) DataRetrieverService._get_coverage(dataset_id) self.assertTrue(dataset_id in DataRetrieverService._retrieve_cache) DataRetrieverService._refresh_interval = 100 self.publish_hifi(stream_id, route, 1) self.wait_until_we_have_enough_granules(dataset_id, data_size=20) event = gevent.event.Event() with gevent.Timeout(20): while not event.wait(0.1): if dataset_id not in DataRetrieverService._retrieve_cache: event.set() self.assertTrue(event.is_set()) @unittest.skip('Outdated due to ingestion retry') @attr('LOCOINT') @unittest.skipIf(os.getenv( 'CEI_LAUNCH_TEST', False ), 'Host requires file-system access to coverage files, CEI mode does not support.' ) def test_ingestion_failover(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset( ) self.start_ingestion(stream_id, dataset_id) event = Event() def cb(*args, **kwargs): event.set() sub = EventSubscriber(event_type="ExceptionEvent", callback=cb, origin="stream_exception") sub.start() self.publish_fake_data(stream_id, route) self.wait_until_we_have_enough_granules(dataset_id, 40) file_path = DatasetManagementService._get_coverage_path(dataset_id) master_file = os.path.join(file_path, '%s_master.hdf5' % dataset_id) with open(master_file, 'w') as f: f.write('this will crash HDF') self.publish_hifi(stream_id, route, 5) self.assertTrue(event.wait(10)) sub.stop()
class TestParticleConversion(IonIntegrationTestCase): """ Test cases to confirm all data particles convert to granules successfully. """ ############################################################################ # Setup, teardown. ############################################################################ def setUp(self): # Start container. log.info('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a pubsub client to create streams. self.pubsub_client = PubsubManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() def create_granule(self, stream_name, param_dict_name, particle_list): pd_id = self.dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = self.pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) stream_def = self.pubsub_client.read_stream_definition(stream_def_id) rdt = RecordDictionaryTool(stream_definition=stream_def) rdt = populate_rdt(rdt, particle_list) print '###################### RDT:' print str(rdt) g = rdt.to_granule(data_producer_id='fake_agent_id') return g def test_sbe37_particles(self): """ test_particles """ stream_name = 'parsed' param_dict_name = 'ctd_parsed_param_dict' particle_list = [{u'quality_flag': u'ok', u'preferred_timestamp': u'port_timestamp', u'stream_name': u'parsed', u'port_timestamp': 3578927139.3578925, u'pkt_format_id': u'JSON_Data', u'pkt_version': 1, u'values': [{u'value_id': u'temp', u'value': 68.5895}, {u'value_id': u'conductivity', u'value': 26.72304}, {u'value_id': u'pressure', u'value': 733.303}], u'driver_timestamp': 3578927139.4226017}] try: g = self.create_granule(stream_name, param_dict_name, particle_list) #self.assertSBE37ParsedGranule(g) except Exception as e: errmsg = 'Granule creation failed: %s' % str(e) errmsg += '\n stream_name: ' + stream_name errmsg += '\n param_dict_name: ' + param_dict_name errmsg += '\n particle list: %s' % str(particle_list) self.fail(errmsg) stream_name = 'raw' param_dict_name = 'ctd_raw_param_dict' particle_list = [{u'quality_flag': u'ok', u'preferred_timestamp': u'port_timestamp', u'stream_name': u'raw', u'port_timestamp': 3578927113.3578925, u'pkt_format_id': u'JSON_Data', u'pkt_version': 1, u'values': [{u'binary': True, u'value_id': u'raw', u'value': u'ZAA='}, {u'value_id': u'length', u'value': 2}, {u'value_id': u'type', u'value': 1}, {u'value_id': u'checksum', u'value': None}], u'driver_timestamp': 3578927113.75216}] try: g = self.create_granule(stream_name, param_dict_name, particle_list) #self.assertSBE37RawGranule(g) except Exception as e: errmsg = 'Granule creation failed: %s' % str(s) errmsg += '\n stream_name: ' + stream_name errmsg += '\n param_dict_name: ' + param_dict_name errmsg += '\n particle list: %s' % str(particle_list) self.fail(errmsg) def test_vel3d_particles(self): """ test_particles """ stream_name = 'vel3d_parsed' param_dict_name = 'vel3d_b_sample' particle_list = [ { "driver_timestamp": 3579022766.361967, "internal_timestamp": 3579047922.0, "pkt_format_id": "JSON_Data", "pkt_version": 1, "port_timestamp": 3579022762.357902, "preferred_timestamp": "port_timestamp", "quality_flag": "ok", "stream_name": "vel3d_b_sample", "values": [ {"value": 3579047922.0, "value_id": "date_time_string"}, {"value": 5, "value_id": "fractional_second"}, {"value": "8000", "value_id": "velocity_beam_a"}, {"value": "8000", "value_id": "velocity_beam_b"}, {"value": "8000", "value_id": "velocity_beam_c"}, {"value": "8000", "value_id": "velocity_beam_d"}, {"value": 999.0, "value_id": "turbulent_velocity_east"}, {"value": 999.0, "value_id": "turbulent_velocity_north"}, {"value": 999.0, "value_id": "turbulent_velocity_up"}, {"value": 2.16, "value_id": "temperature"}, {"value": 1.0, "value_id": "mag_comp_x"}, {"value": -0.0, "value_id": "mag_comp_y"}, {"value": -7.9, "value_id": "pitch"}, {"value": -78.2, "value_id": "roll"}] } ] class RDT(dict): def __init__(self): super(RDT, self).__init__() self.temporal_parameter = None rdt = RDT() for x in particle_list[0]['values']: rdt[x['value_id']] = None rdt = populate_rdt(rdt, particle_list)
class RecordDictionaryIntegrationTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dataset_management = DatasetManagementServiceClient() self.pubsub_management = PubsubManagementServiceClient() self.rdt = None self.data_producer_id = None self.provider_metadata_update = None self.event = Event() def verify_incoming(self, m, r, s): rdt = RecordDictionaryTool.load_from_granule(m) self.assertEquals(rdt, self.rdt) self.assertEquals(m.data_producer_id, self.data_producer_id) self.assertEquals(m.provider_metadata_update, self.provider_metadata_update) self.assertNotEqual(m.creation_timestamp, None) self.event.set() def test_serialize_compatability(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_extended_parsed() stream_def_id = self.pubsub_management.create_stream_definition( 'ctd extended', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) stream_id, route = self.pubsub_management.create_stream( 'ctd1', 'xp1', stream_definition_id=stream_def_id) self.addCleanup(self.pubsub_management.delete_stream, stream_id) sub_id = self.pubsub_management.create_subscription( 'sub1', stream_ids=[stream_id]) self.addCleanup(self.pubsub_management.delete_subscription, sub_id) self.pubsub_management.activate_subscription(sub_id) self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id) verified = Event() def verifier(msg, route, stream_id): for k, v in msg.record_dictionary.iteritems(): if v is not None: self.assertIsInstance(v, np.ndarray) rdt = RecordDictionaryTool.load_from_granule(msg) for field in rdt.fields: self.assertIsInstance(rdt[field], np.ndarray) verified.set() subscriber = StandaloneStreamSubscriber('sub1', callback=verifier) subscriber.start() self.addCleanup(subscriber.stop) publisher = StandaloneStreamPublisher(stream_id, route) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) ph.fill_rdt(rdt, 10) publisher.publish(rdt.to_granule()) self.assertTrue(verified.wait(10)) def test_granule(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) stream_def_id = self.pubsub_management.create_stream_definition( 'ctd', parameter_dictionary_id=pdict_id, stream_configuration={ 'reference_designator': "GA03FLMA-RI001-13-CTDMOG999" }) pdict = DatasetManagementService.get_parameter_dictionary_by_name( 'ctd_parsed_param_dict') self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) stream_id, route = self.pubsub_management.create_stream( 'ctd_stream', 'xp1', stream_definition_id=stream_def_id) self.addCleanup(self.pubsub_management.delete_stream, stream_id) publisher = StandaloneStreamPublisher(stream_id, route) subscriber = StandaloneStreamSubscriber('sub', self.verify_incoming) subscriber.start() self.addCleanup(subscriber.stop) subscription_id = self.pubsub_management.create_subscription( 'sub', stream_ids=[stream_id]) self.pubsub_management.activate_subscription(subscription_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) rdt['temp'] = np.random.randn(10) * 10 + 30 rdt['pressure'] = [20] * 10 self.assertEquals(set(pdict.keys()), set(rdt.fields)) self.assertEquals(pdict.temporal_parameter_name, rdt.temporal_parameter) self.assertEquals(rdt._stream_config['reference_designator'], "GA03FLMA-RI001-13-CTDMOG999") self.rdt = rdt self.data_producer_id = 'data_producer' self.provider_metadata_update = {1: 1} publisher.publish( rdt.to_granule(data_producer_id='data_producer', provider_metadata_update={1: 1})) self.assertTrue(self.event.wait(10)) self.pubsub_management.deactivate_subscription(subscription_id) self.pubsub_management.delete_subscription(subscription_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.array([None, None, None]) self.assertTrue(rdt['time'] is None) rdt['time'] = np.array([None, 1, 2]) self.assertEquals(rdt['time'][0], rdt.fill_value('time')) stream_def_obj = self.pubsub_management.read_stream_definition( stream_def_id) rdt = RecordDictionaryTool(stream_definition=stream_def_obj) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) granule = rdt.to_granule() rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_equal(rdt['time'], np.arange(20)) np.testing.assert_array_equal(rdt['temp'], np.arange(20)) def test_filter(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) filtered_stream_def_id = self.pubsub_management.create_stream_definition( 'filtered', parameter_dictionary_id=pdict_id, available_fields=['time', 'temp']) self.addCleanup(self.pubsub_management.delete_stream_definition, filtered_stream_def_id) rdt = RecordDictionaryTool(stream_definition_id=filtered_stream_def_id) self.assertEquals(rdt._available_fields, ['time', 'temp']) rdt['time'] = np.arange(20) rdt['temp'] = np.arange(20) with self.assertRaises(KeyError): rdt['pressure'] = np.arange(20) granule = rdt.to_granule(connection_id='c1', connection_index='0') rdt2 = RecordDictionaryTool.load_from_granule(granule) self.assertEquals(rdt._available_fields, rdt2._available_fields) self.assertEquals(rdt.fields, rdt2.fields) self.assertEquals(rdt2.connection_id, 'c1') self.assertEquals(rdt2.connection_index, '0') for k, v in rdt.iteritems(): self.assertTrue(np.array_equal(rdt[k], rdt2[k])) def test_rdt_param_funcs(self): rdt = self.create_rdt() rdt['TIME'] = [0] rdt['TEMPWAT_L0'] = [280000] rdt['CONDWAT_L0'] = [100000] rdt['PRESWAT_L0'] = [2789] rdt['LAT'] = [45] rdt['LON'] = [-71] np.testing.assert_array_almost_equal( rdt['DENSITY'], np.array([1001.76506258], dtype='float32')) def test_rdt_lookup(self): rdt = self.create_lookup_rdt() self.assertTrue('offset_a' in rdt.lookup_values()) self.assertFalse('offset_b' in rdt.lookup_values()) rdt['time'] = [0] rdt['temp'] = [10.0] rdt['offset_a'] = [2.0] self.assertEquals(rdt['offset_b'], None) self.assertEquals(rdt.lookup_values(), ['offset_a']) np.testing.assert_array_almost_equal(rdt['calibrated'], np.array([12.0])) svm = StoredValueManager(self.container) svm.stored_value_cas('coefficient_document', {'offset_b': 2.0}) svm.stored_value_cas("GA03FLMA-RI001-13-CTDMOG999_OFFSETC", {'offset_c': 3.0}) rdt.fetch_lookup_values() np.testing.assert_array_equal(rdt['offset_b'], np.array([2.0])) np.testing.assert_array_equal(rdt['calibrated_b'], np.array([14.0])) np.testing.assert_array_equal(rdt['offset_c'], np.array([3.0])) def create_rdt(self): contexts, pfuncs = self.create_pfuncs() context_ids = [_id for ct, _id in contexts.itervalues()] pdict_id = self.dataset_management.create_parameter_dictionary( name='functional_pdict', parameter_context_ids=context_ids, temporal_context='test_TIME') self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id) stream_def_id = self.pubsub_management.create_stream_definition( 'functional', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) return rdt def create_lookup_rdt(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_lookups() stream_def_id = self.pubsub_management.create_stream_definition( 'lookup', parameter_dictionary_id=pdict_id, stream_configuration={ 'reference_designator': "GA03FLMA-RI001-13-CTDMOG999" }) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) return rdt def create_pfuncs(self): contexts = {} funcs = {} t_ctxt = ParameterContext( 'TIME', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 1900-01-01' t_ctxt_id = self.dataset_management.create_parameter_context( name='test_TIME', parameter_context=t_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, t_ctxt_id) contexts['TIME'] = (t_ctxt, t_ctxt_id) lat_ctxt = ParameterContext( 'LAT', param_type=ConstantType( QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt_id = self.dataset_management.create_parameter_context( name='test_LAT', parameter_context=lat_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, lat_ctxt_id) contexts['LAT'] = lat_ctxt, lat_ctxt_id lon_ctxt = ParameterContext( 'LON', param_type=ConstantType( QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt_id = self.dataset_management.create_parameter_context( name='test_LON', parameter_context=lon_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, lon_ctxt_id) contexts['LON'] = lon_ctxt, lon_ctxt_id # Independent Parameters # Temperature - values expected to be the decimal results of conversion from hex temp_ctxt = ParameterContext( 'TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) temp_ctxt.uom = 'deg_C' temp_ctxt_id = self.dataset_management.create_parameter_context( name='test_TEMPWAT_L0', parameter_context=temp_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, temp_ctxt_id) contexts['TEMPWAT_L0'] = temp_ctxt, temp_ctxt_id # Conductivity - values expected to be the decimal results of conversion from hex cond_ctxt = ParameterContext( 'CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) cond_ctxt.uom = 'S m-1' cond_ctxt_id = self.dataset_management.create_parameter_context( name='test_CONDWAT_L0', parameter_context=cond_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, cond_ctxt_id) contexts['CONDWAT_L0'] = cond_ctxt, cond_ctxt_id # Pressure - values expected to be the decimal results of conversion from hex press_ctxt = ParameterContext( 'PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) press_ctxt.uom = 'dbar' press_ctxt_id = self.dataset_management.create_parameter_context( name='test_PRESWAT_L0', parameter_context=press_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, press_ctxt_id) contexts['PRESWAT_L0'] = press_ctxt, press_ctxt_id # Dependent Parameters # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10 tl1_func = '(T / 10000) - 10' expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T']) expr_id = self.dataset_management.create_parameter_function( name='test_TEMPWAT_L1', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['TEMPWAT_L1'] = expr, expr_id tl1_pmap = {'T': 'TEMPWAT_L0'} expr.param_map = tl1_pmap tempL1_ctxt = ParameterContext( 'TEMPWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) tempL1_ctxt.uom = 'deg_C' tempL1_ctxt_id = self.dataset_management.create_parameter_context( name='test_TEMPWAT_L1', parameter_context=tempL1_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, tempL1_ctxt_id) contexts['TEMPWAT_L1'] = tempL1_ctxt, tempL1_ctxt_id # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5 cl1_func = '(C / 100000) - 0.5' expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C']) expr_id = self.dataset_management.create_parameter_function( name='test_CONDWAT_L1', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['CONDWAT_L1'] = expr, expr_id cl1_pmap = {'C': 'CONDWAT_L0'} expr.param_map = cl1_pmap condL1_ctxt = ParameterContext( 'CONDWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) condL1_ctxt.uom = 'S m-1' condL1_ctxt_id = self.dataset_management.create_parameter_context( name='test_CONDWAT_L1', parameter_context=condL1_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, condL1_ctxt_id) contexts['CONDWAT_L1'] = condL1_ctxt, condL1_ctxt_id # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721 # PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range) pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)' expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range']) expr_id = self.dataset_management.create_parameter_function( name='test_PRESWAT_L1', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['PRESWAT_L1'] = expr, expr_id pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721} expr.param_map = pl1_pmap presL1_ctxt = ParameterContext( 'PRESWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) presL1_ctxt.uom = 'S m-1' presL1_ctxt_id = self.dataset_management.create_parameter_context( name='test_CONDWAT_L1', parameter_context=presL1_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, presL1_ctxt_id) contexts['PRESWAT_L1'] = presL1_ctxt, presL1_ctxt_id # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1) owner = 'gsw' sal_func = 'SP_from_C' sal_arglist = ['C', 't', 'p'] expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist) expr_id = self.dataset_management.create_parameter_function( name='test_PRACSAL', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['PRACSAL'] = expr, expr_id # A magic function that may or may not exist actually forms the line below at runtime. sal_pmap = { 'C': NumexprFunction('CONDWAT_L1*10', 'C*10', ['C'], param_map={'C': 'CONDWAT_L1'}), 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1' } expr.param_map = sal_pmap sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL) sal_ctxt.uom = 'g kg-1' sal_ctxt_id = self.dataset_management.create_parameter_context( name='test_PRACSAL', parameter_context=sal_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, sal_ctxt_id) contexts['PRACSAL'] = sal_ctxt, sal_ctxt_id # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1) # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1) owner = 'gsw' abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT']) cons_temp_expr = PythonFunction( 'cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1']) dens_expr = PythonFunction( 'DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1']) dens_ctxt = ParameterContext( 'DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL) dens_ctxt.uom = 'kg m-3' dens_ctxt_id = self.dataset_management.create_parameter_context( name='test_DENSITY', parameter_context=dens_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, dens_ctxt_id) contexts['DENSITY'] = dens_ctxt, dens_ctxt_id return contexts, funcs
class ExternalDatasetAgentTestBase(object): # Agent parameters. EDA_RESOURCE_ID = '123xyz' EDA_NAME = 'ExampleEDA' EDA_MOD = 'ion.agents.data.external_dataset_agent' EDA_CLS = 'ExternalDatasetAgent' """ Test cases for instrument agent class. Functions in this class provide instrument agent integration tests and provide a tutorial on use of the agent setup and interface. """ def setUp(self): """ Initialize test members. """ #log.warn('Starting the container') # Start container. self._start_container() # Bring up services in a deploy file #log.warn('Starting the rel') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a pubsub client to create streams. # log.warn('Init a pubsub client') self._pubsub_client = PubsubManagementServiceClient(node=self.container.node) # log.warn('Init a ContainerAgentClient') self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) # Data async and subscription TODO: Replace with new subscriber self._finished_count = None #TODO: Switch to gevent.queue.Queue self._async_finished_result = AsyncResult() self._finished_events_received = [] self._finished_event_subscriber = None self._start_finished_event_subscriber() self.addCleanup(self._stop_finished_event_subscriber) # TODO: Finish dealing with the resources and whatnot # TODO: DVR_CONFIG and (potentially) stream_config could both be reconfigured in self._setup_resources() self._setup_resources() #TG: Setup/configure the granule logger to log granules as they're published # Create agent config. agent_config = { 'driver_config': self.DVR_CONFIG, 'stream_config': {}, 'agent': {'resource_id': self.EDA_RESOURCE_ID}, 'test_mode': True } # Start instrument agent. self._ia_pid = None log.debug('TestInstrumentAgent.setup(): starting EDA.') self._ia_pid = self._container_client.spawn_process( name=self.EDA_NAME, module=self.EDA_MOD, cls=self.EDA_CLS, config=agent_config ) log.info('Agent pid=%s.', str(self._ia_pid)) # Start a resource agent client to talk with the instrument agent. self._ia_client = None self._ia_client = ResourceAgentClient(self.EDA_RESOURCE_ID, process=FakeProcess()) log.info('Got ia client %s.', str(self._ia_client)) ######################################## # Private "setup" functions ######################################## def _setup_resources(self): raise NotImplementedError('_setup_resources must be implemented in the subclass') def create_stream_and_logger(self, name, stream_id='', pdict=None): stream_def_id = '' if not stream_id or stream_id is '': if pdict: stream_def_id = self._pubsub_client.create_stream_definition(parameter_dictionary=pdict.dump(), stream_type='stream') stream_id, route = self._pubsub_client.create_stream(name=name, exchange_point='science_data', stream_definition_id=stream_def_id) else: route = self._pubsub_client.read_stream_route(stream_id=stream_id) stream_def = self._pubsub_client.read_stream_definition(stream_id=stream_id) stream_def_id = stream_def._id pid = self._container_client.spawn_process( name=name + '_logger', module='ion.processes.data.stream_granule_logger', cls='StreamGranuleLogger', config={'process': {'stream_id': stream_id}} ) log.info('Started StreamGranuleLogger \'{0}\' subscribed to stream_id={1}'.format(pid, stream_id)) return stream_id, route, stream_def_id def _start_finished_event_subscriber(self): def consume_event(*args, **kwargs): if args[0].description == 'TestingFinished': log.debug('TestingFinished event received') self._finished_events_received.append(args[0]) if self._finished_count and self._finished_count == len(self._finished_events_received): log.debug('Finishing test...') self._async_finished_result.set(len(self._finished_events_received)) log.debug('Called self._async_finished_result.set({0})'.format(len(self._finished_events_received))) self._finished_event_subscriber = EventSubscriber(event_type='DeviceEvent', callback=consume_event) self._finished_event_subscriber.start() def _stop_finished_event_subscriber(self): if self._finished_event_subscriber: self._finished_event_subscriber.stop() self._finished_event_subscriber = None ######################################## # Custom assertion functions ######################################## def assertListsEqual(self, lst1, lst2): lst1.sort() lst2.sort() return lst1 == lst2 def assertSampleDict(self, val): """ Verify the value is a sample dictionary for the sbe37. """ #{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]} self.assertTrue(isinstance(val, dict)) self.assertTrue('c' in val) self.assertTrue('t' in val) self.assertTrue('p' in val) self.assertTrue('time' in val) c = val['c'][0] t = val['t'][0] p = val['p'][0] time = val['time'][0] self.assertTrue(isinstance(c, float)) self.assertTrue(isinstance(t, float)) self.assertTrue(isinstance(p, float)) self.assertTrue(isinstance(time, float)) def assertParamDict(self, pd, all_params=False): """ Verify all device parameters exist and are correct type. """ if all_params: self.assertEqual(set(pd.keys()), set(PARAMS.keys())) for (key, type_val) in PARAMS.iteritems(): if type_val == list or type_val == tuple: self.assertTrue(isinstance(pd[key], (list, tuple))) else: self.assertTrue(isinstance(pd[key], type_val)) else: for (key, val) in pd.iteritems(): self.assertTrue(key in PARAMS) self.assertTrue(isinstance(val, PARAMS[key])) def assertParamVals(self, params, correct_params): """ Verify parameters take the correct values. """ self.assertEqual(set(params.keys()), set(correct_params.keys())) for (key, val) in params.iteritems(): correct_val = correct_params[key] if isinstance(val, float): # Verify to 5% of the larger value. max_val = max(abs(val), abs(correct_val)) self.assertAlmostEqual(val, correct_val, delta=max_val * .01) elif isinstance(val, (list, tuple)): # list of tuple. self.assertEqual(list(val), list(correct_val)) else: # int, bool, str. self.assertEqual(val, correct_val) ######################################## # Test functions ######################################## def test_acquire_data_while_streaming(self): # Test instrument driver execute interface to start and stop streaming mode. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) params = { 'POLLING_INTERVAL': 3 } self._ia_client.set_resource(params) self._finished_count = 1 cmd = AgentCommand(command=DriverEvent.START_AUTOSAMPLE) self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STREAMING) config = get_safe(self.DVR_CONFIG, 'dh_cfg', {}) log.info('Send a constrained request for data: constraints = HIST_CONSTRAINTS_1') config['stream_id'], config['stream_route'], _ = self.create_stream_and_logger(name='stream_id_for_historical_1') config['constraints'] = self.HIST_CONSTRAINTS_1 cmd = AgentCommand(command=DriverEvent.ACQUIRE_SAMPLE, args=[config]) self._ia_client.execute_resource(cmd) cmd = AgentCommand(command=DriverEvent.STOP_AUTOSAMPLE) self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) finished = self._async_finished_result.get(timeout=120) self.assertEqual(finished, self._finished_count) cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_acquire_data(self): cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) log.warn('Send an unconstrained request for data (\'new data\')') cmd = AgentCommand(command=DriverEvent.ACQUIRE_SAMPLE) self._ia_client.execute_resource(command=cmd) state = self._ia_client.get_agent_state() log.info(state) self.assertEqual(state, ResourceAgentState.COMMAND) self._finished_count = 2 config_mods = {} log.info('Send a constrained request for data: constraints = HIST_CONSTRAINTS_1') config_mods['stream_id'], config_mods['stream_route'], _ = self.create_stream_and_logger(name='stream_id_for_historical_1') config_mods['constraints'] = self.HIST_CONSTRAINTS_1 cmd = AgentCommand(command=DriverEvent.ACQUIRE_SAMPLE, args=[config_mods]) self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) log.info('Send a second constrained request for data: constraints = HIST_CONSTRAINTS_2') config_mods['stream_id'], config_mods['stream_route'], _ = self.create_stream_and_logger(name='stream_id_for_historical_2') config_mods['constraints'] = self.HIST_CONSTRAINTS_2 cmd = AgentCommand(command=DriverEvent.ACQUIRE_SAMPLE, args=[config_mods]) self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) finished = self._async_finished_result.get(timeout=120) self.assertEqual(finished, self._finished_count) cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_streaming(self): state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) params = { 'POLLING_INTERVAL': 3 } self._ia_client.set_resource(params) self._finished_count = 3 cmd = AgentCommand(command=DriverEvent.START_AUTOSAMPLE) self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STREAMING) #Assert that data was received # self._async_finished_result.get(timeout=600) # self.assertTrue(len(self._finished_events_received) >= 3) cmd = AgentCommand(command=DriverEvent.STOP_AUTOSAMPLE) self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_command(self): # Test instrument driver get and set interface. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) # Retrieve all resource parameters. reply = self._ia_client.get_resource(params=['DRIVER_PARAMETER_ALL']) self.assertParamDict(reply, True) ## Retrieve a subset of resource parameters. params = [ 'POLLING_INTERVAL' ] reply = self._ia_client.get_resource(params=params) self.assertParamDict(reply) orig_params = reply # Set a subset of resource parameters. new_params = { 'POLLING_INTERVAL': (orig_params['POLLING_INTERVAL'] * 2), } self._ia_client.set_resource(params=new_params) check_new_params = self._ia_client.get_resource(params) self.assertParamVals(check_new_params, new_params) cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_get_set_resource(self): cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) self._ia_client.execute_agent(cmd) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) # Get a couple parameters retval = self._ia_client.get_resource(['POLLING_INTERVAL', 'PATCHABLE_CONFIG_KEYS']) log.debug('Retrieved parameters from agent: {0}'.format(retval)) self.assertTrue(isinstance(retval, dict)) self.assertEqual(type(retval['POLLING_INTERVAL']), int) self.assertEqual(type(retval['PATCHABLE_CONFIG_KEYS']), list) # Attempt to get a parameter that doesn't exist log.debug('Try getting a non-existent parameter \'BAD_PARAM\'') with self.assertRaises(ServerError): self._ia_client.get_resource(['BAD_PARAM']) # Set the polling_interval to a new value, then get it to make sure it set properly self._ia_client.set_resource({'POLLING_INTERVAL': 10}) retval = self._ia_client.get_resource(['POLLING_INTERVAL']) log.debug('Retrieved parameters from agent: {0}'.format(retval)) self.assertTrue(isinstance(retval, dict)) self.assertEqual(retval['POLLING_INTERVAL'], 10) # Attempt to set a parameter that doesn't exist log.debug('Try setting a non-existent parameter \'BAD_PARAM\'') with self.assertRaises(ServerError): self._ia_client.set_resource({'BAD_PARAM': 'bad_val'}) # Attempt to set one parameter that does exist, and one that doesn't with self.assertRaises(ServerError): self._ia_client.set_resource({'POLLING_INTERVAL': 20, 'BAD_PARAM': 'bad_val'}) retval = self._ia_client.get_resource(['POLLING_INTERVAL']) log.debug('Retrieved parameters from agent: {0}'.format(retval)) self.assertTrue(isinstance(retval, dict)) self.assertEqual(retval['POLLING_INTERVAL'], 20) cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_initialize(self): # Test agent initialize command. This causes creation of driver process and transition to inactive. # We start in uninitialized state. # In this state there is no driver process. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Initialize the agent. # The agent is spawned with a driver config, but you can pass one in # optinally with the initialize command. This validates the driver # config, launches a driver process and connects to it via messaging. # If successful, we switch to the inactive state. cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) # Reset the agent. This causes the driver messaging to be stopped, # the driver process to end and switches us back to uninitialized. cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_states(self): # Test agent state transitions. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STOPPED) cmd = AgentCommand(command=ResourceAgentEvent.RESUME) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.PAUSE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STOPPED) cmd = AgentCommand(command=ResourceAgentEvent.CLEAR) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=DriverEvent.START_AUTOSAMPLE) self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.STREAMING) cmd = AgentCommand(command=DriverEvent.STOP_AUTOSAMPLE) self._ia_client.execute_resource(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_capabilities(self): """ Test the ability to retrieve agent and resource parameter and command capabilities in various system states. """ # Test the ability to retrieve agent and resource parameter and command capabilities. acmds = self._ia_client.get_capabilities(['AGT_CMD']) log.debug('Agent Commands: {0}'.format(acmds)) # acmds = [item[1] for item in acmds] self.assertListsEqual(acmds, AGT_CMDS.keys()) apars = self._ia_client.get_capabilities(['AGT_PAR']) log.debug('Agent Parameters: {0}'.format(apars)) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) rcmds = self._ia_client.get_capabilities(['RES_CMD']) log.debug('Resource Commands: {0}'.format(rcmds)) # rcmds = [item[1] for item in rcmds] self.assertListsEqual(rcmds, CMDS.keys()) rpars = self._ia_client.get_capabilities(['RES_PAR']) log.debug('Resource Parameters: {0}'.format(rpars)) # rpars = [item[1] for item in rpars] self.assertListsEqual(rpars, PARAMS.keys()) cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) def test_errors(self): # Test illegal behavior and replies. state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED) # Can't go active in unitialized state. # Status 660 is state error. cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) with self.assertRaises(Conflict): self._ia_client.execute_agent(cmd) # Can't command driver in this state. cmd = AgentCommand(command=DriverEvent.ACQUIRE_SAMPLE) with self.assertRaises(Conflict): self._ia_client.execute_resource(cmd) #self.assertEqual(reply.status, 660) cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.INACTIVE) cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.IDLE) cmd = AgentCommand(command=ResourceAgentEvent.RUN) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.COMMAND) # 404 unknown agent command. cmd = AgentCommand(command='kiss_edward') with self.assertRaises(BadRequest): self._ia_client.execute_agent(cmd) # 670 unknown driver command. cmd = AgentCommand(command='acquire_sample_please') with self.assertRaises(ServerError): self._ia_client.execute_resource(cmd) # 630 Parameter error. #self.assertRaises(InstParameterError, self._ia_client.get_param, 'bogus bogus') cmd = AgentCommand(command=ResourceAgentEvent.RESET) self._ia_client.execute_agent(cmd) state = self._ia_client.get_agent_state() self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
class TestDMEnd2End(IonIntegrationTestCase): def setUp(self): # Love the non pep-8 convention self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.process_dispatcher = ProcessDispatcherServiceClient() self.pubsub_management = PubsubManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() self.ingestion_management = IngestionManagementServiceClient() self.data_retriever = DataRetrieverServiceClient() self.pids = [] self.event = Event() self.exchange_space_name = 'test_granules' self.exchange_point_name = 'science_data' self.i = 0 self.purge_queues() self.queue_buffer = [] self.streams = [] self.addCleanup(self.stop_all_ingestion) def purge_queues(self): xn = self.container.ex_manager.create_xn_queue('science_granule_ingestion') xn.purge() def tearDown(self): self.purge_queues() for pid in self.pids: self.container.proc_manager.terminate_process(pid) IngestionManagementIntTest.clean_subscriptions() for queue in self.queue_buffer: if isinstance(queue, ExchangeNameQueue): queue.delete() elif isinstance(queue, str): xn = self.container.ex_manager.create_xn_queue(queue) xn.delete() #-------------------------------------------------------------------------------- # Helper/Utility methods #-------------------------------------------------------------------------------- def create_dataset(self, parameter_dict_id=''): ''' Creates a time-series dataset ''' tdom, sdom = time_series_domain() sdom = sdom.dump() tdom = tdom.dump() if not parameter_dict_id: parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) dataset_id = self.dataset_management.create_dataset('test_dataset_%i'%self.i, parameter_dictionary_id=parameter_dict_id, spatial_domain=sdom, temporal_domain=tdom) return dataset_id def get_datastore(self, dataset_id): ''' Gets an instance of the datastore This method is primarily used to defeat a bug where integration tests in multiple containers may sometimes delete a CouchDB datastore and the other containers are unaware of the new state of the datastore. ''' dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def get_ingestion_config(self): ''' Grab the ingestion configuration from the resource registry ''' # The ingestion configuration should have been created by the bootstrap service # which is configured through r2deploy.yml ingest_configs, _ = self.resource_registry.find_resources(restype=RT.IngestionConfiguration,id_only=True) return ingest_configs[0] def launch_producer(self, stream_id=''): ''' Launch the producer ''' pid = self.container.spawn_process('better_data_producer', 'ion.processes.data.example_data_producer', 'BetterDataProducer', {'process':{'stream_id':stream_id}}) self.pids.append(pid) def make_simple_dataset(self): ''' Makes a stream, a stream definition and a dataset, the essentials for most of these tests ''' pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) stream_def_id = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream('ctd stream %i' % self.i, 'xp1', stream_definition_id=stream_def_id) dataset_id = self.create_dataset(pdict_id) self.get_datastore(dataset_id) self.i += 1 return stream_id, route, stream_def_id, dataset_id def publish_hifi(self,stream_id,stream_route,offset=0): ''' Publish deterministic data ''' pub = StandaloneStreamPublisher(stream_id, stream_route) stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id) stream_def_id = stream_def._id rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) + (offset * 10) rdt['temp'] = np.arange(10) + (offset * 10) pub.publish(rdt.to_granule()) def publish_fake_data(self,stream_id, route): ''' Make four granules ''' for i in xrange(4): self.publish_hifi(stream_id,route,i) def start_ingestion(self, stream_id, dataset_id): ''' Starts ingestion/persistence for a given dataset ''' ingest_config_id = self.get_ingestion_config() self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id) def stop_ingestion(self, stream_id): ingest_config_id = self.get_ingestion_config() self.ingestion_management.unpersist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id) def stop_all_ingestion(self): try: [self.stop_ingestion(sid) for sid in self.streams] except: pass def validate_granule_subscription(self, msg, route, stream_id): ''' Validation for granule format ''' if msg == {}: return rdt = RecordDictionaryTool.load_from_granule(msg) log.info('%s', rdt.pretty_print()) self.assertIsInstance(msg,Granule,'Message is improperly formatted. (%s)' % type(msg)) self.event.set() def wait_until_we_have_enough_granules(self, dataset_id='',data_size=40): ''' Loops until there is a sufficient amount of data in the dataset ''' done = False with gevent.Timeout(40): while not done: extents = self.dataset_management.dataset_extents(dataset_id, 'time')[0] granule = self.data_retriever.retrieve_last_data_points(dataset_id, 1) rdt = RecordDictionaryTool.load_from_granule(granule) if rdt['time'] and rdt['time'][0] != rdt._pdict.get_context('time').fill_value and extents >= data_size: done = True else: gevent.sleep(0.2) #-------------------------------------------------------------------------------- # Test Methods #-------------------------------------------------------------------------------- @attr('SMOKE') def test_dm_end_2_end(self): #-------------------------------------------------------------------------------- # Set up a stream and have a mock instrument (producer) send data #-------------------------------------------------------------------------------- self.event.clear() # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_definition = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream('producer', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition) #-------------------------------------------------------------------------------- # Start persisting the data on the stream # - Get the ingestion configuration from the resource registry # - Create the dataset # - call persist_data_stream to setup the subscription for the ingestion workers # on the stream that you specify which causes the data to be persisted #-------------------------------------------------------------------------------- ingest_config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id) #-------------------------------------------------------------------------------- # Now the granules are ingesting and persisted #-------------------------------------------------------------------------------- self.launch_producer(stream_id) self.wait_until_we_have_enough_granules(dataset_id,40) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_id) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) self.assertTrue((rdt['time'][:10] == np.arange(10)).all(),'%s' % rdt['time'][:]) self.assertTrue((rdt['binary'][:10] == np.array(['hi']*10, dtype='object')).all()) #-------------------------------------------------------------------------------- # Now to try the streamed approach #-------------------------------------------------------------------------------- replay_stream_id, replay_route = self.pubsub_management.create_stream('replay_out', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition) self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream_id) log.info('Process ID: %s', process_id) replay_client = ReplayClient(process_id) #-------------------------------------------------------------------------------- # Create the listening endpoint for the the retriever to talk to #-------------------------------------------------------------------------------- xp = self.container.ex_manager.create_xp(self.exchange_point_name) subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription) self.queue_buffer.append(self.exchange_space_name) subscriber.start() subscriber.xn.bind(replay_route.routing_key, xp) self.data_retriever.start_replay_agent(self.replay_id) self.assertTrue(replay_client.await_agent_ready(5), 'The process never launched') replay_client.start_replay() self.assertTrue(self.event.wait(10)) subscriber.stop() self.data_retriever.cancel_replay_agent(self.replay_id) #-------------------------------------------------------------------------------- # Test the slicing capabilities #-------------------------------------------------------------------------------- granule = self.data_retriever.retrieve(dataset_id=dataset_id, query={'tdoa':slice(0,5)}) rdt = RecordDictionaryTool.load_from_granule(granule) b = rdt['time'] == np.arange(5) self.assertTrue(b.all() if not isinstance(b,bool) else b) self.streams.append(stream_id) self.stop_ingestion(stream_id) def test_coverage_transform(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_parsed() stream_def_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) stream_id, route = self.pubsub_management.create_stream('example', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) self.addCleanup(self.pubsub_management.delete_stream, stream_id) ingestion_config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id) self.addCleanup(self.ingestion_management.unpersist_data_stream, stream_id, ingestion_config_id) publisher = StandaloneStreamPublisher(stream_id, route) rdt = ph.get_rdt(stream_def_id) ph.fill_parsed_rdt(rdt) dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_monitor.event.wait(30)) replay_granule = self.data_retriever.retrieve(dataset_id) rdt_out = RecordDictionaryTool.load_from_granule(replay_granule) np.testing.assert_array_almost_equal(rdt_out['time'], rdt['time']) np.testing.assert_array_almost_equal(rdt_out['temp'], rdt['temp']) np.testing.assert_array_almost_equal(rdt_out['conductivity_L1'], np.array([42.914])) np.testing.assert_array_almost_equal(rdt_out['temp_L1'], np.array([20.])) np.testing.assert_array_almost_equal(rdt_out['pressure_L1'], np.array([3.068])) np.testing.assert_array_almost_equal(rdt_out['density'], np.array([1021.7144739593881])) np.testing.assert_array_almost_equal(rdt_out['salinity'], np.array([30.935132729668283])) def test_qc_events(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_qc_pdict() stream_def_id = self.pubsub_management.create_stream_definition('qc stream def', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) stream_id, route = self.pubsub_management.create_stream('qc stream', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) self.addCleanup(self.pubsub_management.delete_stream, stream_id) ingestion_config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) config = DotDict() self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id, config=config) self.addCleanup(self.ingestion_management.unpersist_data_stream, stream_id, ingestion_config_id) publisher = StandaloneStreamPublisher(stream_id, route) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) rdt['temp'] = np.arange(10) * 3 verified = Event() def verification(event, *args, **kwargs): self.assertEquals(event.qc_parameter, 'temp_qc') self.assertEquals(event.temporal_value, 7) verified.set() es = EventSubscriber(event_type=OT.ParameterQCEvent, origin=dataset_id, callback=verification, auto_delete=True) es.start() self.addCleanup(es.stop) publisher.publish(rdt.to_granule()) self.assertTrue(verified.wait(10)) def test_lookup_values_ingest_replay(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_lookups() stream_def_id = self.pubsub_management.create_stream_definition('lookups', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) stream_id, route = self.pubsub_management.create_stream('example', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) self.addCleanup(self.pubsub_management.delete_stream, stream_id) ingestion_config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) config = DotDict() config.process.lookup_docs = ['test1', 'test2'] self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id, config=config) self.addCleanup(self.ingestion_management.unpersist_data_stream, stream_id, ingestion_config_id) stored_value_manager = StoredValueManager(self.container) stored_value_manager.stored_value_cas('test1',{'offset_a':10.0, 'offset_b':13.1}) publisher = StandaloneStreamPublisher(stream_id, route) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(20) rdt['temp'] = [20.0] * 20 granule = rdt.to_granule() dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) publisher.publish(granule) self.assertTrue(dataset_monitor.event.wait(30)) replay_granule = self.data_retriever.retrieve(dataset_id) rdt_out = RecordDictionaryTool.load_from_granule(replay_granule) np.testing.assert_array_almost_equal(rdt_out['time'], np.arange(20)) np.testing.assert_array_almost_equal(rdt_out['temp'], np.array([20.] * 20)) np.testing.assert_array_almost_equal(rdt_out['calibrated'], np.array([30.]*20)) np.testing.assert_array_equal(rdt_out['offset_b'], np.array([rdt_out.fill_value('offset_b')] * 20)) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(20,40) rdt['temp'] = [20.0] * 20 granule = rdt.to_granule() dataset_monitor.event.clear() stored_value_manager.stored_value_cas('test1',{'offset_a':20.0}) stored_value_manager.stored_value_cas('coefficient_document',{'offset_b':10.0}) gevent.sleep(2) publisher.publish(granule) self.assertTrue(dataset_monitor.event.wait(30)) replay_granule = self.data_retriever.retrieve(dataset_id) rdt_out = RecordDictionaryTool.load_from_granule(replay_granule) np.testing.assert_array_almost_equal(rdt_out['time'], np.arange(40)) np.testing.assert_array_almost_equal(rdt_out['temp'], np.array([20.] * 20 + [20.] * 20)) np.testing.assert_array_equal(rdt_out['offset_b'], np.array([10.] * 40)) np.testing.assert_array_almost_equal(rdt_out['calibrated'], np.array([30.]*20 + [40.]*20)) np.testing.assert_array_almost_equal(rdt_out['calibrated_b'], np.array([40.] * 20 + [50.] * 20)) @unittest.skip('Doesnt work') @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_replay_pause(self): # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id) replay_stream, replay_route = self.pubsub_management.create_stream('replay', 'xp1', stream_definition_id=stream_def_id) dataset_id = self.create_dataset(pdict_id) scov = DatasetManagementService._get_simplex_coverage(dataset_id) bb = CoverageCraft(scov) bb.rdt['time'] = np.arange(100) bb.rdt['temp'] = np.random.random(100) + 30 bb.sync_with_granule() DatasetManagementService._persist_coverage(dataset_id, bb.coverage) # This invalidates it for multi-host configurations # Set up the subscriber to verify the data subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription) xp = self.container.ex_manager.create_xp('xp1') self.queue_buffer.append(self.exchange_space_name) subscriber.start() subscriber.xn.bind(replay_route.routing_key, xp) # Set up the replay agent and the client wrapper # 1) Define the Replay (dataset and stream to publish on) self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream) # 2) Make a client to the interact with the process (optionall provide it a process to bind with) replay_client = ReplayClient(process_id) # 3) Start the agent (launch the process) self.data_retriever.start_replay_agent(self.replay_id) # 4) Start replaying... replay_client.start_replay() # Wait till we get some granules self.assertTrue(self.event.wait(5)) # We got granules, pause the replay, clear the queue and allow the process to finish consuming replay_client.pause_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure there's no remaining messages being consumed self.assertFalse(self.event.wait(1)) # Resume the replay and wait until we start getting granules again replay_client.resume_replay() self.assertTrue(self.event.wait(5)) # Stop the replay, clear the queues replay_client.stop_replay() gevent.sleep(1) subscriber.xn.purge() self.event.clear() # Make sure that it did indeed stop self.assertFalse(self.event.wait(1)) subscriber.stop() def test_retrieve_and_transform(self): # Make a simple dataset and start ingestion, pretty standard stuff. ctd_stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() self.start_ingestion(ctd_stream_id, dataset_id) # Stream definition for the salinity data salinity_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) sal_stream_def_id = self.pubsub_management.create_stream_definition('sal data', parameter_dictionary_id=salinity_pdict_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) rdt['temp'] = np.random.randn(10) * 10 + 30 rdt['conductivity'] = np.random.randn(10) * 2 + 10 rdt['pressure'] = np.random.randn(10) * 1 + 12 publisher = StandaloneStreamPublisher(ctd_stream_id, route) publisher.publish(rdt.to_granule()) rdt['time'] = np.arange(10,20) publisher.publish(rdt.to_granule()) self.wait_until_we_have_enough_granules(dataset_id, 20) granule = self.data_retriever.retrieve(dataset_id, None, None, 'ion.processes.data.transforms.ctd.ctd_L2_salinity', 'CTDL2SalinityTransformAlgorithm', kwargs=dict(params=sal_stream_def_id)) rdt = RecordDictionaryTool.load_from_granule(granule) for i in rdt['salinity']: self.assertNotEquals(i,0) self.streams.append(ctd_stream_id) self.stop_ingestion(ctd_stream_id) def test_last_granule(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() self.start_ingestion(stream_id, dataset_id) self.publish_hifi(stream_id,route, 0) self.publish_hifi(stream_id,route, 1) self.wait_until_we_have_enough_granules(dataset_id,20) # I just need two success = False def verifier(): replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id, 10) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(10) + 10 if not isinstance(comp,bool): return comp.all() return False success = poll(verifier) self.assertTrue(success) success = False def verify_points(): replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id,5) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(15,20) if not isinstance(comp,bool): return comp.all() return False success = poll(verify_points) self.assertTrue(success) self.streams.append(stream_id) self.stop_ingestion(stream_id) def test_replay_with_parameters(self): #-------------------------------------------------------------------------------- # Create the configurations and the dataset #-------------------------------------------------------------------------------- # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream('replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id) dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) self.publish_fake_data(stream_id, route) self.assertTrue(dataset_monitor.event.wait(30)) query = { 'start_time': 0 - 2208988800, 'end_time': 20 - 2208988800, 'stride_time' : 2, 'parameters': ['time','temp'] } retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id,query=query) rdt = RecordDictionaryTool.load_from_granule(retrieved_data) comp = np.arange(0,20,2) == rdt['time'] self.assertTrue(comp.all(),'%s' % rdt.pretty_print()) self.assertEquals(set(rdt.iterkeys()), set(['time','temp'])) extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=['time','temp']) self.assertTrue(extents['time']>=20) self.assertTrue(extents['temp']>=20) self.streams.append(stream_id) self.stop_ingestion(stream_id) def test_repersist_data(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() self.start_ingestion(stream_id, dataset_id) self.publish_hifi(stream_id,route,0) self.publish_hifi(stream_id,route,1) self.wait_until_we_have_enough_granules(dataset_id,20) config_id = self.get_ingestion_config() self.ingestion_management.unpersist_data_stream(stream_id=stream_id,ingestion_configuration_id=config_id) self.ingestion_management.persist_data_stream(stream_id=stream_id,ingestion_configuration_id=config_id,dataset_id=dataset_id) self.publish_hifi(stream_id,route,2) self.publish_hifi(stream_id,route,3) self.wait_until_we_have_enough_granules(dataset_id,40) success = False with gevent.timeout.Timeout(5): while not success: replay_granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(0,40) if not isinstance(comp,bool): success = comp.all() gevent.sleep(1) self.assertTrue(success) self.streams.append(stream_id) self.stop_ingestion(stream_id) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.') def test_correct_time(self): # There are 2208988800 seconds between Jan 1 1900 and Jan 1 1970, i.e. # the conversion factor between unix and NTP time unix_now = np.floor(time.time()) ntp_now = unix_now + 2208988800 unix_ago = unix_now - 20 ntp_ago = unix_ago + 2208988800 stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() coverage = DatasetManagementService._get_simplex_coverage(dataset_id) coverage.insert_timesteps(20) coverage.set_parameter_values('time', np.arange(ntp_ago,ntp_now)) temporal_bounds = self.dataset_management.dataset_temporal_bounds(dataset_id) self.assertTrue( np.abs(temporal_bounds[0] - unix_ago) < 2) self.assertTrue( np.abs(temporal_bounds[1] - unix_now) < 2) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.') def test_empty_coverage_time(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() coverage = DatasetManagementService._get_coverage(dataset_id) temporal_bounds = self.dataset_management.dataset_temporal_bounds(dataset_id) self.assertEquals([coverage.get_parameter_context('time').fill_value] *2, temporal_bounds) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.') def test_out_of_band_retrieve(self): # Setup the environemnt stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() self.start_ingestion(stream_id, dataset_id) # Fill the dataset self.publish_fake_data(stream_id, route) self.wait_until_we_have_enough_granules(dataset_id,40) # Retrieve the data granule = DataRetrieverService.retrieve_oob(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) self.assertTrue((rdt['time'] == np.arange(40)).all()) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.') def test_retrieve_cache(self): DataRetrieverService._refresh_interval = 1 datasets = [self.make_simple_dataset() for i in xrange(10)] for stream_id, route, stream_def_id, dataset_id in datasets: coverage = DatasetManagementService._get_simplex_coverage(dataset_id) coverage.insert_timesteps(10) coverage.set_parameter_values('time', np.arange(10)) coverage.set_parameter_values('temp', np.arange(10)) # Verify cache hit and refresh dataset_ids = [i[3] for i in datasets] self.assertTrue(dataset_ids[0] not in DataRetrieverService._retrieve_cache) DataRetrieverService._get_coverage(dataset_ids[0]) # Hit the chache cov, age = DataRetrieverService._retrieve_cache[dataset_ids[0]] # Verify that it was hit and it's now in there self.assertTrue(dataset_ids[0] in DataRetrieverService._retrieve_cache) gevent.sleep(DataRetrieverService._refresh_interval + 0.2) DataRetrieverService._get_coverage(dataset_ids[0]) # Hit the chache cov, age2 = DataRetrieverService._retrieve_cache[dataset_ids[0]] self.assertTrue(age2 != age) for dataset_id in dataset_ids: DataRetrieverService._get_coverage(dataset_id) self.assertTrue(dataset_ids[0] not in DataRetrieverService._retrieve_cache) stream_id, route, stream_def, dataset_id = datasets[0] self.start_ingestion(stream_id, dataset_id) DataRetrieverService._get_coverage(dataset_id) self.assertTrue(dataset_id in DataRetrieverService._retrieve_cache) DataRetrieverService._refresh_interval = 100 self.publish_hifi(stream_id,route,1) self.wait_until_we_have_enough_granules(dataset_id, data_size=20) event = gevent.event.Event() with gevent.Timeout(20): while not event.wait(0.1): if dataset_id not in DataRetrieverService._retrieve_cache: event.set() self.assertTrue(event.is_set()) def publish_and_wait(self, dataset_id, granule): stream_ids, _ = self.resource_registry.find_objects(dataset_id, PRED.hasStream,id_only=True) stream_id=stream_ids[0] route = self.pubsub_management.read_stream_route(stream_id) publisher = StandaloneStreamPublisher(stream_id,route) dataset_monitor = DatasetMonitor(dataset_id) publisher.publish(granule) self.assertTrue(dataset_monitor.event.wait(10)) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.') def test_thorough_gap_analysis(self): dataset_id = self.test_ingestion_gap_analysis() vcov = DatasetManagementService._get_coverage(dataset_id) self.assertIsInstance(vcov,ViewCoverage) ccov = vcov.reference_coverage self.assertIsInstance(ccov, ComplexCoverage) self.assertEquals(len(ccov._reference_covs), 3) def test_ingestion_gap_analysis(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() self.start_ingestion(stream_id, dataset_id) self.addCleanup(self.stop_ingestion, stream_id) connection1 = uuid4().hex connection2 = uuid4().hex rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = [0] rdt['temp'] = [0] self.publish_and_wait(dataset_id, rdt.to_granule(connection_id=connection1,connection_index='0')) rdt['time'] = [1] rdt['temp'] = [1] self.publish_and_wait(dataset_id, rdt.to_granule(connection_id=connection1,connection_index=1)) rdt['time'] = [2] rdt['temp'] = [2] self.publish_and_wait(dataset_id, rdt.to_granule(connection_id=connection1,connection_index='3')) # Gap, missed message rdt['time'] = [3] rdt['temp'] = [3] self.publish_and_wait(dataset_id, rdt.to_granule(connection_id=connection2,connection_index='3')) # Gap, new connection rdt['time'] = [4] rdt['temp'] = [4] self.publish_and_wait(dataset_id, rdt.to_granule(connection_id=connection2,connection_index='4')) rdt['time'] = [5] rdt['temp'] = [5] self.publish_and_wait(dataset_id, rdt.to_granule(connection_id=connection2,connection_index=5)) granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_equal(rdt['time'], np.arange(6)) np.testing.assert_array_equal(rdt['temp'], np.arange(6)) return dataset_id @unittest.skip('Outdated due to ingestion retry') @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.') def test_ingestion_failover(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() self.start_ingestion(stream_id, dataset_id) event = Event() def cb(*args, **kwargs): event.set() sub = EventSubscriber(event_type="ExceptionEvent", callback=cb, origin="stream_exception") sub.start() self.publish_fake_data(stream_id, route) self.wait_until_we_have_enough_granules(dataset_id, 40) file_path = DatasetManagementService._get_coverage_path(dataset_id) master_file = os.path.join(file_path, '%s_master.hdf5' % dataset_id) with open(master_file, 'w') as f: f.write('this will crash HDF') self.publish_hifi(stream_id, route, 5) self.assertTrue(event.wait(10)) sub.stop() @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.') def test_coverage_types(self): # Make a simple dataset and start ingestion, pretty standard stuff. ctd_stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() cov = DatasetManagementService._get_coverage(dataset_id=dataset_id) self.assertIsInstance(cov, ViewCoverage) cov = DatasetManagementService._get_simplex_coverage(dataset_id=dataset_id) self.assertIsInstance(cov, SimplexCoverage)
class TestParticleConversion(IonIntegrationTestCase): """ Test cases to confirm all data particles convert to granules successfully. """ ############################################################################ # Setup, teardown. ############################################################################ def setUp(self): # Start container. log.info('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a pubsub client to create streams. self.pubsub_client = PubsubManagementServiceClient(node=self.container.node) self.dataset_management = DatasetManagementServiceClient() def create_granule(self, stream_name, param_dict_name, particle_list): pd_id = self.dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True) stream_def_id = self.pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id) stream_def = self.pubsub_client.read_stream_definition(stream_def_id) rdt = RecordDictionaryTool(stream_definition=stream_def) rdt = populate_rdt(rdt, particle_list) log.trace("RDT: %s", str(rdt)) g = rdt.to_granule(data_producer_id='fake_agent_id') return g def test_sbe37_particles(self): """ test_particles """ stream_name = 'parsed' param_dict_name = 'ctd_parsed_param_dict' particle_list = [{u'quality_flag': u'ok', u'preferred_timestamp': u'port_timestamp', u'stream_name': u'parsed', u'port_timestamp': 3578927139.3578925, u'pkt_format_id': u'JSON_Data', u'pkt_version': 1, u'values': [{u'value_id': u'temp', u'value': 68.5895}, {u'value_id': u'conductivity', u'value': 26.72304}, {u'value_id': u'pressure', u'value': 733.303}], u'driver_timestamp': 3578927139.4226017}] try: g = self.create_granule(stream_name, param_dict_name, particle_list) self.assert_granule_time(g, particle_list[0]['port_timestamp']) except Exception as e: errmsg = 'Granule creation failed: %s' % str(e) errmsg += '\n stream_name: ' + stream_name errmsg += '\n param_dict_name: ' + param_dict_name errmsg += '\n particle list: %s' % str(particle_list) self.fail(errmsg) stream_name = 'raw' param_dict_name = 'ctd_raw_param_dict' particle_list = [{u'quality_flag': u'ok', u'preferred_timestamp': u'port_timestamp', u'stream_name': u'raw', u'port_timestamp': 3578927113.3578925, u'pkt_format_id': u'JSON_Data', u'pkt_version': 1, u'values': [{u'binary': True, u'value_id': u'raw', u'value': u'ZAA='}, {u'value_id': u'length', u'value': 2}, {u'value_id': u'type', u'value': 1}, {u'value_id': u'checksum', u'value': None}], u'driver_timestamp': 3578927113.75216}] try: g = self.create_granule(stream_name, param_dict_name, particle_list) self.assert_granule_time(g, particle_list[0]['port_timestamp']) except Exception as e: errmsg = 'Granule creation failed: %s' % str(e) errmsg += '\n stream_name: ' + stream_name errmsg += '\n param_dict_name: ' + param_dict_name errmsg += '\n particle list: %s' % str(particle_list) self.fail(errmsg) def test_internal_time_particle(self): """ Test a particle that has an internal time listed for its preferred time """ stream_name = 'parsed' param_dict_name = 'ctd_parsed_param_dict' particle_list = [{u'quality_flag': u'ok', u'preferred_timestamp': u'internal_timestamp', u'stream_name': u'parsed', u'port_timestamp': 3578927139.3578925, u'internal_timestamp': 3578927039.3178925, u'pkt_format_id': u'JSON_Data', u'pkt_version': 1, u'values': [{u'value_id': u'temp', u'value': 68.5895}, {u'value_id': u'conductivity', u'value': 26.72304}, {u'value_id': u'pressure', u'value': 733.303}], u'driver_timestamp': 3578927139.4226017}] try: g = self.create_granule(stream_name, param_dict_name, particle_list) self.assert_granule_time(g, particle_list[0]['internal_timestamp']) except Exception as e: errmsg = 'Granule creation failed: %s' % str(e) errmsg += '\n stream_name: ' + stream_name errmsg += '\n param_dict_name: ' + param_dict_name errmsg += '\n particle list: %s' % str(particle_list) self.fail(errmsg) def test_driver_time_particle(self): """ Test a particle that has a driver time listed for its preferred time """ stream_name = 'parsed' param_dict_name = 'ctd_parsed_param_dict' particle_list = [{u'quality_flag': u'ok', u'preferred_timestamp': u'driver_timestamp', u'stream_name': u'parsed', u'port_timestamp': 3578927139.3578925, u'internal_timestamp': 3578927039.3178925, u'pkt_format_id': u'JSON_Data', u'pkt_version': 1, u'values': [{u'value_id': u'temp', u'value': 68.5895}, {u'value_id': u'conductivity', u'value': 26.72304}, {u'value_id': u'pressure', u'value': 733.303}], u'driver_timestamp': 3578927139.4226017}] try: g = self.create_granule(stream_name, param_dict_name, particle_list) self.assert_granule_time(g, particle_list[0]['driver_timestamp']) except Exception as e: errmsg = 'Granule creation failed: %s' % str(e) errmsg += '\n stream_name: ' + stream_name errmsg += '\n param_dict_name: ' + param_dict_name errmsg += '\n particle list: %s' % str(particle_list) self.fail(errmsg) def test_vel3d_particles(self): """ test_particles """ stream_name = 'vel3d_parsed' param_dict_name = 'vel3d_b_sample' particle_list = [ { "driver_timestamp": 3579022766.361967, "internal_timestamp": 3579047922.0, "pkt_format_id": "JSON_Data", "pkt_version": 1, "port_timestamp": 3579022762.357902, "preferred_timestamp": "port_timestamp", "quality_flag": "ok", "stream_name": "vel3d_b_sample", "values": [ {"value": 3579047922.0, "value_id": "date_time_string"}, {"value": 5, "value_id": "fractional_second"}, {"value": "8000", "value_id": "velocity_beam_a"}, {"value": "8000", "value_id": "velocity_beam_b"}, {"value": "8000", "value_id": "velocity_beam_c"}, {"value": "8000", "value_id": "velocity_beam_d"}, {"value": 999.0, "value_id": "turbulent_velocity_east"}, {"value": 999.0, "value_id": "turbulent_velocity_north"}, {"value": 999.0, "value_id": "turbulent_velocity_up"}, {"value": 2.16, "value_id": "temperature"}, {"value": 1.0, "value_id": "mag_comp_x"}, {"value": -0.0, "value_id": "mag_comp_y"}, {"value": -7.9, "value_id": "pitch"}, {"value": -78.2, "value_id": "roll"}] } ] class RDT(dict): def __init__(self): super(RDT, self).__init__() self.temporal_parameter = None rdt = RDT() for x in particle_list[0]['values']: rdt[x['value_id']] = None rdt = populate_rdt(rdt, particle_list) def assert_granule_time(self, granule, target_time): """ Assert that the granule's time matches the target time @param granule The granule to be searched @param target_time The time that should match the granule's overall time """ rdt = RecordDictionaryTool.load_from_granule(granule) rdt_time = rdt['time'][0] log.debug("assert_granule_time granule time: %s", rdt_time) log.debug("assert_granule_time target timestamp: %s", target_time) self.assertEqual(rdt_time, target_time)
class TestDataProcessManagementPrime(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dataset_management = DatasetManagementServiceClient() self.resource_registry = self.container.resource_registry self.pubsub_management = PubsubManagementServiceClient() self.data_process_management = DataProcessManagementServiceClient() self.data_product_management = DataProductManagementServiceClient() self.validators = 0 def lc_preload(self): config = DotDict() config.op = 'load' config.scenario = 'BASE,LC_TEST' config.categories = 'ParameterFunctions,ParameterDefs,ParameterDictionary' config.path = 'res/preload/r2_ioc' self.container.spawn_process('preload','ion.processes.bootstrap.ion_loader','IONLoader', config) def ctd_plain_input_data_product(self): available_fields = [ 'internal_timestamp', 'temp', 'preferred_timestamp', 'time', 'port_timestamp', 'quality_flag', 'lat', 'conductivity', 'driver_timestamp', 'lon', 'pressure'] return self.make_data_product('ctd_parsed_param_dict', 'ctd plain test', available_fields) def ctd_plain_salinity(self): available_fields = [ 'internal_timestamp', 'preferred_timestamp', 'time', 'port_timestamp', 'quality_flag', 'lat', 'driver_timestamp', 'lon', 'salinity'] return self.make_data_product('ctd_parsed_param_dict', 'salinity', available_fields) def ctd_plain_density(self): available_fields = [ 'internal_timestamp', 'preferred_timestamp', 'time', 'port_timestamp', 'quality_flag', 'lat', 'driver_timestamp', 'lon', 'density'] return self.make_data_product('ctd_parsed_param_dict', 'density', available_fields) def ctd_instrument_data_product(self): available_fields = [ 'internal_timestamp', 'temp', 'preferred_timestamp', 'time', 'port_timestamp', 'quality_flag', 'lat', 'conductivity', 'driver_timestamp', 'lon', 'pressure'] return self.make_data_product('ctd_LC_TEST', 'ctd instrument', available_fields) def make_data_product(self, pdict_name, dp_name, available_fields=[]): pdict_id = self.dataset_management.read_parameter_dictionary_by_name(pdict_name, id_only=True) stream_def_id = self.pubsub_management.create_stream_definition('%s stream_def' % dp_name, parameter_dictionary_id=pdict_id, available_fields=available_fields or None) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dp_obj = DataProduct(name=dp_name) dp_obj.temporal_domain = tdom dp_obj.spatial_domain = sdom data_product_id = self.data_product_management.create_data_product(dp_obj, stream_definition_id=stream_def_id) self.addCleanup(self.data_product_management.delete_data_product, data_product_id) return data_product_id def google_dt_data_product(self): return self.make_data_product('google_dt', 'visual') def ctd_derived_data_product(self): return self.make_data_product('ctd_LC_TEST', 'ctd derived products') def publish_to_plain_data_product(self, data_product_id): stream_ids, _ = self.resource_registry.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True) self.assertTrue(len(stream_ids)) stream_id = stream_ids.pop() route = self.pubsub_management.read_stream_route(stream_id) stream_definition = self.pubsub_management.read_stream_definition(stream_id=stream_id) stream_def_id = stream_definition._id publisher = StandaloneStreamPublisher(stream_id, route) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) now = time.time() ntp_now = now + 2208988800 # Do not use in production, this is a loose translation rdt['internal_timestamp'] = [ntp_now] rdt['temp'] = [20.0] rdt['preferred_timestamp'] = ['driver_timestamp'] rdt['time'] = [ntp_now] rdt['port_timestamp'] = [ntp_now] rdt['quality_flag'] = [None] rdt['lat'] = [45] rdt['conductivity'] = [4.2914] rdt['driver_timestamp'] = [ntp_now] rdt['lon'] = [-71] rdt['pressure'] = [3.068] granule = rdt.to_granule() publisher.publish(granule) def publish_to_data_product(self, data_product_id): stream_ids, _ = self.resource_registry.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True) self.assertTrue(len(stream_ids)) stream_id = stream_ids.pop() route = self.pubsub_management.read_stream_route(stream_id) stream_definition = self.pubsub_management.read_stream_definition(stream_id=stream_id) stream_def_id = stream_definition._id publisher = StandaloneStreamPublisher(stream_id, route) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) now = time.time() ntp_now = now + 2208988800 # Do not use in production, this is a loose translation rdt['internal_timestamp'] = [ntp_now] rdt['temp'] = [300000] rdt['preferred_timestamp'] = ['driver_timestamp'] rdt['time'] = [ntp_now] rdt['port_timestamp'] = [ntp_now] rdt['quality_flag'] = [None] rdt['lat'] = [45] rdt['conductivity'] = [4341400] rdt['driver_timestamp'] = [ntp_now] rdt['lon'] = [-71] rdt['pressure'] = [256.8] granule = rdt.to_granule() publisher.publish(granule) def setup_subscriber(self, data_product_id, callback): stream_ids, _ = self.resource_registry.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True) self.assertTrue(len(stream_ids)) stream_id = stream_ids.pop() sub_id = self.pubsub_management.create_subscription('validator_%s'%self.validators, stream_ids=[stream_id]) self.addCleanup(self.pubsub_management.delete_subscription, sub_id) self.pubsub_management.activate_subscription(sub_id) self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id) subscriber = StandaloneStreamSubscriber('validator_%s' % self.validators, callback=callback) subscriber.start() self.addCleanup(subscriber.stop) self.validators+=1 return subscriber def create_density_transform_function(self): tf = TransformFunction(name='ctdbp_l2_density', module='ion.processes.data.transforms.ctdbp.ctdbp_L2_density', cls='CTDBP_DensityTransformAlgorithm') tf_id = self.data_process_management.create_transform_function(tf) self.addCleanup(self.data_process_management.delete_transform_function, tf_id) return tf_id def create_salinity_transform_function(self): tf = TransformFunction(name='ctdbp_l2_salinity', module='ion.processes.data.transforms.ctdbp.ctdbp_L2_salinity', cls='CTDBP_SalinityTransformAlgorithm') tf_id = self.data_process_management.create_transform_function(tf) self.addCleanup(self.data_process_management.delete_transform_function, tf_id) return tf_id @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_data_process_prime(self): self.lc_preload() instrument_data_product_id = self.ctd_instrument_data_product() derived_data_product_id = self.ctd_derived_data_product() data_process_id = self.data_process_management.create_data_process2(in_data_product_ids=[instrument_data_product_id], out_data_product_ids=[derived_data_product_id]) self.addCleanup(self.data_process_management.delete_data_process2, data_process_id) self.data_process_management.activate_data_process2(data_process_id) self.addCleanup(self.data_process_management.deactivate_data_process2, data_process_id) validated = Event() def validation(msg, route, stream_id): rdt = RecordDictionaryTool.load_from_granule(msg) np.testing.assert_array_almost_equal(rdt['conductivity_L1'], np.array([42.914])) np.testing.assert_array_almost_equal(rdt['temp_L1'], np.array([20.])) np.testing.assert_array_almost_equal(rdt['pressure_L1'], np.array([3.068])) np.testing.assert_array_almost_equal(rdt['density'], np.array([1021.7144739593881])) np.testing.assert_array_almost_equal(rdt['salinity'], np.array([30.935132729668283])) validated.set() self.setup_subscriber(derived_data_product_id, callback=validation) self.publish_to_data_product(instrument_data_product_id) self.assertTrue(validated.wait(10)) def test_older_transform(self): input_data_product_id = self.ctd_plain_input_data_product() conductivity_data_product_id = self.make_data_product('ctd_parsed_param_dict', 'conductivity_product', ['time', 'conductivity']) conductivity_stream_def_id = self.get_named_stream_def('conductivity_product stream_def') temperature_data_product_id = self.make_data_product('ctd_parsed_param_dict', 'temperature_product', ['time', 'temp']) temperature_stream_def_id = self.get_named_stream_def('temperature_product stream_def') pressure_data_product_id = self.make_data_product('ctd_parsed_param_dict', 'pressure_product', ['time', 'pressure']) pressure_stream_def_id = self.get_named_stream_def('pressure_product stream_def') dpd = DataProcessDefinition(name='ctdL0') dpd.data_process_type = DataProcessTypeEnum.TRANSFORM dpd.module = 'ion.processes.data.transforms.ctd.ctd_L0_all' dpd.class_name = 'ctd_L0_all' data_process_definition_id = self.data_process_management.create_data_process_definition(dpd) self.addCleanup(self.data_process_management.delete_data_process_definition, data_process_definition_id) self.data_process_management.assign_stream_definition_to_data_process_definition(conductivity_stream_def_id, data_process_definition_id, binding='conductivity') self.data_process_management.assign_stream_definition_to_data_process_definition(temperature_stream_def_id, data_process_definition_id, binding='temperature') self.data_process_management.assign_stream_definition_to_data_process_definition(pressure_stream_def_id, data_process_definition_id, binding='pressure') data_process_id = self.data_process_management.create_data_process2(data_process_definition_id=data_process_definition_id, in_data_product_ids=[input_data_product_id], out_data_product_ids=[conductivity_data_product_id, temperature_data_product_id, pressure_data_product_id]) self.addCleanup(self.data_process_management.delete_data_process2, data_process_id) self.data_process_management.activate_data_process2(data_process_id) self.addCleanup(self.data_process_management.deactivate_data_process2, data_process_id) conductivity_validated = Event() def validate_conductivity(msg, route, stream_id): rdt = RecordDictionaryTool.load_from_granule(msg) np.testing.assert_array_almost_equal(rdt['conductivity'], np.array([4.2914])) conductivity_validated.set() self.setup_subscriber(conductivity_data_product_id, callback=validate_conductivity) temperature_validated = Event() def validate_temperature(msg, route, stream_id): rdt = RecordDictionaryTool.load_from_granule(msg) np.testing.assert_array_almost_equal(rdt['temp'], np.array([20.0])) temperature_validated.set() self.setup_subscriber(temperature_data_product_id, callback=validate_temperature) pressure_validated = Event() def validate_pressure(msg, route, stream_id): rdt = RecordDictionaryTool.load_from_granule(msg) np.testing.assert_array_almost_equal(rdt['pressure'], np.array([3.068])) pressure_validated.set() self.setup_subscriber(pressure_data_product_id, callback=validate_pressure) self.publish_to_plain_data_product(input_data_product_id) self.assertTrue(conductivity_validated.wait(10)) self.assertTrue(temperature_validated.wait(10)) self.assertTrue(pressure_validated.wait(10)) def get_named_stream_def(self, name): stream_def_ids, _ = self.resource_registry.find_resources(name=name, restype=RT.StreamDefinition, id_only=True) return stream_def_ids[0] def test_actors(self): input_data_product_id = self.ctd_plain_input_data_product() output_data_product_id = self.ctd_plain_density() actor = self.create_density_transform_function() route = {input_data_product_id: {output_data_product_id: actor}} config = DotDict() config.process.routes = route config.process.params.lat = 45. config.process.params.lon = -71. data_process_id = self.data_process_management.create_data_process2(in_data_product_ids=[input_data_product_id], out_data_product_ids=[output_data_product_id], configuration=config) self.addCleanup(self.data_process_management.delete_data_process2, data_process_id) self.data_process_management.activate_data_process2(data_process_id) self.addCleanup(self.data_process_management.deactivate_data_process2, data_process_id) validated = Event() def validation(msg, route, stream_id): rdt = RecordDictionaryTool.load_from_granule(msg) # The value I use is a double, the value coming back is only a float32 so there's some data loss but it should be precise to the 4th digit np.testing.assert_array_almost_equal(rdt['density'], np.array([1021.6839775385847]), decimal=4) validated.set() self.setup_subscriber(output_data_product_id, callback=validation) self.publish_to_plain_data_product(input_data_product_id) self.assertTrue(validated.wait(10)) def test_multi_in_out(self): input1 = self.ctd_plain_input_data_product() input2 = self.make_data_product('ctd_parsed_param_dict', 'input2') density_dp_id = self.ctd_plain_density() salinity_dp_id = self.ctd_plain_salinity() density_actor = self.create_density_transform_function() salinity_actor = self.create_salinity_transform_function() routes = { input1 : { density_dp_id : density_actor, salinity_dp_id : salinity_actor }, input2 : { density_dp_id : density_actor } } config = DotDict() config.process.routes = routes config.process.params.lat = 45. config.process.params.lon = -71. data_process_id = self.data_process_management.create_data_process2(in_data_product_ids=[input1, input2], out_data_product_ids=[density_dp_id, salinity_dp_id], configuration=config) self.addCleanup(self.data_process_management.delete_data_process2, data_process_id) self.data_process_management.activate_data_process2(data_process_id) self.addCleanup(self.data_process_management.deactivate_data_process2, data_process_id) density_validated = Event() salinity_validated = Event() def density_validation(msg, route, stream_id): rdt = RecordDictionaryTool.load_from_granule(msg) np.testing.assert_array_almost_equal(rdt['density'], np.array([1021.6839775385847]), decimal=4) density_validated.set() def salinity_validation(msg, route, stream_id): rdt = RecordDictionaryTool.load_from_granule(msg) np.testing.assert_array_almost_equal(rdt['salinity'], np.array([30.93513240786831]), decimal=4) salinity_validated.set() self.setup_subscriber(density_dp_id, callback=density_validation) self.setup_subscriber(salinity_dp_id, callback=salinity_validation) self.publish_to_plain_data_product(input1) self.assertTrue(density_validated.wait(10)) self.assertTrue(salinity_validated.wait(10)) density_validated.clear() salinity_validated.clear() self.publish_to_plain_data_product(input2) self.assertTrue(density_validated.wait(10)) self.assertFalse(salinity_validated.wait(0.75)) density_validated.clear() salinity_validated.clear() def test_visual_transform(self): input_data_product_id = self.ctd_plain_input_data_product() output_data_product_id = self.google_dt_data_product() dpd = DataProcessDefinition(name='visual transform') dpd.data_process_type = DataProcessTypeEnum.TRANSFORM dpd.module = 'ion.processes.data.transforms.viz.google_dt' dpd.class_name = 'VizTransformGoogleDT' #-------------------------------------------------------------------------------- # Walk before we base jump #-------------------------------------------------------------------------------- data_process_definition_id = self.data_process_management.create_data_process_definition(dpd) self.addCleanup(self.data_process_management.delete_data_process_definition, data_process_definition_id) data_process_id = self.data_process_management.create_data_process2(data_process_definition_id=data_process_definition_id, in_data_product_ids=[input_data_product_id], out_data_product_ids=[output_data_product_id]) self.addCleanup(self.data_process_management.delete_data_process2,data_process_id) self.data_process_management.activate_data_process2(data_process_id) self.addCleanup(self.data_process_management.deactivate_data_process2, data_process_id) validated = Event() def validation(msg, route, stream_id): rdt = RecordDictionaryTool.load_from_granule(msg) self.assertTrue(rdt['google_dt_components'] is not None) validated.set() self.setup_subscriber(output_data_product_id, callback=validation) self.publish_to_plain_data_product(input_data_product_id) self.assertTrue(validated.wait(10))
class PubsubManagementIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.pubsub_management = PubsubManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() self.queue_cleanup = list() self.exchange_cleanup = list() def tearDown(self): for queue in self.queue_cleanup: xn = self.container.ex_manager.create_xn_queue(queue) xn.delete() for exchange in self.exchange_cleanup: xp = self.container.ex_manager.create_xp(exchange) xp.delete() def test_stream_def_crud(self): # Test Creation pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict') stream_definition_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict.identifier) # Make sure there is an assoc self.assertTrue(self.resource_registry.find_associations(subject=stream_definition_id, predicate=PRED.hasParameterDictionary, object=pdict.identifier, id_only=True)) # Test Reading stream_definition = self.pubsub_management.read_stream_definition(stream_definition_id) self.assertTrue(PubsubManagementService._compare_pdicts(pdict.dump(), stream_definition.parameter_dictionary)) # Test Deleting self.pubsub_management.delete_stream_definition(stream_definition_id) self.assertFalse(self.resource_registry.find_associations(subject=stream_definition_id, predicate=PRED.hasParameterDictionary, object=pdict.identifier, id_only=True)) # Test comparisons in_stream_definition_id = self.pubsub_management.create_stream_definition('L0 products', parameter_dictionary=pdict.identifier, available_fields=['time','temp','conductivity','pressure']) self.addCleanup(self.pubsub_management.delete_stream_definition, in_stream_definition_id) out_stream_definition_id = in_stream_definition_id self.assertTrue(self.pubsub_management.compare_stream_definition(in_stream_definition_id, out_stream_definition_id)) self.assertTrue(self.pubsub_management.compatible_stream_definitions(in_stream_definition_id, out_stream_definition_id)) out_stream_definition_id = self.pubsub_management.create_stream_definition('L2 Products', parameter_dictionary=pdict.identifier, available_fields=['time','salinity','density']) self.addCleanup(self.pubsub_management.delete_stream_definition, out_stream_definition_id) self.assertFalse(self.pubsub_management.compare_stream_definition(in_stream_definition_id, out_stream_definition_id)) self.assertTrue(self.pubsub_management.compatible_stream_definitions(in_stream_definition_id, out_stream_definition_id)) def publish_on_stream(self, stream_id, msg): stream = self.pubsub_management.read_stream(stream_id) stream_route = stream.stream_route publisher = StandaloneStreamPublisher(stream_id=stream_id, stream_route=stream_route) publisher.publish(msg) def test_stream_crud(self): stream_def_id = self.pubsub_management.create_stream_definition('test_definition', stream_type='stream') topic_id = self.pubsub_management.create_topic(name='test_topic', exchange_point='test_exchange') self.exchange_cleanup.append('test_exchange') topic2_id = self.pubsub_management.create_topic(name='another_topic', exchange_point='outside') stream_id, route = self.pubsub_management.create_stream(name='test_stream', topic_ids=[topic_id, topic2_id], exchange_point='test_exchange', stream_definition_id=stream_def_id) topics, assocs = self.resource_registry.find_objects(subject=stream_id, predicate=PRED.hasTopic, id_only=True) self.assertEquals(topics,[topic_id]) defs, assocs = self.resource_registry.find_objects(subject=stream_id, predicate=PRED.hasStreamDefinition, id_only=True) self.assertTrue(len(defs)) stream = self.pubsub_management.read_stream(stream_id) self.assertEquals(stream.name,'test_stream') self.pubsub_management.delete_stream(stream_id) with self.assertRaises(NotFound): self.pubsub_management.read_stream(stream_id) defs, assocs = self.resource_registry.find_objects(subject=stream_id, predicate=PRED.hasStreamDefinition, id_only=True) self.assertFalse(len(defs)) topics, assocs = self.resource_registry.find_objects(subject=stream_id, predicate=PRED.hasTopic, id_only=True) self.assertFalse(len(topics)) self.pubsub_management.delete_topic(topic_id) self.pubsub_management.delete_topic(topic2_id) self.pubsub_management.delete_stream_definition(stream_def_id) def test_subscription_crud(self): stream_def_id = self.pubsub_management.create_stream_definition('test_definition', stream_type='stream') stream_id, route = self.pubsub_management.create_stream(name='test_stream', exchange_point='test_exchange', stream_definition_id=stream_def_id) subscription_id = self.pubsub_management.create_subscription(name='test subscription', stream_ids=[stream_id], exchange_name='test_queue') self.exchange_cleanup.append('test_exchange') subs, assocs = self.resource_registry.find_objects(subject=subscription_id,predicate=PRED.hasStream,id_only=True) self.assertEquals(subs,[stream_id]) res, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='test_queue', id_only=True) self.assertEquals(len(res),1) subs, assocs = self.resource_registry.find_subjects(object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertEquals(subs[0], res[0]) subscription = self.pubsub_management.read_subscription(subscription_id) self.assertEquals(subscription.exchange_name, 'test_queue') self.pubsub_management.delete_subscription(subscription_id) subs, assocs = self.resource_registry.find_objects(subject=subscription_id,predicate=PRED.hasStream,id_only=True) self.assertFalse(len(subs)) subs, assocs = self.resource_registry.find_subjects(object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertFalse(len(subs)) self.pubsub_management.delete_stream(stream_id) self.pubsub_management.delete_stream_definition(stream_def_id) def test_move_before_activate(self): stream_id, route = self.pubsub_management.create_stream(name='test_stream', exchange_point='test_xp') #-------------------------------------------------------------------------------- # Test moving before activate #-------------------------------------------------------------------------------- subscription_id = self.pubsub_management.create_subscription('first_queue', stream_ids=[stream_id]) xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='first_queue', id_only=True) subjects, _ = self.resource_registry.find_subjects(object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertEquals(xn_ids[0], subjects[0]) self.pubsub_management.move_subscription(subscription_id, exchange_name='second_queue') xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='second_queue', id_only=True) subjects, _ = self.resource_registry.find_subjects(object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertEquals(len(subjects),1) self.assertEquals(subjects[0], xn_ids[0]) self.pubsub_management.delete_subscription(subscription_id) self.pubsub_management.delete_stream(stream_id) def test_move_activated_subscription(self): stream_id, route = self.pubsub_management.create_stream(name='test_stream', exchange_point='test_xp') #-------------------------------------------------------------------------------- # Test moving after activate #-------------------------------------------------------------------------------- subscription_id = self.pubsub_management.create_subscription('first_queue', stream_ids=[stream_id]) self.pubsub_management.activate_subscription(subscription_id) xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='first_queue', id_only=True) subjects, _ = self.resource_registry.find_subjects(object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertEquals(xn_ids[0], subjects[0]) self.verified = Event() def verify(m,r,s): self.assertEquals(m,'verified') self.verified.set() subscriber = StandaloneStreamSubscriber('second_queue', verify) subscriber.start() self.pubsub_management.move_subscription(subscription_id, exchange_name='second_queue') xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='second_queue', id_only=True) subjects, _ = self.resource_registry.find_subjects(object=subscription_id, predicate=PRED.hasSubscription, id_only=True) self.assertEquals(len(subjects),1) self.assertEquals(subjects[0], xn_ids[0]) publisher = StandaloneStreamPublisher(stream_id, route) publisher.publish('verified') self.assertTrue(self.verified.wait(2)) self.pubsub_management.deactivate_subscription(subscription_id) self.pubsub_management.delete_subscription(subscription_id) self.pubsub_management.delete_stream(stream_id) def test_queue_cleanup(self): stream_id, route = self.pubsub_management.create_stream('test_stream','xp1') xn_objs, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='queue1') for xn_obj in xn_objs: xn = self.container.ex_manager.create_xn_queue(xn_obj.name) xn.delete() subscription_id = self.pubsub_management.create_subscription('queue1',stream_ids=[stream_id]) xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='queue1') self.assertEquals(len(xn_ids),1) self.pubsub_management.delete_subscription(subscription_id) xn_ids, _ = self.resource_registry.find_resources(restype=RT.ExchangeName, name='queue1') self.assertEquals(len(xn_ids),0) def test_activation_and_deactivation(self): stream_id, route = self.pubsub_management.create_stream('stream1','xp1') subscription_id = self.pubsub_management.create_subscription('sub1', stream_ids=[stream_id]) self.check1 = Event() def verifier(m,r,s): self.check1.set() subscriber = StandaloneStreamSubscriber('sub1',verifier) subscriber.start() publisher = StandaloneStreamPublisher(stream_id, route) publisher.publish('should not receive') self.assertFalse(self.check1.wait(0.25)) self.pubsub_management.activate_subscription(subscription_id) publisher.publish('should receive') self.assertTrue(self.check1.wait(2)) self.check1.clear() self.assertFalse(self.check1.is_set()) self.pubsub_management.deactivate_subscription(subscription_id) publisher.publish('should not receive') self.assertFalse(self.check1.wait(0.5)) self.pubsub_management.activate_subscription(subscription_id) publisher.publish('should receive') self.assertTrue(self.check1.wait(2)) subscriber.stop() self.pubsub_management.deactivate_subscription(subscription_id) self.pubsub_management.delete_subscription(subscription_id) self.pubsub_management.delete_stream(stream_id) def test_topic_crud(self): topic_id = self.pubsub_management.create_topic(name='test_topic', exchange_point='test_xp') self.exchange_cleanup.append('test_xp') topic = self.pubsub_management.read_topic(topic_id) self.assertEquals(topic.name,'test_topic') self.assertEquals(topic.exchange_point, 'test_xp') self.pubsub_management.delete_topic(topic_id) with self.assertRaises(NotFound): self.pubsub_management.read_topic(topic_id) def test_full_pubsub(self): self.sub1_sat = Event() self.sub2_sat = Event() def subscriber1(m,r,s): self.sub1_sat.set() def subscriber2(m,r,s): self.sub2_sat.set() sub1 = StandaloneStreamSubscriber('sub1', subscriber1) self.queue_cleanup.append(sub1.xn.queue) sub1.start() sub2 = StandaloneStreamSubscriber('sub2', subscriber2) self.queue_cleanup.append(sub2.xn.queue) sub2.start() log_topic = self.pubsub_management.create_topic('instrument_logs', exchange_point='instruments') science_topic = self.pubsub_management.create_topic('science_data', exchange_point='instruments') events_topic = self.pubsub_management.create_topic('notifications', exchange_point='events') log_stream, route = self.pubsub_management.create_stream('instrument1-logs', topic_ids=[log_topic], exchange_point='instruments') ctd_stream, route = self.pubsub_management.create_stream('instrument1-ctd', topic_ids=[science_topic], exchange_point='instruments') event_stream, route = self.pubsub_management.create_stream('notifications', topic_ids=[events_topic], exchange_point='events') raw_stream, route = self.pubsub_management.create_stream('temp', exchange_point='global.data') self.exchange_cleanup.extend(['instruments','events','global.data']) subscription1 = self.pubsub_management.create_subscription('subscription1', stream_ids=[log_stream,event_stream], exchange_name='sub1') subscription2 = self.pubsub_management.create_subscription('subscription2', exchange_points=['global.data'], stream_ids=[ctd_stream], exchange_name='sub2') self.pubsub_management.activate_subscription(subscription1) self.pubsub_management.activate_subscription(subscription2) self.publish_on_stream(log_stream, 1) self.assertTrue(self.sub1_sat.wait(4)) self.assertFalse(self.sub2_sat.is_set()) self.publish_on_stream(raw_stream,1) self.assertTrue(self.sub1_sat.wait(4)) sub1.stop() sub2.stop() def test_topic_craziness(self): self.msg_queue = Queue() def subscriber1(m,r,s): self.msg_queue.put(m) sub1 = StandaloneStreamSubscriber('sub1', subscriber1) self.queue_cleanup.append(sub1.xn.queue) sub1.start() topic1 = self.pubsub_management.create_topic('topic1', exchange_point='xp1') topic2 = self.pubsub_management.create_topic('topic2', exchange_point='xp1', parent_topic_id=topic1) topic3 = self.pubsub_management.create_topic('topic3', exchange_point='xp1', parent_topic_id=topic1) topic4 = self.pubsub_management.create_topic('topic4', exchange_point='xp1', parent_topic_id=topic2) topic5 = self.pubsub_management.create_topic('topic5', exchange_point='xp1', parent_topic_id=topic2) topic6 = self.pubsub_management.create_topic('topic6', exchange_point='xp1', parent_topic_id=topic3) topic7 = self.pubsub_management.create_topic('topic7', exchange_point='xp1', parent_topic_id=topic3) # Tree 2 topic8 = self.pubsub_management.create_topic('topic8', exchange_point='xp2') topic9 = self.pubsub_management.create_topic('topic9', exchange_point='xp2', parent_topic_id=topic8) topic10 = self.pubsub_management.create_topic('topic10', exchange_point='xp2', parent_topic_id=topic9) topic11 = self.pubsub_management.create_topic('topic11', exchange_point='xp2', parent_topic_id=topic9) topic12 = self.pubsub_management.create_topic('topic12', exchange_point='xp2', parent_topic_id=topic11) topic13 = self.pubsub_management.create_topic('topic13', exchange_point='xp2', parent_topic_id=topic11) self.exchange_cleanup.extend(['xp1','xp2']) stream1_id, route = self.pubsub_management.create_stream('stream1', topic_ids=[topic7, topic4, topic5], exchange_point='xp1') stream2_id, route = self.pubsub_management.create_stream('stream2', topic_ids=[topic8], exchange_point='xp2') stream3_id, route = self.pubsub_management.create_stream('stream3', topic_ids=[topic10,topic13], exchange_point='xp2') stream4_id, route = self.pubsub_management.create_stream('stream4', topic_ids=[topic9], exchange_point='xp2') stream5_id, route = self.pubsub_management.create_stream('stream5', topic_ids=[topic11], exchange_point='xp2') subscription1 = self.pubsub_management.create_subscription('sub1', topic_ids=[topic1]) subscription2 = self.pubsub_management.create_subscription('sub2', topic_ids=[topic8], exchange_name='sub1') subscription3 = self.pubsub_management.create_subscription('sub3', topic_ids=[topic9], exchange_name='sub1') subscription4 = self.pubsub_management.create_subscription('sub4', topic_ids=[topic10,topic13, topic11], exchange_name='sub1') #-------------------------------------------------------------------------------- self.pubsub_management.activate_subscription(subscription1) self.publish_on_stream(stream1_id,1) self.assertEquals(self.msg_queue.get(timeout=10), 1) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.1) self.pubsub_management.deactivate_subscription(subscription1) self.pubsub_management.delete_subscription(subscription1) #-------------------------------------------------------------------------------- self.pubsub_management.activate_subscription(subscription2) self.publish_on_stream(stream2_id,2) self.assertEquals(self.msg_queue.get(timeout=10), 2) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.1) self.pubsub_management.deactivate_subscription(subscription2) self.pubsub_management.delete_subscription(subscription2) #-------------------------------------------------------------------------------- self.pubsub_management.activate_subscription(subscription3) self.publish_on_stream(stream2_id, 3) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.3) self.publish_on_stream(stream3_id, 4) self.assertEquals(self.msg_queue.get(timeout=10),4) self.pubsub_management.deactivate_subscription(subscription3) self.pubsub_management.delete_subscription(subscription3) #-------------------------------------------------------------------------------- self.pubsub_management.activate_subscription(subscription4) self.publish_on_stream(stream4_id, 5) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.3) self.publish_on_stream(stream5_id, 6) self.assertEquals(self.msg_queue.get(timeout=10),6) with self.assertRaises(Empty): self.msg_queue.get(timeout=0.3) self.pubsub_management.deactivate_subscription(subscription4) self.pubsub_management.delete_subscription(subscription4) #-------------------------------------------------------------------------------- sub1.stop() self.pubsub_management.delete_topic(topic13) self.pubsub_management.delete_topic(topic12) self.pubsub_management.delete_topic(topic11) self.pubsub_management.delete_topic(topic10) self.pubsub_management.delete_topic(topic9) self.pubsub_management.delete_topic(topic8) self.pubsub_management.delete_topic(topic7) self.pubsub_management.delete_topic(topic6) self.pubsub_management.delete_topic(topic5) self.pubsub_management.delete_topic(topic4) self.pubsub_management.delete_topic(topic3) self.pubsub_management.delete_topic(topic2) self.pubsub_management.delete_topic(topic1) self.pubsub_management.delete_stream(stream1_id) self.pubsub_management.delete_stream(stream2_id) self.pubsub_management.delete_stream(stream3_id) self.pubsub_management.delete_stream(stream4_id) self.pubsub_management.delete_stream(stream5_id)
class TestDMEnd2End(IonIntegrationTestCase): def setUp(self): # Love the non pep-8 convention self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.process_dispatcher = ProcessDispatcherServiceClient() self.pubsub_management = PubsubManagementServiceClient() self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() self.ingestion_management = IngestionManagementServiceClient() self.data_retriever = DataRetrieverServiceClient() self.event = Event() self.exchange_space_name = 'test_granules' self.exchange_point_name = 'science_data' self.i = 0 self.cci = 0 #-------------------------------------------------------------------------------- # Helper/Utility methods #-------------------------------------------------------------------------------- def create_dataset(self, parameter_dict_id=''): ''' Creates a time-series dataset ''' if not parameter_dict_id: parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) dataset = Dataset('test_dataset_%i'%self.i) dataset_id = self.dataset_management.create_dataset(dataset, parameter_dictionary_id=parameter_dict_id) self.addCleanup(self.dataset_management.delete_dataset, dataset_id) return dataset_id def get_datastore(self, dataset_id): ''' Gets an instance of the datastore This method is primarily used to defeat a bug where integration tests in multiple containers may sometimes delete a CouchDB datastore and the other containers are unaware of the new state of the datastore. ''' dataset = self.dataset_management.read_dataset(dataset_id) datastore_name = dataset.datastore_name datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) return datastore def get_ingestion_config(self): ''' Grab the ingestion configuration from the resource registry ''' # The ingestion configuration should have been created by the bootstrap service # which is configured through r2deploy.yml ingest_configs, _ = self.resource_registry.find_resources(restype=RT.IngestionConfiguration,id_only=True) return ingest_configs[0] def launch_producer(self, stream_id=''): ''' Launch the producer ''' pid = self.container.spawn_process('better_data_producer', 'ion.processes.data.example_data_producer', 'BetterDataProducer', {'process':{'stream_id':stream_id}}) self.addCleanup(self.container.terminate_process, pid) def make_simple_dataset(self): ''' Makes a stream, a stream definition and a dataset, the essentials for most of these tests ''' pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) stream_def_id = self.pubsub_management.create_stream_definition('ctd data %i' % self.i, parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) stream_id, route = self.pubsub_management.create_stream('ctd stream %i' % self.i, 'xp1', stream_definition_id=stream_def_id) self.addCleanup(self.pubsub_management.delete_stream, stream_id) dataset_id = self.create_dataset(pdict_id) # self.get_datastore(dataset_id) self.i += 1 return stream_id, route, stream_def_id, dataset_id def publish_hifi(self,stream_id,stream_route,offset=0): ''' Publish deterministic data ''' pub = StandaloneStreamPublisher(stream_id, stream_route) stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id) stream_def_id = stream_def._id rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) + (offset * 10) rdt['temp'] = np.arange(10) + (offset * 10) pub.publish(rdt.to_granule()) def publish_fake_data(self,stream_id, route): ''' Make four granules ''' for i in xrange(4): self.publish_hifi(stream_id,route,i) def start_ingestion(self, stream_id, dataset_id): ''' Starts ingestion/persistence for a given dataset ''' ingest_config_id = self.get_ingestion_config() self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id) def stop_ingestion(self, stream_id): ingest_config_id = self.get_ingestion_config() self.ingestion_management.unpersist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id) def validate_granule_subscription(self, msg, route, stream_id): ''' Validation for granule format ''' if msg == {}: return rdt = RecordDictionaryTool.load_from_granule(msg) log.info('%s', rdt.pretty_print()) self.assertIsInstance(msg,Granule,'Message is improperly formatted. (%s)' % type(msg)) self.event.set() def wait_until_we_have_enough_granules(self, dataset_id='',data_size=40): ''' Loops until there is a sufficient amount of data in the dataset ''' done = False with gevent.Timeout(40): while not done: extents = self.dataset_management.dataset_extents(dataset_id, 'time') granule = self.data_retriever.retrieve_last_data_points(dataset_id, 1) rdt = RecordDictionaryTool.load_from_granule(granule) if rdt['time'] and rdt['time'][0] != rdt._pdict.get_context('time').fill_value and extents >= data_size: done = True else: gevent.sleep(0.2) #-------------------------------------------------------------------------------- # Test Methods #-------------------------------------------------------------------------------- def test_dm_end_2_end(self): #-------------------------------------------------------------------------------- # Set up a stream and have a mock instrument (producer) send data #-------------------------------------------------------------------------------- self.event.clear() # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_definition = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream('producer', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition) #-------------------------------------------------------------------------------- # Start persisting the data on the stream # - Get the ingestion configuration from the resource registry # - Create the dataset # - call persist_data_stream to setup the subscription for the ingestion workers # on the stream that you specify which causes the data to be persisted #-------------------------------------------------------------------------------- ingest_config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id) self.addCleanup(self.stop_ingestion, stream_id) #-------------------------------------------------------------------------------- # Now the granules are ingesting and persisted #-------------------------------------------------------------------------------- self.launch_producer(stream_id) self.wait_until_we_have_enough_granules(dataset_id,40) #-------------------------------------------------------------------------------- # Now get the data in one chunk using an RPC Call to start_retreive #-------------------------------------------------------------------------------- replay_data = self.data_retriever.retrieve(dataset_id) self.assertIsInstance(replay_data, Granule) rdt = RecordDictionaryTool.load_from_granule(replay_data) self.assertTrue((rdt['time'][:10] == np.arange(10)).all(),'%s' % rdt['time'][:]) self.assertTrue((rdt['binary'][:10] == np.array(['hi']*10, dtype='object')).all()) #-------------------------------------------------------------------------------- # Now to try the streamed approach #-------------------------------------------------------------------------------- replay_stream_id, replay_route = self.pubsub_management.create_stream('replay_out', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition) self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream_id) log.info('Process ID: %s', process_id) replay_client = ReplayClient(process_id) #-------------------------------------------------------------------------------- # Create the listening endpoint for the the retriever to talk to #-------------------------------------------------------------------------------- sub_id = self.pubsub_management.create_subscription(self.exchange_space_name,stream_ids=[replay_stream_id]) self.addCleanup(self.pubsub_management.delete_subscription, sub_id) self.pubsub_management.activate_subscription(sub_id) self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id) subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription) subscriber.start() self.addCleanup(subscriber.stop) self.data_retriever.start_replay_agent(self.replay_id) self.assertTrue(replay_client.await_agent_ready(5), 'The process never launched') replay_client.start_replay() self.assertTrue(self.event.wait(10)) self.data_retriever.cancel_replay_agent(self.replay_id) #-------------------------------------------------------------------------------- # Test the slicing capabilities #-------------------------------------------------------------------------------- granule = self.data_retriever.retrieve(dataset_id=dataset_id, query={'tdoa':slice(0,5)}) rdt = RecordDictionaryTool.load_from_granule(granule) b = rdt['time'] == np.arange(5) self.assertTrue(b.all() if not isinstance(b,bool) else b) def test_coverage_transform(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_parsed() stream_def_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) stream_id, route = self.pubsub_management.create_stream('example', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) self.addCleanup(self.pubsub_management.delete_stream, stream_id) ingestion_config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id) self.addCleanup(self.ingestion_management.unpersist_data_stream, stream_id, ingestion_config_id) publisher = StandaloneStreamPublisher(stream_id, route) rdt = ph.get_rdt(stream_def_id) ph.fill_parsed_rdt(rdt) dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_monitor.wait()) replay_granule = self.data_retriever.retrieve(dataset_id) rdt_out = RecordDictionaryTool.load_from_granule(replay_granule) np.testing.assert_array_almost_equal(rdt_out['time'], rdt['time']) np.testing.assert_array_almost_equal(rdt_out['temp'], rdt['temp']) np.testing.assert_allclose(rdt_out['conductivity_L1'], np.array([42.914])) np.testing.assert_allclose(rdt_out['temp_L1'], np.array([20.])) np.testing.assert_allclose(rdt_out['pressure_L1'], np.array([3.068])) np.testing.assert_allclose(rdt_out['density'], np.array([1021.7144739593881], dtype='float32')) np.testing.assert_allclose(rdt_out['salinity'], np.array([30.935132729668283], dtype='float32')) def test_ingestion_pause(self): ctd_stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() ingestion_config_id = self.get_ingestion_config() self.start_ingestion(ctd_stream_id, dataset_id) self.addCleanup(self.stop_ingestion, ctd_stream_id) rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) rdt['time'] = np.arange(10) publisher = StandaloneStreamPublisher(ctd_stream_id, route) monitor = DatasetMonitor(dataset_id) self.addCleanup(monitor.stop) publisher.publish(rdt.to_granule()) self.assertTrue(monitor.wait()) granule = self.data_retriever.retrieve(dataset_id) self.ingestion_management.pause_data_stream(ctd_stream_id, ingestion_config_id) monitor.event.clear() rdt['time'] = np.arange(10,20) publisher.publish(rdt.to_granule()) self.assertFalse(monitor.event.wait(1)) self.ingestion_management.resume_data_stream(ctd_stream_id, ingestion_config_id) self.assertTrue(monitor.wait()) granule = self.data_retriever.retrieve(dataset_id) rdt2 = RecordDictionaryTool.load_from_granule(granule) np.testing.assert_array_almost_equal(rdt2['time'], np.arange(20)) def test_last_granule(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() self.start_ingestion(stream_id, dataset_id) self.addCleanup(self.stop_ingestion, stream_id) self.publish_hifi(stream_id,route, 0) self.publish_hifi(stream_id,route, 1) self.wait_until_we_have_enough_granules(dataset_id,20) # I just need two success = False def verifier(): replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id, 10) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(10) + 10 if not isinstance(comp,bool): return comp.all() return False success = poll(verifier) self.assertTrue(success) success = False def verify_points(): replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id,5) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(15,20) if not isinstance(comp,bool): return comp.all() return False success = poll(verify_points) self.assertTrue(success) def test_replay_with_parameters(self): #-------------------------------------------------------------------------------- # Create the configurations and the dataset #-------------------------------------------------------------------------------- # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True) # Add a field that supports binary data input. bin_context = ParameterContext('binary', param_type=ArrayType()) context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump())) # Add another field that supports dictionary elements. rec_context = ParameterContext('records', param_type=RecordType()) context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump())) pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time') stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id) stream_id, route = self.pubsub_management.create_stream('replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) config_id = self.get_ingestion_config() dataset_id = self.create_dataset(pdict_id) self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id) self.addCleanup(self.stop_ingestion, stream_id) dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) self.publish_fake_data(stream_id, route) self.assertTrue(dataset_monitor.wait()) query = { 'start_time': 0 - 2208988800, 'end_time': 19 - 2208988800, 'stride_time' : 2, 'parameters': ['time','temp'] } retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id,query=query) rdt = RecordDictionaryTool.load_from_granule(retrieved_data) np.testing.assert_array_equal(rdt['time'], np.arange(0,20,2)) self.assertEquals(set(rdt.iterkeys()), set(['time','temp'])) extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=['time','temp']) self.assertTrue(extents['time']>=20) self.assertTrue(extents['temp']>=20) def test_repersist_data(self): stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() self.start_ingestion(stream_id, dataset_id) self.publish_hifi(stream_id,route,0) self.publish_hifi(stream_id,route,1) self.wait_until_we_have_enough_granules(dataset_id,20) config_id = self.get_ingestion_config() self.ingestion_management.unpersist_data_stream(stream_id=stream_id,ingestion_configuration_id=config_id) self.ingestion_management.persist_data_stream(stream_id=stream_id,ingestion_configuration_id=config_id,dataset_id=dataset_id) self.addCleanup(self.stop_ingestion, stream_id) self.publish_hifi(stream_id,route,2) self.publish_hifi(stream_id,route,3) self.wait_until_we_have_enough_granules(dataset_id,40) success = False with gevent.timeout.Timeout(5): while not success: replay_granule = self.data_retriever.retrieve(dataset_id) rdt = RecordDictionaryTool.load_from_granule(replay_granule) comp = rdt['time'] == np.arange(0,40) if not isinstance(comp,bool): success = comp.all() gevent.sleep(1) self.assertTrue(success) @unittest.skip('deprecated') def test_correct_time(self): # There are 2208988800 seconds between Jan 1 1900 and Jan 1 1970, i.e. # the conversion factor between unix and NTP time unix_now = np.floor(time.time()) ntp_now = unix_now + 2208988800 unix_ago = unix_now - 20 ntp_ago = unix_ago + 2208988800 stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() coverage = DatasetManagementService._get_simplex_coverage(dataset_id, mode='a') coverage.insert_timesteps(20) coverage.set_parameter_values('time', np.arange(ntp_ago,ntp_now)) temporal_bounds = self.dataset_management.dataset_temporal_bounds(dataset_id) self.assertTrue( np.abs(temporal_bounds[0] - unix_ago) < 2) self.assertTrue( np.abs(temporal_bounds[1] - unix_now) < 2) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.') def test_out_of_band_retrieve(self): # Setup the environemnt stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset() self.start_ingestion(stream_id, dataset_id) # Fill the dataset self.publish_fake_data(stream_id, route) self.wait_until_we_have_enough_granules(dataset_id,40) # Retrieve the data granule = DataRetrieverService.retrieve_oob(dataset_id) rdt = RecordDictionaryTool.load_from_granule(granule) self.assertTrue((rdt['time'] == np.arange(40)).all()) def publish_and_wait(self, dataset_id, granule): stream_ids, _ = self.resource_registry.find_objects(dataset_id, PRED.hasStream,id_only=True) stream_id=stream_ids[0] route = self.pubsub_management.read_stream_route(stream_id) publisher = StandaloneStreamPublisher(stream_id,route) dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) publisher.publish(granule) self.assertTrue(dataset_monitor.wait()) def test_sparse_values(self): ph = ParameterHelper(self.dataset_management, self.addCleanup) pdict_id = ph.create_sparse() stream_def_id = self.pubsub_management.create_stream_definition('sparse', parameter_dictionary_id=pdict_id) self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id) stream_id, route = self.pubsub_management.create_stream('example', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id) self.addCleanup(self.pubsub_management.delete_stream, stream_id) dataset_id = self.create_dataset(pdict_id) self.start_ingestion(stream_id,dataset_id) self.addCleanup(self.stop_ingestion, stream_id) # Publish initial granule # the first one has the sparse value set inside it, sets lat to 45 and lon to -71 ntp_now = time.time() + 2208988800 rdt = ph.get_rdt(stream_def_id) rdt['time'] = [ntp_now] rdt['internal_timestamp'] = [ntp_now] rdt['temp'] = [300000] rdt['preferred_timestamp'] = ['driver_timestamp'] rdt['port_timestamp'] = [ntp_now] rdt['quality_flag'] = [''] rdt['lat'] = [45] rdt['conductivity'] = [4341400] rdt['driver_timestamp'] = [ntp_now] rdt['lon'] = [-71] rdt['pressure'] = [256.8] publisher = StandaloneStreamPublisher(stream_id, route) dataset_monitor = DatasetMonitor(dataset_id) self.addCleanup(dataset_monitor.stop) publisher.publish(rdt.to_granule()) self.assertTrue(dataset_monitor.wait()) dataset_monitor.reset() replay_granule = self.data_retriever.retrieve(dataset_id) rdt_out = RecordDictionaryTool.load_from_granule(replay_granule) # Check the values and make sure they're correct np.testing.assert_allclose(rdt_out['time'], rdt['time']) np.testing.assert_allclose(rdt_out['temp'], rdt['temp']) np.testing.assert_allclose(rdt_out['lat'], np.array([45])) np.testing.assert_allclose(rdt_out['lon'], np.array([-71])) np.testing.assert_allclose(rdt_out['conductivity_L1'], np.array([42.914])) np.testing.assert_allclose(rdt_out['temp_L1'], np.array([20.])) np.testing.assert_allclose(rdt_out['pressure_L1'], np.array([3.068])) np.testing.assert_allclose(rdt_out['density'], np.array([1021.7144739593881], dtype='float32')) np.testing.assert_allclose(rdt_out['salinity'], np.array([30.935132729668283], dtype='float32')) # We're going to change the lat/lon rdt = ph.get_rdt(stream_def_id) rdt['time'] = time.time() + 2208988800 rdt['lat'] = [46] rdt['lon'] = [-73] publisher.publish(rdt.to_granule()) self.assertTrue(dataset_monitor.wait()) dataset_monitor.reset() replay_granule = self.data_retriever.retrieve(dataset_id) rdt_out = RecordDictionaryTool.load_from_granule(replay_granule) np.testing.assert_allclose(rdt_out['time'], rdt['time']) for i in xrange(9): ntp_now = time.time() + 2208988800 rdt['time'] = [ntp_now] rdt['internal_timestamp'] = [ntp_now] rdt['temp'] = [300000] rdt['preferred_timestamp'] = ['driver_timestamp'] rdt['port_timestamp'] = [ntp_now] rdt['quality_flag'] = [None] rdt['conductivity'] = [4341400] rdt['driver_timestamp'] = [ntp_now] rdt['pressure'] = [256.8] publisher.publish(rdt.to_granule()) self.assertTrue(dataset_monitor.wait()) dataset_monitor.reset() replay_granule = self.data_retriever.retrieve(dataset_id) rdt_out = RecordDictionaryTool.load_from_granule(replay_granule) np.testing.assert_allclose(rdt_out['pressure'], np.array([256.8] * 10)) np.testing.assert_allclose(rdt_out['lat'], np.array([45] + [46] * 9)) np.testing.assert_allclose(rdt_out['lon'], np.array([-71] + [-73] * 9))
class TransformPrototypeIntTest(IonIntegrationTestCase): def setUp(self): super(TransformPrototypeIntTest, self).setUp() self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.rrc = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() self.pubsub_management = PubsubManagementServiceClient() self.ssclient = SchedulerServiceClient() self.event_publisher = EventPublisher() self.user_notification = UserNotificationServiceClient() self.process_dispatcher = ProcessDispatcherServiceClient() self.exchange_names = [] self.exchange_points = [] def tearDown(self): for xn in self.exchange_names: xni = self.container.ex_manager.create_xn_queue(xn) xni.delete() for xp in self.exchange_points: xpi = self.container.ex_manager.create_xp(xp) xpi.delete() def now_utc(self): return time.time() def _create_interval_timer_with_end_time(self,timer_interval= None, end_time = None ): ''' A convenience method to set up an interval timer with an end time ''' self.timer_received_time = 0 self.timer_interval = timer_interval start_time = self.now_utc() if not end_time: end_time = start_time + 2 * timer_interval + 1 log.debug("got the end time here!! %s" % end_time) # Set up the interval timer. The scheduler will publish event with origin set as "Interval Timer" sid = self.ssclient.create_interval_timer(start_time="now" , interval=self.timer_interval, end_time=end_time, event_origin="Interval Timer", event_subtype="") def cleanup_timer(scheduler, schedule_id): """ Do a friendly cancel of the scheduled event. If it fails, it's ok. """ try: scheduler.cancel_timer(schedule_id) except: log.warn("Couldn't cancel") self.addCleanup(cleanup_timer, self.ssclient, sid) return sid @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_event_processing(self): ''' Test that events are processed by the transforms according to a provided algorithm ''' #------------------------------------------------------------------------------------- # Set up the scheduler for an interval timer with an end time #------------------------------------------------------------------------------------- id = self._create_interval_timer_with_end_time(timer_interval=2) self.assertIsNotNone(id) #------------------------------------------------------------------------------------- # Create an event alert transform.... # The configuration for the Event Alert Transform... set up the event types to listen to #------------------------------------------------------------------------------------- configuration = { 'process':{ 'event_type': 'ResourceEvent', 'timer_origin': 'Interval Timer', 'instrument_origin': 'My_favorite_instrument' } } #------------------------------------------------------------------------------------- # Create the process #------------------------------------------------------------------------------------- pid = TransformPrototypeIntTest.create_process( name= 'event_alert_transform', module='ion.processes.data.transforms.event_alert_transform', class_name='EventAlertTransform', configuration= configuration) self.addCleanup(self.process_dispatcher.cancel_process, pid) self.assertIsNotNone(pid) #------------------------------------------------------------------------------------- # Publish events and make assertions about alerts #------------------------------------------------------------------------------------- queue = gevent.queue.Queue() def event_received(message, headers): queue.put(message) event_subscriber = EventSubscriber( origin="EventAlertTransform", event_type="DeviceEvent", callback=event_received) event_subscriber.start() self.addCleanup(event_subscriber.stop) # publish event twice for i in xrange(5): self.event_publisher.publish_event( event_type = 'ExampleDetectableEvent', origin = "My_favorite_instrument", voltage = 5, telemetry = 10, temperature = 20) gevent.sleep(0.1) self.assertTrue(queue.empty()) #publish event the third time but after a time interval larger than 2 seconds gevent.sleep(5) #------------------------------------------------------------------------------------- # Make assertions about the alert event published by the EventAlertTransform #------------------------------------------------------------------------------------- event = queue.get(timeout=10) log.debug("Alarm event received from the EventAertTransform %s" % event) self.assertEquals(event.type_, "DeviceEvent") self.assertEquals(event.origin, "EventAlertTransform") #------------------------------------------------------------------------------------------------ # Now clear the event queue being populated by alarm events and publish normally once again #------------------------------------------------------------------------------------------------ queue.queue.clear() for i in xrange(5): self.event_publisher.publish_event( event_type = 'ExampleDetectableEvent', origin = "My_favorite_instrument", voltage = 5, telemetry = 10, temperature = 20) gevent.sleep(0.1) self.assertTrue(queue.empty()) log.debug("This completes the requirement that the EventAlertTransform publishes \ an alarm event when it does not hear from the instrument for some time.") @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_stream_processing(self): #-------------------------------------------------------------------------------- #Test that streams are processed by the transforms according to a provided algorithm #-------------------------------------------------------------------------------- #todo: In this simple implementation, we are checking if the stream has the word, PUBLISH, #todo(contd) and if the word VALUE=<number> exists and that number is less than something #todo later on we are going to use complex algorithms to make this prototype powerful #------------------------------------------------------------------------------------- # Start a subscriber to listen for an alert event from the Stream Alert Transform #------------------------------------------------------------------------------------- queue = gevent.queue.Queue() def event_received(message, headers): queue.put(message) event_subscriber = EventSubscriber( origin="StreamAlertTransform", event_type="DeviceEvent", callback=event_received) event_subscriber.start() self.addCleanup(event_subscriber.stop) #------------------------------------------------------------------------------------- # The configuration for the Stream Alert Transform... set up the event types to listen to #------------------------------------------------------------------------------------- config = { 'process':{ 'queue_name': 'a_queue', 'value': 10, 'event_type':'DeviceEvent' } } #------------------------------------------------------------------------------------- # Create the process #------------------------------------------------------------------------------------- pid = TransformPrototypeIntTest.create_process( name= 'transform_data_process', module='ion.processes.data.transforms.event_alert_transform', class_name='StreamAlertTransform', configuration= config) self.addCleanup(self.process_dispatcher.cancel_process, pid) self.assertIsNotNone(pid) #------------------------------------------------------------------------------------- # Publish streams and make assertions about alerts #------------------------------------------------------------------------------------- exchange_name = 'a_queue' exchange_point = 'test_exchange' routing_key = 'stream_id.stream' stream_route = StreamRoute(exchange_point, routing_key) xn = self.container.ex_manager.create_xn_queue(exchange_name) xp = self.container.ex_manager.create_xp(exchange_point) xn.bind('stream_id.stream', xp) pub = StandaloneStreamPublisher('stream_id', stream_route) message = "A dummy example message containing the word PUBLISH, and with VALUE = 5 . This message" +\ " will trigger an alert event from the StreamAlertTransform because the value provided is "\ "less than 10 that was passed in through the config." pub.publish(message) event = queue.get(timeout=10) self.assertEquals(event.type_, "DeviceEvent") self.assertEquals(event.origin, "StreamAlertTransform") # self.purge_queues(exchange_name) # def purge_queues(self, exchange_name): # xn = self.container.ex_manager.create_xn_queue(exchange_name) # xn.purge() @staticmethod def create_process(name= '', module = '', class_name = '', configuration = None): ''' A helper method to create a process ''' producer_definition = ProcessDefinition(name=name) producer_definition.executable = { 'module':module, 'class': class_name } process_dispatcher = ProcessDispatcherServiceClient() procdef_id = process_dispatcher.create_process_definition(process_definition=producer_definition) pid = process_dispatcher.schedule_process(process_definition_id= procdef_id, configuration=configuration) return pid @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_demo_stream_granules_processing(self): """ Test that the Demo Stream Alert Transform is functioning. The transform coordinates with the scheduler. It is configured to listen to a source that publishes granules. It publishes a DeviceStatusEvent if it receives a granule with bad data or a DeviceCommsEvent if no granule has arrived between two timer events. The transform is configured at launch using a config dictionary. """ #------------------------------------------------------------------------------------- # Start a subscriber to listen for an alert event from the Stream Alert Transform #------------------------------------------------------------------------------------- queue_bad_data = gevent.queue.Queue() queue_no_data = gevent.queue.Queue() def bad_data(message, headers): log.debug("Got a BAD data event: %s" % message) if message.type_ == "DeviceStatusEvent": queue_bad_data.put(message) def no_data(message, headers): log.debug("Got a NO data event: %s" % message) queue_no_data.put(message) event_subscriber_bad_data = EventSubscriber( origin="instrument_1", event_type="DeviceStatusEvent", callback=bad_data) event_subscriber_no_data = EventSubscriber( origin="instrument_1", event_type="DeviceCommsEvent", callback=no_data) event_subscriber_bad_data.start() event_subscriber_no_data.start() self.addCleanup(event_subscriber_bad_data.stop) self.addCleanup(event_subscriber_no_data.stop) #------------------------------------------------------------------------------------- # The configuration for the Stream Alert Transform... set up the event types to listen to #------------------------------------------------------------------------------------- self.valid_values = [-100, 100] self.timer_interval = 5 self.queue_name = 'a_queue' config = { 'process':{ 'timer_interval': self.timer_interval, 'queue_name': self.queue_name, 'variable_name': 'input_voltage', 'time_field_name': 'preferred_timestamp', 'valid_values': self.valid_values, 'timer_origin': 'Interval Timer', 'event_origin': 'instrument_1' } } #------------------------------------------------------------------------------------- # Create the process #------------------------------------------------------------------------------------- pid = TransformPrototypeIntTest.create_process( name= 'DemoStreamAlertTransform', module='ion.processes.data.transforms.event_alert_transform', class_name='DemoStreamAlertTransform', configuration= config) self.addCleanup(self.process_dispatcher.cancel_process, pid) self.assertIsNotNone(pid) #------------------------------------------------------------------------------------- # Publish streams and make assertions about alerts #------------------------------------------------------------------------------------- pdict_id = self.dataset_management.read_parameter_dictionary_by_name(name= 'platform_eng_parsed', id_only=True) stream_def_id = self.pubsub_management.create_stream_definition('demo_stream', parameter_dictionary_id=pdict_id) stream_id, stream_route = self.pubsub_management.create_stream( name='test_demo_alert', exchange_point='exch_point_1', stream_definition_id=stream_def_id) sub_1 = self.pubsub_management.create_subscription(name='sub_1', stream_ids=[stream_id], exchange_points=['exch_point_1'], exchange_name=self.queue_name) self.pubsub_management.activate_subscription(sub_1) self.exchange_names.append('sub_1') self.exchange_points.append('exch_point_1') #------------------------------------------------------------------------------------- # publish a *GOOD* granule #------------------------------------------------------------------------------------- self.length = 2 val = numpy.array([random.uniform(0,50) for l in xrange(self.length)]) self._publish_granules(stream_id= stream_id, stream_route= stream_route, number=1, values=val) self.assertTrue(queue_bad_data.empty()) #------------------------------------------------------------------------------------- # publish a few *BAD* granules #------------------------------------------------------------------------------------- self.number = 2 val = numpy.array([(110 + l) for l in xrange(self.length)]) self._publish_granules(stream_id= stream_id, stream_route= stream_route, number= self.number, values=val) for number in xrange(self.number): event = queue_bad_data.get(timeout=40) self.assertEquals(event.type_, "DeviceStatusEvent") self.assertEquals(event.origin, "instrument_1") self.assertEquals(event.state, DeviceStatusType.STATUS_WARNING) self.assertEquals(event.valid_values, self.valid_values) self.assertEquals(event.sub_type, 'input_voltage') self.assertTrue(set(event.values) == set(val)) s = set(event.time_stamps) cond = s in [set(numpy.array([1 for l in xrange(self.length)]).tolist()), set(numpy.array([2 for l in xrange(self.length)]).tolist())] self.assertTrue(cond) # To ensure that only the bad values generated the alert events. Queue should be empty now self.assertEquals(queue_bad_data.qsize(), 0) #------------------------------------------------------------------------------------- # Do not publish any granules for some time. This should generate a DeviceCommsEvent for the communication status #------------------------------------------------------------------------------------- event = queue_no_data.get(timeout=15) self.assertEquals(event.type_, "DeviceCommsEvent") self.assertEquals(event.origin, "instrument_1") self.assertEquals(event.origin_type, "PlatformDevice") self.assertEquals(event.state, DeviceCommsType.DATA_DELIVERY_INTERRUPTION) self.assertEquals(event.sub_type, 'input_voltage') #------------------------------------------------------------------------------------- # Empty the queues and repeat tests #------------------------------------------------------------------------------------- queue_bad_data.queue.clear() queue_no_data.queue.clear() #------------------------------------------------------------------------------------- # publish a *GOOD* granule again #------------------------------------------------------------------------------------- val = numpy.array([(l + 20) for l in xrange(self.length)]) self._publish_granules(stream_id= stream_id, stream_route= stream_route, number=1, values=val) self.assertTrue(queue_bad_data.empty()) #------------------------------------------------------------------------------------- # Again do not publish any granules for some time. This should generate a DeviceCommsEvent for the communication status #------------------------------------------------------------------------------------- event = queue_no_data.get(timeout=20) self.assertEquals(event.type_, "DeviceCommsEvent") self.assertEquals(event.origin, "instrument_1") self.assertEquals(event.origin_type, "PlatformDevice") self.assertEquals(event.state, DeviceCommsType.DATA_DELIVERY_INTERRUPTION) self.assertEquals(event.sub_type, 'input_voltage') #------------------------------------------------------------------------------------- # Again do not publish any granules for some time. This should generate a DeviceCommsEvent for the communication status #------------------------------------------------------------------------------------- ar = gevent.event.AsyncResult() def poller(ar, method, *args): events_in_db = method(*args) if len(events_in_db) > 0: ar.set(events_in_db) return True else: return False poll(poller, ar, self.user_notification.find_events, 'instrument_1') # events_in_db = self.user_notification.find_events(origin='instrument_1') events_in_db = ar.get(10) log.debug("events::: %s" % events_in_db) bad_data_events = [] no_data_events = [] for event in events_in_db: if event.type_ == 'DeviceStatusEvent': bad_data_events.append(event) self.assertEquals(event.origin, "instrument_1") self.assertEquals(event.status, DeviceStatusType.STATUS_WARNING) self.assertEquals(event.valid_values, self.valid_values) self.assertEquals(event.sub_type, 'input_voltage') elif event.type_ == 'DeviceCommsEvent': no_data_events.append(event) self.assertEquals(event.origin, "instrument_1") self.assertEquals(event.origin_type, "PlatformDevice") self.assertEquals(event.status, DeviceCommsType.DATA_DELIVERY_INTERRUPTION) self.assertEquals(event.sub_type, 'input_voltage') self.assertTrue(len(bad_data_events) > 0) self.assertTrue(len(no_data_events) > 0) log.debug("This satisfies L4-CI-SA-RQ-114 : 'Marine facility shall monitor marine infrastructure usage by instruments.'" " The req is satisfied because the stream alert transform" "is able to send device status and communication events over selected time intervals. This capability will be " "augmented in the future.") def _publish_granules(self, stream_id=None, stream_route=None, values = None,number=None): pub = StandaloneStreamPublisher(stream_id, stream_route) stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id) stream_def_id = stream_def._id rdt = RecordDictionaryTool(stream_definition_id=stream_def_id) times = numpy.array([number for l in xrange(self.length)]) for i in xrange(number): rdt['input_voltage'] = values rdt['preferred_timestamp'] = ['time' for l in xrange(len(times))] rdt['time'] = times g = rdt.to_granule() g.data_producer_id = 'instrument_1' log.debug("granule #%s published by instrument:: %s" % ( number,g)) pub.publish(g) @staticmethod def makeEpochTime(date_time = None): """ provides the seconds since epoch give a python datetime object. @param date_time Python datetime object @retval seconds_since_epoch int """ seconds_since_epoch = calendar.timegm(date_time.timetuple()) return seconds_since_epoch