class TestScionAgentData(IonIntegrationTestCase):
    """Test for Scion with agents and data streaming
    """

    def setUp(self):
        self._start_container()
        self.patch_alt_cfg('scion.process.preload.preloader.CFG',
                           {'scion': {'preload': {'enabled': False}}})
        self.container.start_rel_from_url('res/deploy/scion.yml')

        self.rr = self.container.resource_registry
        self.scion_client = ScionManagementClient()
        self.idm_client = IdentityManagementServiceClient()
        self.system_actor_id = None

        self.ui_server_proc = self.container.proc_manager.procs_by_name["ui_server"]
        self.scion_proc = self.container.proc_manager.procs_by_name["scion_management"]
        self.ui_base_url = self.ui_server_proc.base_url
        self.sg_base_url = self.ui_server_proc.gateway_base_url

    def tearDown(self):
        pass

    def test_scion_agent(self):
        # Create user
        actor_id = self.scion_client.define_user(
                first_name="John", last_name="Doe",
                username="******", password="******", email="*****@*****.**")

        # Create instrument
        agent_info=[dict(agent_type="data_agent",
                         config=dict(plugin="scion.agent.model.vmmon.vmmon_plugin.VMMON_DataAgentPlugin",
                                     sampling_interval=0.5, stream_name="basic_streams",
                                     auto_streaming=False))]
        inst_obj = Instrument(name="Sensor 1", description="CPU monitor",
                              location=GeospatialLocation(latitude=32.867079, longitude=-117.257324),
                              agent_info=agent_info)
        inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)

        # Create dataset
        schema_def = DataSchemaParser.parse_schema_ref("ds_vmmon_main")
        ds_obj = Dataset(name="Dataset Sensor 1",
                         schema_definition=schema_def)
        ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)

        self.rr.create_association(inst_id, PRED.hasDataset, ds_id)

        ds_filename = self.container.file_system.get("%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
        self.assertFalse(os.path.exists(ds_filename))

        inst_data_t0 = self.scion_client.get_asset_data(inst_id)
        self.assertEquals(inst_data_t0["dataset_id"], ds_id)
        self.assertEquals(inst_data_t0["num_rows"], 0)

        # Install a data packet catcher
        self.recv_packets, self.recv_rows = [], 0
        def process_packet_cb(packet, route, stream):
            if not isinstance(packet, DataPacket):
                log.warn("Received a non DataPacket message")
            self.recv_packets.append(packet)
            self.recv_rows += len(packet.data["data"])
            log.info("Received data packet #%s: rows=%s, cols=%s", len(self.recv_packets), len(packet.data["data"]),
                     packet.data["cols"])
        def cleanup_stream_sub():
            if self.stream_sub:
                self.stream_sub.stop()
                self.stream_sub = None

        self.stream_sub = StreamSubscriber(process=self.scion_proc, stream="basic_streams", callback=process_packet_cb)
        self.stream_sub.start()

        self.addCleanup(cleanup_stream_sub)

        # Start agent
        self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))

        agent_pid = self.scion_client.start_agent(inst_id)

        self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))

        sac = StreamingAgentClient(resource_id=inst_id, process=self.scion_proc)
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)

        sac.connect()
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)

        sac.start_streaming()
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_STREAMING)

        # Retrieve data
        gevent.sleep(1.1)
        self.assertTrue(os.path.exists(ds_filename))

        inst_data_t1 = self.scion_client.get_asset_data(inst_id)
        self.assertEquals(inst_data_t1["dataset_id"], ds_id)
        # self.assertEquals(inst_data_t1["variables"], ['time', 'cpu_percent'])
        self.assertIn("cpu_percent", inst_data_t1["data"])
        num_rows_t1 = inst_data_t1["num_rows"]
        self.assertGreaterEqual(num_rows_t1, 2)

        gevent.sleep(1)
        inst_data_t2 = self.scion_client.get_asset_data(inst_id)
        num_rows_t2 = inst_data_t2["num_rows"]
        self.assertGreater(num_rows_t2, num_rows_t1)

        self.assertGreaterEqual(len(self.recv_packets), self.recv_rows)
        self.assertLessEqual(abs(self.recv_rows - num_rows_t2), 2)

        # Take down agent
        sac.stop_streaming()  # Not required to stop agent, just to test here
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)

        sac.disconnect()
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)

        self.scion_client.stop_agent(inst_id)

        self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
Esempio n. 2
0
class TestScionCDIPAgentData(IonIntegrationTestCase):
    """Test for Scion with agents streaming data from CDIP source
    """
    def setUp(self):
        self._start_container()
        self.patch_alt_cfg('scion.process.preload.preloader.CFG',
                           {'scion': {
                               'preload': {
                                   'enabled': False
                               }
                           }})
        self.container.start_rel_from_url('res/deploy/scion.yml')

        self.rr = self.container.resource_registry
        self.scion_client = ScionManagementClient()
        self.idm_client = IdentityManagementServiceClient()
        self.system_actor_id = None

        self.ui_server_proc = self.container.proc_manager.procs_by_name[
            "ui_server"]
        self.scion_proc = self.container.proc_manager.procs_by_name[
            "scion_management"]
        self.ui_base_url = self.ui_server_proc.base_url
        self.sg_base_url = self.ui_server_proc.gateway_base_url

    def tearDown(self):
        pass

    def test_scion_agent(self):
        # Create user
        actor_id = self.scion_client.define_user(first_name="John",
                                                 last_name="Doe",
                                                 username="******",
                                                 password="******",
                                                 email="*****@*****.**")

        # Create instrument
        agent_info = [
            dict(agent_type="data_agent",
                 config=dict(
                     plugin=
                     "scion.agent.model.cdip.cdip_plugin.CDIP_DataAgentPlugin",
                     sampling_interval=10,
                     stream_name="basic_streams",
                     auto_streaming=False))
        ]
        inst_obj = Instrument(name="TA_121A/MGENC/M40",
                              description="CDIP buoy data",
                              location=GeospatialLocation(
                                  latitude=37.94831666666667,
                                  longitude=-123.4675),
                              agent_info=agent_info)
        inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)

        # Create dataset
        schema_def = DataSchemaParser.parse_schema_ref("ds_cdip01_main")
        ds_obj = Dataset(name="Dataset Sensor", schema_definition=schema_def)
        ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)

        self.rr.create_association(inst_id, PRED.hasDataset, ds_id)

        ds_filename = self.container.file_system.get(
            "%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
        self.assertFalse(os.path.exists(ds_filename))

        inst_data_t0 = self.scion_client.get_asset_data(inst_id)
        self.assertEquals(inst_data_t0["dataset_id"], ds_id)
        self.assertEquals(inst_data_t0["num_rows"], 0)

        # Install a data packet catcher
        self.recv_packets, self.recv_rows = [], 0

        def process_packet_cb(packet, route, stream):
            if not isinstance(packet, DataPacket):
                log.warn("Received a non DataPacket message")
            self.recv_packets.append(packet)
            self.recv_rows += len(packet.data["data"])
            log.info("Received data packet #%s: rows=%s, cols=%s",
                     len(self.recv_packets), len(packet.data["data"]),
                     packet.data["cols"])
            #log.info('Packet data: ' + str(packet.data))
        def cleanup_stream_sub():
            if self.stream_sub:
                self.stream_sub.stop()
                self.stream_sub = None

        self.stream_sub = StreamSubscriber(process=self.scion_proc,
                                           stream="basic_streams",
                                           callback=process_packet_cb)
        self.stream_sub.start()

        self.addCleanup(cleanup_stream_sub)

        # Start agent
        self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))

        agent_pid = self.scion_client.start_agent(inst_id)

        self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))

        sac = StreamingAgentClient(resource_id=inst_id,
                                   process=self.scion_proc)
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"],
                          StreamingAgent.AGENTSTATE_INITIALIZED)

        sac.connect()
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"],
                          StreamingAgent.AGENTSTATE_CONNECTED)

        # Coming in from the agent config.
        streaming_args = {
            'url': 'http://cdip.ucsd.edu/data_access/justdar.cdip?029+pm',
            'sampling_interval': 10
        }

        sac.start_streaming(streaming_args)
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"],
                          StreamingAgent.AGENTSTATE_STREAMING)

        # Set to progressively high values for real data stream tests.
        gevent.sleep(20)

        # Retrieve data
        self.assertTrue(os.path.exists(ds_filename))

        inst_data = self.scion_client.get_asset_data(inst_id)
        """
        {'data': {'Dp': [[1465682100000, 325]],
            'Hs': [[1465682100000, 3.03]],
            'Ta': [[1465682100000, 6.92]],
            'Temp': [[1465682100000, 12.2]],
            'Tp': [[1465682100000, 9.09]]},
         'dataset_id': '08bc829159e6401182462b713b180dbe',
         'num_rows': 1,
         'ts_generated': '1465685467675',
         'var_def': [{'base_type': 'ntp_time',
              'description': 'NTPv4 timestamp',
              'name': 'time',
              'storage_dtype': 'i8',
              'unit': ''},
             {'base_type': 'float',
              'description': 'Significant wave height',
              'name': 'Hs',
              'storage_dtype': 'f8',
              'unit': 'meters'},
             {'base_type': 'float',
              'description': 'Peak wave period',
              'name': 'Tp',
              'storage_dtype': 'f8',
              'unit': 'seconds'},
             {'base_type': 'int',
              'description': 'Peak wave direction',
              'name': 'Dp',
              'storage_dtype': 'i4',
              'unit': 'degrees'},
             {'base_type': 'float',
              'description': 'Average wave period',
              'name': 'Ta',
              'storage_dtype': 'f8',
              'unit': 'seconds'},
             {'base_type': 'float',
              'description': 'Surface temperature',
              'name': 'Temp',
              'storage_dtype': 'f8',
              'unit': 'celcius'}],
         'variables': ['time', 'Hs', 'Tp', 'Dp', 'Ta', 'Temp']}
        """
        num_rows = inst_data["num_rows"]
        log.info('CDIP test produced %i data rows.' % num_rows)

        # Take down agent
        sac.stop_streaming()  # Not required to stop agent, just to test here
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"],
                          StreamingAgent.AGENTSTATE_CONNECTED)

        sac.disconnect()
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"],
                          StreamingAgent.AGENTSTATE_INITIALIZED)

        self.scion_client.stop_agent(inst_id)

        self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
Esempio n. 3
0
class TestScionOrbAgentData(IonIntegrationTestCase):
    """Test for Scion with agents and data streaming
    """
    def setUp(self):
        self._start_container()
        self.patch_alt_cfg('scion.process.preload.preloader.CFG',
                           {'scion': {
                               'preload': {
                                   'enabled': False
                               }
                           }})
        gevent.sleep(1)  # Avoid address in use error
        self.container.start_rel_from_url('res/deploy/scion.yml')

        self.rr = self.container.resource_registry
        self.scion_client = ScionManagementClient()
        self.idm_client = IdentityManagementServiceClient()
        self.system_actor_id = None

        self.ui_server_proc = self.container.proc_manager.procs_by_name[
            "ui_server"]
        self.scion_proc = self.container.proc_manager.procs_by_name[
            "scion_management"]
        self.ui_base_url = self.ui_server_proc.base_url
        self.sg_base_url = self.ui_server_proc.gateway_base_url

    def tearDown(self):
        pass

    @unittest.skipIf(getpass.getuser() == "jenkins", "Disabled on Jenkins")
    def test_scion_agent(self):
        # Create user
        actor_id = self.scion_client.define_user(first_name="John",
                                                 last_name="Doe",
                                                 username="******",
                                                 password="******",
                                                 email="*****@*****.**")

        inst_ids = []
        ds_ids = []
        for source in sources:

            # Instrument
            agent_info = [
                dict(
                    agent_type="data_agent",
                    config=dict(
                        plugin=
                        "scion.agent.model.orb.orb_plugin.Orb_DataAgentPlugin",
                        sampling_interval=0.5,
                        stream_name="basic_streams",
                        auto_streaming=False))
            ]
            inst_obj = Instrument(
                name=source,
                description="Multiplexed generic compressed data frame packet",
                location=GeospatialLocation(latitude=42.867079,
                                            longitude=-127.257324),
                agent_info=agent_info)
            inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)
            inst_ids.append(inst_id)

            # Dataset
            schema_def = DataSchemaParser.parse_schema_ref("ds_orb_mgenc_m40")
            ds_obj = Dataset(name=source, schema_definition=schema_def)
            ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)
            self.rr.create_association(inst_id, PRED.hasDataset, ds_id)

            ds_filename = self.container.file_system.get(
                "%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
            self.assertFalse(os.path.exists(ds_filename))

            inst_data_t0 = self.scion_client.get_asset_data(inst_id)
            self.assertEquals(inst_data_t0["dataset_id"], ds_id)
            self.assertEquals(inst_data_t0["num_rows"], 0)
            ds_ids.append(ds_id)

        # Install a data packet catcher
        # TODO

        # Start agent
        sacs = []
        for idx, source in enumerate(sources):
            self.assertFalse(
                StreamingAgentClient.is_agent_active(inst_ids[idx]))
            agent_pid = self.scion_client.start_agent(inst_ids[idx])
            self.assertTrue(StreamingAgentClient.is_agent_active(
                inst_ids[idx]))

            sac = StreamingAgentClient(resource_id=inst_ids[idx],
                                       process=self.scion_proc)
            agent_status = sac.get_status()
            self.assertEquals(agent_status["current_state"],
                              StreamingAgent.AGENTSTATE_INITIALIZED)

            sac.connect()
            agent_status = sac.get_status()
            self.assertEquals(agent_status["current_state"],
                              StreamingAgent.AGENTSTATE_CONNECTED)

            # Coming in from the agent config.
            streaming_args = {
                #'orb_name' : 'taexport.ucsd.edu:usarrayTA',
                'orb_name': 'ceusnexport.ucsd.edu:usarray',
                'select': source,
                '--timeout': 5,
                'sample_interval': 5
            }

            sac.start_streaming(streaming_args)
            agent_status = sac.get_status()
            self.assertEquals(agent_status["current_state"],
                              StreamingAgent.AGENTSTATE_STREAMING)
            sacs.append(sac)

        gevent.sleep(120)

        # Take down agent
        for idx, sac in enumerate(sacs):
            #sac.stop_streaming()  # Not required to stop agent, just to test here
            #agent_status = sac.get_status()
            #self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)

            #sac.disconnect()
            #agent_status = sac.get_status()
            #self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)

            self.scion_client.stop_agent(inst_ids[idx])
            self.assertFalse(
                StreamingAgentClient.is_agent_active(inst_ids[idx]))
Esempio n. 4
0
class TestIngestOrbData(IonIntegrationTestCase):
    """Test for ingestion of ORB structured packets
    """

    def setUp(self):
        self._start_container()
        self.patch_alt_cfg('scion.process.preload.preloader.CFG',
                           {'scion': {'preload': {'enabled': False}}})
        self.container.start_rel_from_url('res/deploy/scion.yml')

        self.rr = self.container.resource_registry
        self.scion_client = ScionManagementClient()
        self.idm_client = IdentityManagementServiceClient()
        self.system_actor_id = None

        self.ui_server_proc = self.container.proc_manager.procs_by_name["ui_server"]
        self.scion_proc = self.container.proc_manager.procs_by_name["scion_management"]
        self.ui_base_url = self.ui_server_proc.base_url
        self.sg_base_url = self.ui_server_proc.gateway_base_url

    def tearDown(self):
        pass

    def test_scion_agent(self):
        # Create user
        actor_id = self.scion_client.define_user(
                first_name="John", last_name="Doe",
                username="******", password="******", email="*****@*****.**")

        # Create instrument
        data_filename = os.path.join(os.path.split(__file__)[0], "testdata/orb_replay_data.yml")
        agent_info=[dict(agent_type="data_agent",
                         config=dict(plugin="scion.agent.model.orb.orb_replay_plugin.ORBReplay_DataAgentPlugin",
                                     sampling_interval=0.2, stream_name="basic_streams",
                                     replay_file=data_filename, replay_scenario="basic",
                                     auto_streaming=False))]
        inst_obj = Instrument(name="ORB Sensor 1", description="Seismic stream",
                              location=GeospatialLocation(latitude=32.867079, longitude=-117.257324),
                              agent_info=agent_info)
        inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)

        # Create dataset
        schema_def = DataSchemaParser.parse_schema_ref("ds_orb01_main")
        ds_obj = Dataset(name="ORB Dataset 1",
                         schema_definition=schema_def)
        ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)

        self.rr.create_association(inst_id, PRED.hasDataset, ds_id)

        ds_filename = self.container.file_system.get("%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))

        # Install a data packet catcher
        self.recv_packets, self.recv_rows = [], 0
        def process_packet_cb(packet, route, stream):
            if not isinstance(packet, DataPacket):
                log.warn("Received a non DataPacket message")
            self.recv_packets.append(packet)
            self.recv_rows += len(packet.data["data"])
            log.info("Received data packet #%s: rows=%s, cols=%s", len(self.recv_packets), len(packet.data["data"]),
                     packet.data["cols"])
        def cleanup_stream_sub():
            if self.stream_sub:
                self.stream_sub.stop()
                self.stream_sub = None

        self.stream_sub = StreamSubscriber(process=self.scion_proc, stream="basic_streams", callback=process_packet_cb)
        self.stream_sub.start()

        self.addCleanup(cleanup_stream_sub)

        # Start agent
        agent_pid = self.scion_client.start_agent(inst_id)

        self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))

        sac = StreamingAgentClient(resource_id=inst_id, process=self.scion_proc)
        sac.connect()
        sac.start_streaming()
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_STREAMING)

        # Retrieve data
        gevent.sleep(1.5)
        self.assertTrue(os.path.exists(ds_filename))
        inst_data_t2 = self.scion_client.get_asset_data(inst_id)
        print "T2", inst_data_t2

        num_rows_t2 = inst_data_t2["num_rows"]
        self.assertEquals(num_rows_t2, 40)
        sample_data = inst_data_t2["data"]["sample_vector"]
        self.assertEquals(sample_data[0][1], 100)
        self.assertEquals(sample_data[39][1], 409)
        self.assertEquals(sample_data[1][0] - sample_data[0][0], 100)
        self.assertEquals(sample_data[39][0] - sample_data[0][0], 3900)

        # Take down agent
        sac.stop_streaming()  # Not required to stop agent, just to test here
        sac.disconnect()
        self.scion_client.stop_agent(inst_id)

        self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))

        inst_data = self.scion_client.get_asset_data(inst_id, data_filter=dict(max_rows=20))
        self.assertEquals(len(inst_data["data"]["sample_vector"]), 20)
        self.assertEquals(inst_data["data"]["sample_vector"][0][1], 300)

        inst_data = self.scion_client.get_asset_data(inst_id, data_filter=dict(start_time="1460694001000"))
        self.assertEquals(len(inst_data["data"]["sample_vector"]), 30)
        self.assertEquals(inst_data["data"]["sample_vector"][0][1], 200)

        # Note: Time filter applies before expansion
        inst_data = self.scion_client.get_asset_data(inst_id, data_filter=dict(start_time="1460694001000", start_time_include=False))
        self.assertEquals(len(inst_data["data"]["sample_vector"]), 20)
        self.assertEquals(inst_data["data"]["sample_vector"][0][1], 300)

        inst_data = self.scion_client.get_asset_data(inst_id, data_filter=dict(start_time="1460694001000", max_rows=10))
        self.assertEquals(len(inst_data["data"]["sample_vector"]), 10)
        self.assertEquals(inst_data["data"]["sample_vector"][0][1], 400)

        inst_data = self.scion_client.get_asset_data(inst_id, data_filter=dict(start_time="1460694001000", end_time="1460694002000"))
        print inst_data
        self.assertEquals(len(inst_data["data"]["sample_vector"]), 20)
        self.assertEquals(inst_data["data"]["sample_vector"][0][1], 200)
Esempio n. 5
0
class TestScionOrbAgentData(IonIntegrationTestCase):
    """Test for Scion with agents and data streaming
    """

    def setUp(self):
        self._start_container()
        self.patch_alt_cfg('scion.process.preload.preloader.CFG',
                           {'scion': {'preload': {'enabled': False}}})
        self.container.start_rel_from_url('res/deploy/scion.yml')

        self.rr = self.container.resource_registry
        self.scion_client = ScionManagementClient()
        self.idm_client = IdentityManagementServiceClient()
        self.system_actor_id = None

        self.ui_server_proc = self.container.proc_manager.procs_by_name["ui_server"]
        self.scion_proc = self.container.proc_manager.procs_by_name["scion_management"]
        self.ui_base_url = self.ui_server_proc.base_url
        self.sg_base_url = self.ui_server_proc.gateway_base_url

    def tearDown(self):
        pass

    @unittest.skipIf(getpass.getuser() == "jenkins", "Disabled on Jenkins")
    def test_scion_agent(self):
        # Create user
        actor_id = self.scion_client.define_user(
                first_name="John", last_name="Doe",
                username="******", password="******", email="*****@*****.**")

        # Create instrument
        agent_info=[dict(agent_type="data_agent",
                         config=dict(plugin="scion.agent.model.orb.orb_plugin.Orb_DataAgentPlugin",
                                     sampling_interval=0.5, stream_name="basic_streams",
                                     auto_streaming=False))]
        inst_obj = Instrument(name="TA_121A/MGENC/M40", description="Multiplexed generic compressed data frame packet",
                              location=GeospatialLocation(latitude=42.867079, longitude=-127.257324),
                              agent_info=agent_info)
        inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)

        # Create dataset
        schema_def = DataSchemaParser.parse_schema_ref("ds_orb_mgenc_m40")
        ds_obj = Dataset(name="Dataset Sensor 3",
                         schema_definition=schema_def)
        ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)

        self.rr.create_association(inst_id, PRED.hasDataset, ds_id)

        ds_filename = self.container.file_system.get("%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
        self.assertFalse(os.path.exists(ds_filename))

        inst_data_t0 = self.scion_client.get_asset_data(inst_id)
        self.assertEquals(inst_data_t0["dataset_id"], ds_id)
        self.assertEquals(inst_data_t0["num_rows"], 0)

        # Install a data packet catcher
        self.recv_packets, self.recv_rows = [], 0
        def process_packet_cb(packet, route, stream):
            if not isinstance(packet, DataPacket):
                log.warn("Received a non DataPacket message")
            self.recv_packets.append(packet)
            self.recv_rows += len(packet.data["data"])
            log.info("Received data packet #%s: rows=%s, cols=%s", len(self.recv_packets), len(packet.data["data"]),
                     packet.data["cols"])
            #log.info('Packet data: ' + str(packet.data))
        def cleanup_stream_sub():
            if self.stream_sub:
                self.stream_sub.stop()
                self.stream_sub = None

        self.stream_sub = StreamSubscriber(process=self.scion_proc, stream="basic_streams", callback=process_packet_cb)
        self.stream_sub.start()
        
        self.addCleanup(cleanup_stream_sub)

        # Start agent
        self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))

        agent_pid = self.scion_client.start_agent(inst_id)

        self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))

       
        sac = StreamingAgentClient(resource_id=inst_id, process=self.scion_proc)
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)

        sac.connect()
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)

        # Coming in from the agent config.
        streaming_args = { 
            'orb_name' : 'taexport.ucsd.edu:usarrayTA',
            #'select' : 'TA_109C/MGENC/M40',
            'select' : 'TA_121A/MGENC/M40',
            '--timeout' : 5,
            'sample_interval' : 5 
        }

        sac.start_streaming(streaming_args)
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_STREAMING)

        gevent.sleep(180)

        # Retrieve data
        """
        gevent.sleep(1)
        self.assertTrue(os.path.exists(ds_filename))

        inst_data_t1 = self.scion_client.get_asset_data(inst_id)
        self.assertEquals(inst_data_t1["dataset_id"], ds_id)
        self.assertEquals(inst_data_t1["variables"], ['time', 'cpu_percent'])
        self.assertIn("cpu_percent", inst_data_t1["data"])
        num_rows_t1 = inst_data_t1["num_rows"]
        self.assertGreaterEqual(num_rows_t1, 2)

        gevent.sleep(1)
        inst_data_t2 = self.scion_client.get_asset_data(inst_id)
        num_rows_t2 = inst_data_t2["num_rows"]
        self.assertGreater(num_rows_t2, num_rows_t1)

        self.assertGreaterEqual(len(self.recv_packets), self.recv_rows)
        self.assertLessEqual(abs(self.recv_rows - num_rows_t2), 2)
        """

        # Take down agent
        sac.stop_streaming()  # Not required to stop agent, just to test here
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)

        sac.disconnect()
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)

        self.scion_client.stop_agent(inst_id)

        self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
Esempio n. 6
0
class TestIngestOrbData(IonIntegrationTestCase):
    """Test for ingestion of ORB structured packets
    """
    def setUp(self):
        self._start_container()
        self.patch_alt_cfg('scion.process.preload.preloader.CFG',
                           {'scion': {
                               'preload': {
                                   'enabled': False
                               }
                           }})
        self.container.start_rel_from_url('res/deploy/scion.yml')

        self.rr = self.container.resource_registry
        self.scion_client = ScionManagementClient()
        self.idm_client = IdentityManagementServiceClient()
        self.system_actor_id = None

        self.ui_server_proc = self.container.proc_manager.procs_by_name[
            "ui_server"]
        self.scion_proc = self.container.proc_manager.procs_by_name[
            "scion_management"]
        self.ui_base_url = self.ui_server_proc.base_url
        self.sg_base_url = self.ui_server_proc.gateway_base_url

    def tearDown(self):
        pass

    def test_scion_agent(self):
        # Create user
        actor_id = self.scion_client.define_user(first_name="John",
                                                 last_name="Doe",
                                                 username="******",
                                                 password="******",
                                                 email="*****@*****.**")

        # Create instrument
        data_filename = os.path.join(
            os.path.split(__file__)[0], "testdata/orb_replay_data.yml")
        agent_info = [
            dict(
                agent_type="data_agent",
                config=dict(
                    plugin=
                    "scion.agent.model.orb.orb_replay_plugin.ORBReplay_DataAgentPlugin",
                    sampling_interval=0.2,
                    stream_name="basic_streams",
                    replay_file=data_filename,
                    replay_scenario="basic",
                    auto_streaming=False))
        ]
        inst_obj = Instrument(name="ORB Sensor 1",
                              description="Seismic stream",
                              location=GeospatialLocation(
                                  latitude=32.867079, longitude=-117.257324),
                              agent_info=agent_info)
        inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)

        # Create dataset
        schema_def = DataSchemaParser.parse_schema_ref("ds_orb01_main")
        ds_obj = Dataset(name="ORB Dataset 1", schema_definition=schema_def)
        ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)

        self.rr.create_association(inst_id, PRED.hasDataset, ds_id)

        ds_filename = self.container.file_system.get(
            "%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))

        # Install a data packet catcher
        self.recv_packets, self.recv_rows = [], 0

        def process_packet_cb(packet, route, stream):
            if not isinstance(packet, DataPacket):
                log.warn("Received a non DataPacket message")
            self.recv_packets.append(packet)
            self.recv_rows += len(packet.data["data"])
            log.info("Received data packet #%s: rows=%s, cols=%s",
                     len(self.recv_packets), len(packet.data["data"]),
                     packet.data["cols"])

        def cleanup_stream_sub():
            if self.stream_sub:
                self.stream_sub.stop()
                self.stream_sub = None

        self.stream_sub = StreamSubscriber(process=self.scion_proc,
                                           stream="basic_streams",
                                           callback=process_packet_cb)
        self.stream_sub.start()

        self.addCleanup(cleanup_stream_sub)

        # Start agent
        agent_pid = self.scion_client.start_agent(inst_id)

        self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))

        sac = StreamingAgentClient(resource_id=inst_id,
                                   process=self.scion_proc)
        sac.connect()
        sac.start_streaming()
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"],
                          StreamingAgent.AGENTSTATE_STREAMING)

        # Retrieve data
        gevent.sleep(1.5)
        self.assertTrue(os.path.exists(ds_filename))
        inst_data_t2 = self.scion_client.get_asset_data(inst_id)
        print "T2", inst_data_t2

        num_rows_t2 = inst_data_t2["num_rows"]
        self.assertEquals(num_rows_t2, 40)
        sample_data = inst_data_t2["data"]["sample_vector"]
        self.assertEquals(sample_data[0][1], 100)
        self.assertEquals(sample_data[39][1], 409)
        self.assertEquals(sample_data[1][0] - sample_data[0][0], 100)
        self.assertEquals(sample_data[39][0] - sample_data[0][0], 3900)

        # Take down agent
        sac.stop_streaming()  # Not required to stop agent, just to test here
        sac.disconnect()
        self.scion_client.stop_agent(inst_id)

        self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))

        inst_data = self.scion_client.get_asset_data(
            inst_id, data_filter=dict(max_rows=20))
        self.assertEquals(len(inst_data["data"]["sample_vector"]), 20)
        self.assertEquals(inst_data["data"]["sample_vector"][0][1], 300)

        inst_data = self.scion_client.get_asset_data(
            inst_id, data_filter=dict(start_time="1460694001000"))
        self.assertEquals(len(inst_data["data"]["sample_vector"]), 30)
        self.assertEquals(inst_data["data"]["sample_vector"][0][1], 200)

        # Note: Time filter applies before expansion
        inst_data = self.scion_client.get_asset_data(
            inst_id,
            data_filter=dict(start_time="1460694001000",
                             start_time_include=False))
        self.assertEquals(len(inst_data["data"]["sample_vector"]), 20)
        self.assertEquals(inst_data["data"]["sample_vector"][0][1], 300)

        inst_data = self.scion_client.get_asset_data(
            inst_id, data_filter=dict(start_time="1460694001000", max_rows=10))
        self.assertEquals(len(inst_data["data"]["sample_vector"]), 10)
        self.assertEquals(inst_data["data"]["sample_vector"][0][1], 400)

        inst_data = self.scion_client.get_asset_data(
            inst_id,
            data_filter=dict(start_time="1460694001000",
                             end_time="1460694002000"))
        print inst_data
        self.assertEquals(len(inst_data["data"]["sample_vector"]), 20)
        self.assertEquals(inst_data["data"]["sample_vector"][0][1], 200)
Esempio n. 7
0
class TestScionCDIPAgentData(IonIntegrationTestCase):
    """Test for Scion with agents streaming data from CDIP source
    """

    def setUp(self):
        self._start_container()
        self.patch_alt_cfg('scion.process.preload.preloader.CFG',
                           {'scion': {'preload': {'enabled': False}}})
        self.container.start_rel_from_url('res/deploy/scion.yml')

        self.rr = self.container.resource_registry
        self.scion_client = ScionManagementClient()
        self.idm_client = IdentityManagementServiceClient()
        self.system_actor_id = None

        self.ui_server_proc = self.container.proc_manager.procs_by_name["ui_server"]
        self.scion_proc = self.container.proc_manager.procs_by_name["scion_management"]
        self.ui_base_url = self.ui_server_proc.base_url
        self.sg_base_url = self.ui_server_proc.gateway_base_url

    def tearDown(self):
        pass

    def test_scion_agent(self):
        # Create user
        actor_id = self.scion_client.define_user(
                first_name="John", last_name="Doe",
                username="******", password="******", email="*****@*****.**")

        # Create instrument
        agent_info=[dict(agent_type="data_agent",
                         config=dict(plugin="scion.agent.model.cdip.cdip_plugin.CDIP_DataAgentPlugin",
                                     sampling_interval=10, stream_name="basic_streams",
                                     auto_streaming=False))]
        inst_obj = Instrument(name="TA_121A/MGENC/M40", description="CDIP buoy data",
                              location=GeospatialLocation(latitude=37.94831666666667, longitude=-123.4675),
                              agent_info=agent_info)
        inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)

        # Create dataset
        schema_def = DataSchemaParser.parse_schema_ref("ds_cdip01_main")
        ds_obj = Dataset(name="Dataset Sensor",
                         schema_definition=schema_def)
        ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)

        self.rr.create_association(inst_id, PRED.hasDataset, ds_id)

        ds_filename = self.container.file_system.get("%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
        self.assertFalse(os.path.exists(ds_filename))

        inst_data_t0 = self.scion_client.get_asset_data(inst_id)
        self.assertEquals(inst_data_t0["dataset_id"], ds_id)
        self.assertEquals(inst_data_t0["num_rows"], 0)

        # Install a data packet catcher
        self.recv_packets, self.recv_rows = [], 0
        def process_packet_cb(packet, route, stream):
            if not isinstance(packet, DataPacket):
                log.warn("Received a non DataPacket message")
            self.recv_packets.append(packet)
            self.recv_rows += len(packet.data["data"])
            log.info("Received data packet #%s: rows=%s, cols=%s", len(self.recv_packets), len(packet.data["data"]),
                     packet.data["cols"])
            #log.info('Packet data: ' + str(packet.data))
        def cleanup_stream_sub():
            if self.stream_sub:
                self.stream_sub.stop()
                self.stream_sub = None

        self.stream_sub = StreamSubscriber(process=self.scion_proc, stream="basic_streams", callback=process_packet_cb)
        self.stream_sub.start()
        
        self.addCleanup(cleanup_stream_sub)

        # Start agent
        self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))

        agent_pid = self.scion_client.start_agent(inst_id)

        self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))

       
        sac = StreamingAgentClient(resource_id=inst_id, process=self.scion_proc)
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)

        sac.connect()
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)

        # Coming in from the agent config.
        streaming_args = { 
            'url' : 'http://cdip.ucsd.edu/data_access/justdar.cdip?029+pm',
            'sampling_interval' : 10
        }

        sac.start_streaming(streaming_args)
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_STREAMING)

        # Set to progressively high values for real data stream tests.
        gevent.sleep(20)

        # Retrieve data
        self.assertTrue(os.path.exists(ds_filename))

        inst_data = self.scion_client.get_asset_data(inst_id)
        """
        {'data': {'Dp': [[1465682100000, 325]],
            'Hs': [[1465682100000, 3.03]],
            'Ta': [[1465682100000, 6.92]],
            'Temp': [[1465682100000, 12.2]],
            'Tp': [[1465682100000, 9.09]]},
         'dataset_id': '08bc829159e6401182462b713b180dbe',
         'num_rows': 1,
         'ts_generated': '1465685467675',
         'var_def': [{'base_type': 'ntp_time',
              'description': 'NTPv4 timestamp',
              'name': 'time',
              'storage_dtype': 'i8',
              'unit': ''},
             {'base_type': 'float',
              'description': 'Significant wave height',
              'name': 'Hs',
              'storage_dtype': 'f8',
              'unit': 'meters'},
             {'base_type': 'float',
              'description': 'Peak wave period',
              'name': 'Tp',
              'storage_dtype': 'f8',
              'unit': 'seconds'},
             {'base_type': 'int',
              'description': 'Peak wave direction',
              'name': 'Dp',
              'storage_dtype': 'i4',
              'unit': 'degrees'},
             {'base_type': 'float',
              'description': 'Average wave period',
              'name': 'Ta',
              'storage_dtype': 'f8',
              'unit': 'seconds'},
             {'base_type': 'float',
              'description': 'Surface temperature',
              'name': 'Temp',
              'storage_dtype': 'f8',
              'unit': 'celcius'}],
         'variables': ['time', 'Hs', 'Tp', 'Dp', 'Ta', 'Temp']}
        """
        num_rows = inst_data["num_rows"]
        log.info('CDIP test produced %i data rows.' % num_rows)

        # Take down agent
        sac.stop_streaming()  # Not required to stop agent, just to test here
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)

        sac.disconnect()
        agent_status = sac.get_status()
        self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)

        self.scion_client.stop_agent(inst_id)

        self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
class TestScionOrbAgentData(IonIntegrationTestCase):
    """Test for Scion with agents and data streaming
    """

    def setUp(self):
        self._start_container()
        self.patch_alt_cfg('scion.process.preload.preloader.CFG',
                           {'scion': {'preload': {'enabled': False}}})
        gevent.sleep(1)  # Avoid address in use error
        self.container.start_rel_from_url('res/deploy/scion.yml')

        self.rr = self.container.resource_registry
        self.scion_client = ScionManagementClient()
        self.idm_client = IdentityManagementServiceClient()
        self.system_actor_id = None

        self.ui_server_proc = self.container.proc_manager.procs_by_name["ui_server"]
        self.scion_proc = self.container.proc_manager.procs_by_name["scion_management"]
        self.ui_base_url = self.ui_server_proc.base_url
        self.sg_base_url = self.ui_server_proc.gateway_base_url

    def tearDown(self):
        pass

    @unittest.skipIf(getpass.getuser() == "jenkins", "Disabled on Jenkins")
    def test_scion_agent(self):
        # Create user
        actor_id = self.scion_client.define_user(
                first_name="John", last_name="Doe",
                username="******", password="******", email="*****@*****.**")

        inst_ids = []
        ds_ids = []
        for source in sources:
   
          # Instrument
          agent_info=[dict(agent_type="data_agent",
            config=dict(plugin="scion.agent.model.orb.orb_plugin.Orb_DataAgentPlugin",
            sampling_interval=0.5, stream_name="basic_streams", auto_streaming=False))]
          inst_obj = Instrument(name=source, description="Multiplexed generic compressed data frame packet",
            location=GeospatialLocation(latitude=42.867079, longitude=-127.257324), agent_info=agent_info)
          inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)
          inst_ids.append(inst_id)

          # Dataset
          schema_def = DataSchemaParser.parse_schema_ref("ds_orb_mgenc_m40")
          ds_obj = Dataset(name=source, schema_definition=schema_def)
          ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)
          self.rr.create_association(inst_id, PRED.hasDataset, ds_id)

          ds_filename = self.container.file_system.get("%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
          self.assertFalse(os.path.exists(ds_filename))

          inst_data_t0 = self.scion_client.get_asset_data(inst_id)
          self.assertEquals(inst_data_t0["dataset_id"], ds_id)
          self.assertEquals(inst_data_t0["num_rows"], 0)
          ds_ids.append(ds_id)

        # Install a data packet catcher
        # TODO

        # Start agent
        sacs = []
        for idx, source in enumerate(sources):
          self.assertFalse(StreamingAgentClient.is_agent_active(inst_ids[idx]))
          agent_pid = self.scion_client.start_agent(inst_ids[idx])
          self.assertTrue(StreamingAgentClient.is_agent_active(inst_ids[idx]))

          sac = StreamingAgentClient(resource_id=inst_ids[idx], process=self.scion_proc)
          agent_status = sac.get_status()
          self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)

          sac.connect()
          agent_status = sac.get_status()
          self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)

          # Coming in from the agent config.
          streaming_args = { 
              #'orb_name' : 'taexport.ucsd.edu:usarrayTA',
              'orb_name' : 'ceusnexport.ucsd.edu:usarray',
              'select' : source,
              '--timeout' : 5,
              'sample_interval' : 5 
          }

          sac.start_streaming(streaming_args)
          agent_status = sac.get_status()
          self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_STREAMING)
          sacs.append(sac)

        gevent.sleep(120)

        # Take down agent
        for idx, sac in enumerate(sacs):
          #sac.stop_streaming()  # Not required to stop agent, just to test here
          #agent_status = sac.get_status()
          #self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)

          #sac.disconnect()
          #agent_status = sac.get_status()
          #self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)

          self.scion_client.stop_agent(inst_ids[idx])
          self.assertFalse(StreamingAgentClient.is_agent_active(inst_ids[idx]))