def on_init(self):
self.ingestion_profile = self.CFG.get_safe(CONFIG_KEY + ".ingestion_profile", "default")
log.info("Ingestion starting using profile '%s'", self.ingestion_profile)
self.exchange_name = "ingestion_process"
self.ingestion_config = self.CFG.get_safe(CONFIG_KEY + ".profile_" + self.ingestion_profile) or {}
if not self.ingestion_config:
raise BadRequest("No config found for profile '%s'" % self.ingestion_profile)
plugin_cls = get_safe(self.ingestion_config, "plugin")
self.plugin = named_any(plugin_cls)(self)
log.info("Started ingestion plugin '%s'", plugin_cls)
self.persistence_formats = {}
self.persistence_objects = {}
self.default_persistence_format = get_safe(self.ingestion_config, "persist.persistence_format")
self._require_persistence_layer(self.default_persistence_format)
self.stream_sub = StreamSubscriber(process=self, exchange_name=self.exchange_name,
callback=self.process_package)
streams = get_safe(self.ingestion_config, "stream_subscriptions") or []
for stream in streams:
if isinstance(stream, list):
stream = StreamRoute(exchange_point=stream[0], routing_key=stream[1])
log.info("Ingestion subscribed to stream '%s'", stream)
self.stream_sub.add_stream_subscription(stream)
self.plugin.on_init()
self.stream_sub.start()
def on_init(self):
self.ingestion_profile = self.CFG.get_safe(CONFIG_KEY + ".ingestion_profile", "default")
log.info("Ingestion starting using profile '%s'", self.ingestion_profile)
self.exchange_name = "ingestion_process"
self.ingestion_config = self.CFG.get_safe(CONFIG_KEY + ".profile_" + self.ingestion_profile) or {}
if not self.ingestion_config:
raise BadRequest("No config found for profile '%s'" % self.ingestion_profile)
plugin_cls = get_safe(self.ingestion_config, "plugin")
self.plugin = named_any(plugin_cls)(self)
log.info("Started ingestion plugin '%s'", plugin_cls)
self.persistence_formats = {}
self.persistence_objects = {}
self.default_persistence_format = get_safe(self.ingestion_config, "persist.persistence_format")
self._require_persistence_layer(self.default_persistence_format)
self.stream_sub = StreamSubscriber(
process=self, exchange_name=self.exchange_name, callback=self.process_package
)
streams = get_safe(self.ingestion_config, "stream_subscriptions") or []
for stream in streams:
if isinstance(stream, list):
stream = StreamRoute(exchange_point=stream[0], routing_key=stream[1])
log.info("Ingestion subscribed to stream '%s'", stream)
self.stream_sub.add_stream_subscription(stream)
self.plugin.on_init()
self.stream_sub.start()
def on_init(self):
log.info("Ingestion starting")
self.exchange_name = "ingestion_process"
plugin_cls = CFG.get_safe(CONFIG_KEY + ".plugin")
self.plugin = named_any(plugin_cls)(self)
log.info("Started ingestion plugin '%s'", plugin_cls)
self.persistence_formats = {}
self.persistence_objects = {}
self.default_persistence_format = CFG.get_safe(CONFIG_KEY + ".persist.persistence_format")
self._require_persistence_layer(self.default_persistence_format)
self.stream_sub = StreamSubscriber(process=self, exchange_name=self.exchange_name,
callback=self.process_package)
streams = CFG.get_safe(CONFIG_KEY + ".stream_subscriptions") or []
for stream in streams:
if isinstance(stream, list):
stream = StreamRoute(exchange_point=stream[0], routing_key=stream[1])
log.info("Ingestion subscribed to stream '%s'", stream)
self.stream_sub.add_stream_subscription(stream)
self.plugin.on_init()
self.stream_sub.start()
class IngestionProcess(StandaloneProcess):
def on_init(self):
self.ingestion_profile = self.CFG.get_safe(CONFIG_KEY + ".ingestion_profile", "default")
log.info("Ingestion starting using profile '%s'", self.ingestion_profile)
self.exchange_name = "ingestion_process"
self.ingestion_config = self.CFG.get_safe(CONFIG_KEY + ".profile_" + self.ingestion_profile) or {}
if not self.ingestion_config:
raise BadRequest("No config found for profile '%s'" % self.ingestion_profile)
plugin_cls = get_safe(self.ingestion_config, "plugin")
self.plugin = named_any(plugin_cls)(self)
log.info("Started ingestion plugin '%s'", plugin_cls)
self.persistence_formats = {}
self.persistence_objects = {}
self.default_persistence_format = get_safe(self.ingestion_config, "persist.persistence_format")
self._require_persistence_layer(self.default_persistence_format)
self.stream_sub = StreamSubscriber(process=self, exchange_name=self.exchange_name,
callback=self.process_package)
streams = get_safe(self.ingestion_config, "stream_subscriptions") or []
for stream in streams:
if isinstance(stream, list):
stream = StreamRoute(exchange_point=stream[0], routing_key=stream[1])
log.info("Ingestion subscribed to stream '%s'", stream)
self.stream_sub.add_stream_subscription(stream)
self.plugin.on_init()
self.stream_sub.start()
def on_quit(self):
self.stream_sub.stop()
def _require_persistence_layer(self, format_name):
if format_name in self.persistence_formats:
return self.persistence_formats[format_name]
if format_name == "hdf5":
from ion.data.persist.hdf5_dataset import DatasetHDF5Persistence
persistence_factory = DatasetHDF5Persistence.get_persistence
self.persistence_formats[format_name] = persistence_factory
else:
raise BadRequest("Unknown persistence format: %s" % format_name)
return self.persistence_formats[format_name]
def process_package(self, packet, route, stream):
if not isinstance(packet, DataPacket):
log.warn("Ingestion received a non DataPacket message")
#print "INGEST", packet, route, stream
try:
ds_info = self.plugin.get_dataset_info(packet)
self._persist_packet(packet, ds_info)
except Exception as ex:
log.exception("Error during ingestion")
def _persist_packet(self, packet, ds_info):
layer_format = ds_info["schema"]["attributes"].get("persistence", {}).get("format", self.default_persistence_format)
ds_persistence = "%s_%s" % (layer_format, ds_info["dataset_id"])
if ds_persistence in self.persistence_objects:
persistence = self.persistence_objects[ds_persistence]
else:
persistence_factory = self._require_persistence_layer(layer_format)
persistence = persistence_factory(ds_info["dataset_id"], ds_info["schema"], layer_format)
self.persistence_objects[ds_persistence] = persistence
persistence.require_dataset()
persistence.extend_dataset(packet)
def test_scion_agent(self):
# Create user
actor_id = self.scion_client.define_user(
first_name="John", last_name="Doe",
username="******", password="******", email="*****@*****.**")
# Create instrument
agent_info=[dict(agent_type="data_agent",
config=dict(plugin="scion.agent.model.vmmon.vmmon_plugin.VMMON_DataAgentPlugin",
sampling_interval=0.5, stream_name="basic_streams",
auto_streaming=False))]
inst_obj = Instrument(name="Sensor 1", description="CPU monitor",
location=GeospatialLocation(latitude=32.867079, longitude=-117.257324),
agent_info=agent_info)
inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)
# Create dataset
schema_def = DataSchemaParser.parse_schema_ref("ds_vmmon_main")
ds_obj = Dataset(name="Dataset Sensor 1",
schema_definition=schema_def)
ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)
self.rr.create_association(inst_id, PRED.hasDataset, ds_id)
ds_filename = self.container.file_system.get("%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
self.assertFalse(os.path.exists(ds_filename))
inst_data_t0 = self.scion_client.get_asset_data(inst_id)
self.assertEquals(inst_data_t0["dataset_id"], ds_id)
self.assertEquals(inst_data_t0["num_rows"], 0)
# Install a data packet catcher
self.recv_packets, self.recv_rows = [], 0
def process_packet_cb(packet, route, stream):
if not isinstance(packet, DataPacket):
log.warn("Received a non DataPacket message")
self.recv_packets.append(packet)
self.recv_rows += len(packet.data["data"])
log.info("Received data packet #%s: rows=%s, cols=%s", len(self.recv_packets), len(packet.data["data"]),
packet.data["cols"])
def cleanup_stream_sub():
if self.stream_sub:
self.stream_sub.stop()
self.stream_sub = None
self.stream_sub = StreamSubscriber(process=self.scion_proc, stream="basic_streams", callback=process_packet_cb)
self.stream_sub.start()
self.addCleanup(cleanup_stream_sub)
# Start agent
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
agent_pid = self.scion_client.start_agent(inst_id)
self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))
sac = StreamingAgentClient(resource_id=inst_id, process=self.scion_proc)
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)
sac.connect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)
sac.start_streaming()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_STREAMING)
# Retrieve data
gevent.sleep(1.1)
self.assertTrue(os.path.exists(ds_filename))
inst_data_t1 = self.scion_client.get_asset_data(inst_id)
self.assertEquals(inst_data_t1["dataset_id"], ds_id)
# self.assertEquals(inst_data_t1["variables"], ['time', 'cpu_percent'])
self.assertIn("cpu_percent", inst_data_t1["data"])
num_rows_t1 = inst_data_t1["num_rows"]
self.assertGreaterEqual(num_rows_t1, 2)
gevent.sleep(1)
inst_data_t2 = self.scion_client.get_asset_data(inst_id)
num_rows_t2 = inst_data_t2["num_rows"]
self.assertGreater(num_rows_t2, num_rows_t1)
self.assertGreaterEqual(len(self.recv_packets), self.recv_rows)
self.assertLessEqual(abs(self.recv_rows - num_rows_t2), 2)
# Take down agent
sac.stop_streaming() # Not required to stop agent, just to test here
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)
sac.disconnect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)
self.scion_client.stop_agent(inst_id)
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
class IngestionProcess(StandaloneProcess):
def on_init(self):
self.ingestion_profile = self.CFG.get_safe(CONFIG_KEY + ".ingestion_profile", "default")
log.info("Ingestion starting using profile '%s'", self.ingestion_profile)
self.exchange_name = "ingestion_process"
self.ingestion_config = self.CFG.get_safe(CONFIG_KEY + ".profile_" + self.ingestion_profile) or {}
if not self.ingestion_config:
raise BadRequest("No config found for profile '%s'" % self.ingestion_profile)
plugin_cls = get_safe(self.ingestion_config, "plugin")
self.plugin = named_any(plugin_cls)(self)
log.info("Started ingestion plugin '%s'", plugin_cls)
self.persistence_formats = {}
self.persistence_objects = {}
self.default_persistence_format = get_safe(self.ingestion_config, "persist.persistence_format")
self._require_persistence_layer(self.default_persistence_format)
self.stream_sub = StreamSubscriber(
process=self, exchange_name=self.exchange_name, callback=self.process_package
)
streams = get_safe(self.ingestion_config, "stream_subscriptions") or []
for stream in streams:
if isinstance(stream, list):
stream = StreamRoute(exchange_point=stream[0], routing_key=stream[1])
log.info("Ingestion subscribed to stream '%s'", stream)
self.stream_sub.add_stream_subscription(stream)
self.plugin.on_init()
self.stream_sub.start()
def on_quit(self):
self.stream_sub.stop()
def _require_persistence_layer(self, format_name):
if format_name in self.persistence_formats:
return self.persistence_formats[format_name]
if format_name == "hdf5":
from ion.data.persist.hdf5_dataset import DatasetHDF5Persistence
persistence_factory = DatasetHDF5Persistence.get_persistence
self.persistence_formats[format_name] = persistence_factory
else:
raise BadRequest("Unknown persistence format: %s" % format_name)
return self.persistence_formats[format_name]
def process_package(self, packet, route, stream):
if not isinstance(packet, DataPacket):
log.warn("Ingestion received a non DataPacket message")
# print "INGEST", packet, route, stream
try:
ds_info = self.plugin.get_dataset_info(packet)
self._persist_packet(packet, ds_info)
except Exception as ex:
log.exception("Error during ingestion")
def _persist_packet(self, packet, ds_info):
layer_format = (
ds_info["schema"]["attributes"].get("persistence", {}).get("format", self.default_persistence_format)
)
ds_persistence = "%s_%s" % (layer_format, ds_info["dataset_id"])
if ds_persistence in self.persistence_objects:
persistence = self.persistence_objects[ds_persistence]
else:
persistence_factory = self._require_persistence_layer(layer_format)
persistence = persistence_factory(ds_info["dataset_id"], ds_info["schema"], layer_format)
self.persistence_objects[ds_persistence] = persistence
persistence.require_dataset()
persistence.extend_dataset(packet)
def test_scion_agent(self):
# Create user
actor_id = self.scion_client.define_user(first_name="John",
last_name="Doe",
username="******",
password="******",
email="*****@*****.**")
# Create instrument
agent_info = [
dict(agent_type="data_agent",
config=dict(
plugin=
"scion.agent.model.cdip.cdip_plugin.CDIP_DataAgentPlugin",
sampling_interval=10,
stream_name="basic_streams",
auto_streaming=False))
]
inst_obj = Instrument(name="TA_121A/MGENC/M40",
description="CDIP buoy data",
location=GeospatialLocation(
latitude=37.94831666666667,
longitude=-123.4675),
agent_info=agent_info)
inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)
# Create dataset
schema_def = DataSchemaParser.parse_schema_ref("ds_cdip01_main")
ds_obj = Dataset(name="Dataset Sensor", schema_definition=schema_def)
ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)
self.rr.create_association(inst_id, PRED.hasDataset, ds_id)
ds_filename = self.container.file_system.get(
"%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
self.assertFalse(os.path.exists(ds_filename))
inst_data_t0 = self.scion_client.get_asset_data(inst_id)
self.assertEquals(inst_data_t0["dataset_id"], ds_id)
self.assertEquals(inst_data_t0["num_rows"], 0)
# Install a data packet catcher
self.recv_packets, self.recv_rows = [], 0
def process_packet_cb(packet, route, stream):
if not isinstance(packet, DataPacket):
log.warn("Received a non DataPacket message")
self.recv_packets.append(packet)
self.recv_rows += len(packet.data["data"])
log.info("Received data packet #%s: rows=%s, cols=%s",
len(self.recv_packets), len(packet.data["data"]),
packet.data["cols"])
#log.info('Packet data: ' + str(packet.data))
def cleanup_stream_sub():
if self.stream_sub:
self.stream_sub.stop()
self.stream_sub = None
self.stream_sub = StreamSubscriber(process=self.scion_proc,
stream="basic_streams",
callback=process_packet_cb)
self.stream_sub.start()
self.addCleanup(cleanup_stream_sub)
# Start agent
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
agent_pid = self.scion_client.start_agent(inst_id)
self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))
sac = StreamingAgentClient(resource_id=inst_id,
process=self.scion_proc)
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_INITIALIZED)
sac.connect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_CONNECTED)
# Coming in from the agent config.
streaming_args = {
'url': 'http://cdip.ucsd.edu/data_access/justdar.cdip?029+pm',
'sampling_interval': 10
}
sac.start_streaming(streaming_args)
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_STREAMING)
# Set to progressively high values for real data stream tests.
gevent.sleep(20)
# Retrieve data
self.assertTrue(os.path.exists(ds_filename))
inst_data = self.scion_client.get_asset_data(inst_id)
"""
{'data': {'Dp': [[1465682100000, 325]],
'Hs': [[1465682100000, 3.03]],
'Ta': [[1465682100000, 6.92]],
'Temp': [[1465682100000, 12.2]],
'Tp': [[1465682100000, 9.09]]},
'dataset_id': '08bc829159e6401182462b713b180dbe',
'num_rows': 1,
'ts_generated': '1465685467675',
'var_def': [{'base_type': 'ntp_time',
'description': 'NTPv4 timestamp',
'name': 'time',
'storage_dtype': 'i8',
'unit': ''},
{'base_type': 'float',
'description': 'Significant wave height',
'name': 'Hs',
'storage_dtype': 'f8',
'unit': 'meters'},
{'base_type': 'float',
'description': 'Peak wave period',
'name': 'Tp',
'storage_dtype': 'f8',
'unit': 'seconds'},
{'base_type': 'int',
'description': 'Peak wave direction',
'name': 'Dp',
'storage_dtype': 'i4',
'unit': 'degrees'},
{'base_type': 'float',
'description': 'Average wave period',
'name': 'Ta',
'storage_dtype': 'f8',
'unit': 'seconds'},
{'base_type': 'float',
'description': 'Surface temperature',
'name': 'Temp',
'storage_dtype': 'f8',
'unit': 'celcius'}],
'variables': ['time', 'Hs', 'Tp', 'Dp', 'Ta', 'Temp']}
"""
num_rows = inst_data["num_rows"]
log.info('CDIP test produced %i data rows.' % num_rows)
# Take down agent
sac.stop_streaming() # Not required to stop agent, just to test here
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_CONNECTED)
sac.disconnect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_INITIALIZED)
self.scion_client.stop_agent(inst_id)
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
def test_scion_agent(self):
# Create user
actor_id = self.scion_client.define_user(
first_name="John", last_name="Doe",
username="******", password="******", email="*****@*****.**")
# Create instrument
data_filename = os.path.join(os.path.split(__file__)[0], "testdata/orb_replay_data.yml")
agent_info=[dict(agent_type="data_agent",
config=dict(plugin="scion.agent.model.orb.orb_replay_plugin.ORBReplay_DataAgentPlugin",
sampling_interval=0.2, stream_name="basic_streams",
replay_file=data_filename, replay_scenario="basic",
auto_streaming=False))]
inst_obj = Instrument(name="ORB Sensor 1", description="Seismic stream",
location=GeospatialLocation(latitude=32.867079, longitude=-117.257324),
agent_info=agent_info)
inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)
# Create dataset
schema_def = DataSchemaParser.parse_schema_ref("ds_orb01_main")
ds_obj = Dataset(name="ORB Dataset 1",
schema_definition=schema_def)
ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)
self.rr.create_association(inst_id, PRED.hasDataset, ds_id)
ds_filename = self.container.file_system.get("%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
# Install a data packet catcher
self.recv_packets, self.recv_rows = [], 0
def process_packet_cb(packet, route, stream):
if not isinstance(packet, DataPacket):
log.warn("Received a non DataPacket message")
self.recv_packets.append(packet)
self.recv_rows += len(packet.data["data"])
log.info("Received data packet #%s: rows=%s, cols=%s", len(self.recv_packets), len(packet.data["data"]),
packet.data["cols"])
def cleanup_stream_sub():
if self.stream_sub:
self.stream_sub.stop()
self.stream_sub = None
self.stream_sub = StreamSubscriber(process=self.scion_proc, stream="basic_streams", callback=process_packet_cb)
self.stream_sub.start()
self.addCleanup(cleanup_stream_sub)
# Start agent
agent_pid = self.scion_client.start_agent(inst_id)
self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))
sac = StreamingAgentClient(resource_id=inst_id, process=self.scion_proc)
sac.connect()
sac.start_streaming()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_STREAMING)
# Retrieve data
gevent.sleep(1.5)
self.assertTrue(os.path.exists(ds_filename))
inst_data_t2 = self.scion_client.get_asset_data(inst_id)
print "T2", inst_data_t2
num_rows_t2 = inst_data_t2["num_rows"]
self.assertEquals(num_rows_t2, 40)
sample_data = inst_data_t2["data"]["sample_vector"]
self.assertEquals(sample_data[0][1], 100)
self.assertEquals(sample_data[39][1], 409)
self.assertEquals(sample_data[1][0] - sample_data[0][0], 100)
self.assertEquals(sample_data[39][0] - sample_data[0][0], 3900)
# Take down agent
sac.stop_streaming() # Not required to stop agent, just to test here
sac.disconnect()
self.scion_client.stop_agent(inst_id)
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
inst_data = self.scion_client.get_asset_data(inst_id, data_filter=dict(max_rows=20))
self.assertEquals(len(inst_data["data"]["sample_vector"]), 20)
self.assertEquals(inst_data["data"]["sample_vector"][0][1], 300)
inst_data = self.scion_client.get_asset_data(inst_id, data_filter=dict(start_time="1460694001000"))
self.assertEquals(len(inst_data["data"]["sample_vector"]), 30)
self.assertEquals(inst_data["data"]["sample_vector"][0][1], 200)
# Note: Time filter applies before expansion
inst_data = self.scion_client.get_asset_data(inst_id, data_filter=dict(start_time="1460694001000", start_time_include=False))
self.assertEquals(len(inst_data["data"]["sample_vector"]), 20)
self.assertEquals(inst_data["data"]["sample_vector"][0][1], 300)
inst_data = self.scion_client.get_asset_data(inst_id, data_filter=dict(start_time="1460694001000", max_rows=10))
self.assertEquals(len(inst_data["data"]["sample_vector"]), 10)
self.assertEquals(inst_data["data"]["sample_vector"][0][1], 400)
inst_data = self.scion_client.get_asset_data(inst_id, data_filter=dict(start_time="1460694001000", end_time="1460694002000"))
print inst_data
self.assertEquals(len(inst_data["data"]["sample_vector"]), 20)
self.assertEquals(inst_data["data"]["sample_vector"][0][1], 200)
def test_scion_agent(self):
# Create user
actor_id = self.scion_client.define_user(
first_name="John", last_name="Doe",
username="******", password="******", email="*****@*****.**")
# Create instrument
agent_info=[dict(agent_type="data_agent",
config=dict(plugin="scion.agent.model.orb.orb_plugin.Orb_DataAgentPlugin",
sampling_interval=0.5, stream_name="basic_streams",
auto_streaming=False))]
inst_obj = Instrument(name="TA_121A/MGENC/M40", description="Multiplexed generic compressed data frame packet",
location=GeospatialLocation(latitude=42.867079, longitude=-127.257324),
agent_info=agent_info)
inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)
# Create dataset
schema_def = DataSchemaParser.parse_schema_ref("ds_orb_mgenc_m40")
ds_obj = Dataset(name="Dataset Sensor 3",
schema_definition=schema_def)
ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)
self.rr.create_association(inst_id, PRED.hasDataset, ds_id)
ds_filename = self.container.file_system.get("%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
self.assertFalse(os.path.exists(ds_filename))
inst_data_t0 = self.scion_client.get_asset_data(inst_id)
self.assertEquals(inst_data_t0["dataset_id"], ds_id)
self.assertEquals(inst_data_t0["num_rows"], 0)
# Install a data packet catcher
self.recv_packets, self.recv_rows = [], 0
def process_packet_cb(packet, route, stream):
if not isinstance(packet, DataPacket):
log.warn("Received a non DataPacket message")
self.recv_packets.append(packet)
self.recv_rows += len(packet.data["data"])
log.info("Received data packet #%s: rows=%s, cols=%s", len(self.recv_packets), len(packet.data["data"]),
packet.data["cols"])
#log.info('Packet data: ' + str(packet.data))
def cleanup_stream_sub():
if self.stream_sub:
self.stream_sub.stop()
self.stream_sub = None
self.stream_sub = StreamSubscriber(process=self.scion_proc, stream="basic_streams", callback=process_packet_cb)
self.stream_sub.start()
self.addCleanup(cleanup_stream_sub)
# Start agent
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
agent_pid = self.scion_client.start_agent(inst_id)
self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))
sac = StreamingAgentClient(resource_id=inst_id, process=self.scion_proc)
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)
sac.connect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)
# Coming in from the agent config.
streaming_args = {
'orb_name' : 'taexport.ucsd.edu:usarrayTA',
#'select' : 'TA_109C/MGENC/M40',
'select' : 'TA_121A/MGENC/M40',
'--timeout' : 5,
'sample_interval' : 5
}
sac.start_streaming(streaming_args)
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_STREAMING)
gevent.sleep(180)
# Retrieve data
"""
gevent.sleep(1)
self.assertTrue(os.path.exists(ds_filename))
inst_data_t1 = self.scion_client.get_asset_data(inst_id)
self.assertEquals(inst_data_t1["dataset_id"], ds_id)
self.assertEquals(inst_data_t1["variables"], ['time', 'cpu_percent'])
self.assertIn("cpu_percent", inst_data_t1["data"])
num_rows_t1 = inst_data_t1["num_rows"]
self.assertGreaterEqual(num_rows_t1, 2)
gevent.sleep(1)
inst_data_t2 = self.scion_client.get_asset_data(inst_id)
num_rows_t2 = inst_data_t2["num_rows"]
self.assertGreater(num_rows_t2, num_rows_t1)
self.assertGreaterEqual(len(self.recv_packets), self.recv_rows)
self.assertLessEqual(abs(self.recv_rows - num_rows_t2), 2)
"""
# Take down agent
sac.stop_streaming() # Not required to stop agent, just to test here
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)
sac.disconnect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)
self.scion_client.stop_agent(inst_id)
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
class TestScionOrbAgentData(IonIntegrationTestCase):
"""Test for Scion with agents and data streaming
"""
def setUp(self):
self._start_container()
self.patch_alt_cfg('scion.process.preload.preloader.CFG',
{'scion': {'preload': {'enabled': False}}})
self.container.start_rel_from_url('res/deploy/scion.yml')
self.rr = self.container.resource_registry
self.scion_client = ScionManagementClient()
self.idm_client = IdentityManagementServiceClient()
self.system_actor_id = None
self.ui_server_proc = self.container.proc_manager.procs_by_name["ui_server"]
self.scion_proc = self.container.proc_manager.procs_by_name["scion_management"]
self.ui_base_url = self.ui_server_proc.base_url
self.sg_base_url = self.ui_server_proc.gateway_base_url
def tearDown(self):
pass
@unittest.skipIf(getpass.getuser() == "jenkins", "Disabled on Jenkins")
def test_scion_agent(self):
# Create user
actor_id = self.scion_client.define_user(
first_name="John", last_name="Doe",
username="******", password="******", email="*****@*****.**")
# Create instrument
agent_info=[dict(agent_type="data_agent",
config=dict(plugin="scion.agent.model.orb.orb_plugin.Orb_DataAgentPlugin",
sampling_interval=0.5, stream_name="basic_streams",
auto_streaming=False))]
inst_obj = Instrument(name="TA_121A/MGENC/M40", description="Multiplexed generic compressed data frame packet",
location=GeospatialLocation(latitude=42.867079, longitude=-127.257324),
agent_info=agent_info)
inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)
# Create dataset
schema_def = DataSchemaParser.parse_schema_ref("ds_orb_mgenc_m40")
ds_obj = Dataset(name="Dataset Sensor 3",
schema_definition=schema_def)
ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)
self.rr.create_association(inst_id, PRED.hasDataset, ds_id)
ds_filename = self.container.file_system.get("%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
self.assertFalse(os.path.exists(ds_filename))
inst_data_t0 = self.scion_client.get_asset_data(inst_id)
self.assertEquals(inst_data_t0["dataset_id"], ds_id)
self.assertEquals(inst_data_t0["num_rows"], 0)
# Install a data packet catcher
self.recv_packets, self.recv_rows = [], 0
def process_packet_cb(packet, route, stream):
if not isinstance(packet, DataPacket):
log.warn("Received a non DataPacket message")
self.recv_packets.append(packet)
self.recv_rows += len(packet.data["data"])
log.info("Received data packet #%s: rows=%s, cols=%s", len(self.recv_packets), len(packet.data["data"]),
packet.data["cols"])
#log.info('Packet data: ' + str(packet.data))
def cleanup_stream_sub():
if self.stream_sub:
self.stream_sub.stop()
self.stream_sub = None
self.stream_sub = StreamSubscriber(process=self.scion_proc, stream="basic_streams", callback=process_packet_cb)
self.stream_sub.start()
self.addCleanup(cleanup_stream_sub)
# Start agent
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
agent_pid = self.scion_client.start_agent(inst_id)
self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))
sac = StreamingAgentClient(resource_id=inst_id, process=self.scion_proc)
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)
sac.connect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)
# Coming in from the agent config.
streaming_args = {
'orb_name' : 'taexport.ucsd.edu:usarrayTA',
#'select' : 'TA_109C/MGENC/M40',
'select' : 'TA_121A/MGENC/M40',
'--timeout' : 5,
'sample_interval' : 5
}
sac.start_streaming(streaming_args)
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_STREAMING)
gevent.sleep(180)
# Retrieve data
"""
gevent.sleep(1)
self.assertTrue(os.path.exists(ds_filename))
inst_data_t1 = self.scion_client.get_asset_data(inst_id)
self.assertEquals(inst_data_t1["dataset_id"], ds_id)
self.assertEquals(inst_data_t1["variables"], ['time', 'cpu_percent'])
self.assertIn("cpu_percent", inst_data_t1["data"])
num_rows_t1 = inst_data_t1["num_rows"]
self.assertGreaterEqual(num_rows_t1, 2)
gevent.sleep(1)
inst_data_t2 = self.scion_client.get_asset_data(inst_id)
num_rows_t2 = inst_data_t2["num_rows"]
self.assertGreater(num_rows_t2, num_rows_t1)
self.assertGreaterEqual(len(self.recv_packets), self.recv_rows)
self.assertLessEqual(abs(self.recv_rows - num_rows_t2), 2)
"""
# Take down agent
sac.stop_streaming() # Not required to stop agent, just to test here
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)
sac.disconnect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)
self.scion_client.stop_agent(inst_id)
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
def test_scion_agent(self):
# Create user
actor_id = self.scion_client.define_user(first_name="John",
last_name="Doe",
username="******",
password="******",
email="*****@*****.**")
# Create instrument
data_filename = os.path.join(
os.path.split(__file__)[0], "testdata/orb_replay_data.yml")
agent_info = [
dict(
agent_type="data_agent",
config=dict(
plugin=
"scion.agent.model.orb.orb_replay_plugin.ORBReplay_DataAgentPlugin",
sampling_interval=0.2,
stream_name="basic_streams",
replay_file=data_filename,
replay_scenario="basic",
auto_streaming=False))
]
inst_obj = Instrument(name="ORB Sensor 1",
description="Seismic stream",
location=GeospatialLocation(
latitude=32.867079, longitude=-117.257324),
agent_info=agent_info)
inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)
# Create dataset
schema_def = DataSchemaParser.parse_schema_ref("ds_orb01_main")
ds_obj = Dataset(name="ORB Dataset 1", schema_definition=schema_def)
ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)
self.rr.create_association(inst_id, PRED.hasDataset, ds_id)
ds_filename = self.container.file_system.get(
"%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
# Install a data packet catcher
self.recv_packets, self.recv_rows = [], 0
def process_packet_cb(packet, route, stream):
if not isinstance(packet, DataPacket):
log.warn("Received a non DataPacket message")
self.recv_packets.append(packet)
self.recv_rows += len(packet.data["data"])
log.info("Received data packet #%s: rows=%s, cols=%s",
len(self.recv_packets), len(packet.data["data"]),
packet.data["cols"])
def cleanup_stream_sub():
if self.stream_sub:
self.stream_sub.stop()
self.stream_sub = None
self.stream_sub = StreamSubscriber(process=self.scion_proc,
stream="basic_streams",
callback=process_packet_cb)
self.stream_sub.start()
self.addCleanup(cleanup_stream_sub)
# Start agent
agent_pid = self.scion_client.start_agent(inst_id)
self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))
sac = StreamingAgentClient(resource_id=inst_id,
process=self.scion_proc)
sac.connect()
sac.start_streaming()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_STREAMING)
# Retrieve data
gevent.sleep(1.5)
self.assertTrue(os.path.exists(ds_filename))
inst_data_t2 = self.scion_client.get_asset_data(inst_id)
print "T2", inst_data_t2
num_rows_t2 = inst_data_t2["num_rows"]
self.assertEquals(num_rows_t2, 40)
sample_data = inst_data_t2["data"]["sample_vector"]
self.assertEquals(sample_data[0][1], 100)
self.assertEquals(sample_data[39][1], 409)
self.assertEquals(sample_data[1][0] - sample_data[0][0], 100)
self.assertEquals(sample_data[39][0] - sample_data[0][0], 3900)
# Take down agent
sac.stop_streaming() # Not required to stop agent, just to test here
sac.disconnect()
self.scion_client.stop_agent(inst_id)
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
inst_data = self.scion_client.get_asset_data(
inst_id, data_filter=dict(max_rows=20))
self.assertEquals(len(inst_data["data"]["sample_vector"]), 20)
self.assertEquals(inst_data["data"]["sample_vector"][0][1], 300)
inst_data = self.scion_client.get_asset_data(
inst_id, data_filter=dict(start_time="1460694001000"))
self.assertEquals(len(inst_data["data"]["sample_vector"]), 30)
self.assertEquals(inst_data["data"]["sample_vector"][0][1], 200)
# Note: Time filter applies before expansion
inst_data = self.scion_client.get_asset_data(
inst_id,
data_filter=dict(start_time="1460694001000",
start_time_include=False))
self.assertEquals(len(inst_data["data"]["sample_vector"]), 20)
self.assertEquals(inst_data["data"]["sample_vector"][0][1], 300)
inst_data = self.scion_client.get_asset_data(
inst_id, data_filter=dict(start_time="1460694001000", max_rows=10))
self.assertEquals(len(inst_data["data"]["sample_vector"]), 10)
self.assertEquals(inst_data["data"]["sample_vector"][0][1], 400)
inst_data = self.scion_client.get_asset_data(
inst_id,
data_filter=dict(start_time="1460694001000",
end_time="1460694002000"))
print inst_data
self.assertEquals(len(inst_data["data"]["sample_vector"]), 20)
self.assertEquals(inst_data["data"]["sample_vector"][0][1], 200)
def test_scion_agent(self):
# Create user
actor_id = self.scion_client.define_user(
first_name="John", last_name="Doe",
username="******", password="******", email="*****@*****.**")
# Create instrument
agent_info=[dict(agent_type="data_agent",
config=dict(plugin="scion.agent.model.cdip.cdip_plugin.CDIP_DataAgentPlugin",
sampling_interval=10, stream_name="basic_streams",
auto_streaming=False))]
inst_obj = Instrument(name="TA_121A/MGENC/M40", description="CDIP buoy data",
location=GeospatialLocation(latitude=37.94831666666667, longitude=-123.4675),
agent_info=agent_info)
inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)
# Create dataset
schema_def = DataSchemaParser.parse_schema_ref("ds_cdip01_main")
ds_obj = Dataset(name="Dataset Sensor",
schema_definition=schema_def)
ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)
self.rr.create_association(inst_id, PRED.hasDataset, ds_id)
ds_filename = self.container.file_system.get("%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
self.assertFalse(os.path.exists(ds_filename))
inst_data_t0 = self.scion_client.get_asset_data(inst_id)
self.assertEquals(inst_data_t0["dataset_id"], ds_id)
self.assertEquals(inst_data_t0["num_rows"], 0)
# Install a data packet catcher
self.recv_packets, self.recv_rows = [], 0
def process_packet_cb(packet, route, stream):
if not isinstance(packet, DataPacket):
log.warn("Received a non DataPacket message")
self.recv_packets.append(packet)
self.recv_rows += len(packet.data["data"])
log.info("Received data packet #%s: rows=%s, cols=%s", len(self.recv_packets), len(packet.data["data"]),
packet.data["cols"])
#log.info('Packet data: ' + str(packet.data))
def cleanup_stream_sub():
if self.stream_sub:
self.stream_sub.stop()
self.stream_sub = None
self.stream_sub = StreamSubscriber(process=self.scion_proc, stream="basic_streams", callback=process_packet_cb)
self.stream_sub.start()
self.addCleanup(cleanup_stream_sub)
# Start agent
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
agent_pid = self.scion_client.start_agent(inst_id)
self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))
sac = StreamingAgentClient(resource_id=inst_id, process=self.scion_proc)
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)
sac.connect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)
# Coming in from the agent config.
streaming_args = {
'url' : 'http://cdip.ucsd.edu/data_access/justdar.cdip?029+pm',
'sampling_interval' : 10
}
sac.start_streaming(streaming_args)
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_STREAMING)
# Set to progressively high values for real data stream tests.
gevent.sleep(20)
# Retrieve data
self.assertTrue(os.path.exists(ds_filename))
inst_data = self.scion_client.get_asset_data(inst_id)
"""
{'data': {'Dp': [[1465682100000, 325]],
'Hs': [[1465682100000, 3.03]],
'Ta': [[1465682100000, 6.92]],
'Temp': [[1465682100000, 12.2]],
'Tp': [[1465682100000, 9.09]]},
'dataset_id': '08bc829159e6401182462b713b180dbe',
'num_rows': 1,
'ts_generated': '1465685467675',
'var_def': [{'base_type': 'ntp_time',
'description': 'NTPv4 timestamp',
'name': 'time',
'storage_dtype': 'i8',
'unit': ''},
{'base_type': 'float',
'description': 'Significant wave height',
'name': 'Hs',
'storage_dtype': 'f8',
'unit': 'meters'},
{'base_type': 'float',
'description': 'Peak wave period',
'name': 'Tp',
'storage_dtype': 'f8',
'unit': 'seconds'},
{'base_type': 'int',
'description': 'Peak wave direction',
'name': 'Dp',
'storage_dtype': 'i4',
'unit': 'degrees'},
{'base_type': 'float',
'description': 'Average wave period',
'name': 'Ta',
'storage_dtype': 'f8',
'unit': 'seconds'},
{'base_type': 'float',
'description': 'Surface temperature',
'name': 'Temp',
'storage_dtype': 'f8',
'unit': 'celcius'}],
'variables': ['time', 'Hs', 'Tp', 'Dp', 'Ta', 'Temp']}
"""
num_rows = inst_data["num_rows"]
log.info('CDIP test produced %i data rows.' % num_rows)
# Take down agent
sac.stop_streaming() # Not required to stop agent, just to test here
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)
sac.disconnect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)
self.scion_client.stop_agent(inst_id)
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
class TestScionAgentData(IonIntegrationTestCase):
"""Test for Scion with agents and data streaming
"""
def setUp(self):
self._start_container()
self.patch_alt_cfg('scion.process.preload.preloader.CFG',
{'scion': {
'preload': {
'enabled': False
}
}})
self.container.start_rel_from_url('res/deploy/scion.yml')
self.rr = self.container.resource_registry
self.scion_client = ScionManagementClient()
self.idm_client = IdentityManagementServiceClient()
self.system_actor_id = None
self.ui_server_proc = self.container.proc_manager.procs_by_name[
"ui_server"]
self.scion_proc = self.container.proc_manager.procs_by_name[
"scion_management"]
self.ui_base_url = self.ui_server_proc.base_url
self.sg_base_url = self.ui_server_proc.gateway_base_url
def tearDown(self):
pass
def test_scion_agent(self):
# Create user
actor_id = self.scion_client.define_user(first_name="John",
last_name="Doe",
username="******",
password="******",
email="*****@*****.**")
# Create instrument
agent_info = [
dict(
agent_type="data_agent",
config=dict(
plugin=
"scion.agent.model.vmmon.vmmon_plugin.VMMON_DataAgentPlugin",
sampling_interval=0.5,
stream_name="basic_streams",
auto_streaming=False))
]
inst_obj = Instrument(name="Sensor 1",
description="CPU monitor",
location=GeospatialLocation(
latitude=32.867079, longitude=-117.257324),
agent_info=agent_info)
inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)
# Create dataset
schema_def = DataSchemaParser.parse_schema_ref("ds_vmmon_main")
ds_obj = Dataset(name="Dataset Sensor 1", schema_definition=schema_def)
ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)
self.rr.create_association(inst_id, PRED.hasDataset, ds_id)
ds_filename = self.container.file_system.get(
"%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
self.assertFalse(os.path.exists(ds_filename))
inst_data_t0 = self.scion_client.get_asset_data(inst_id)
self.assertEquals(inst_data_t0["dataset_id"], ds_id)
self.assertEquals(inst_data_t0["num_rows"], 0)
# Install a data packet catcher
self.recv_packets, self.recv_rows = [], 0
def process_packet_cb(packet, route, stream):
if not isinstance(packet, DataPacket):
log.warn("Received a non DataPacket message")
self.recv_packets.append(packet)
self.recv_rows += len(packet.data["data"])
log.info("Received data packet #%s: rows=%s, cols=%s",
len(self.recv_packets), len(packet.data["data"]),
packet.data["cols"])
def cleanup_stream_sub():
if self.stream_sub:
self.stream_sub.stop()
self.stream_sub = None
self.stream_sub = StreamSubscriber(process=self.scion_proc,
stream="basic_streams",
callback=process_packet_cb)
self.stream_sub.start()
self.addCleanup(cleanup_stream_sub)
# Start agent
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
agent_pid = self.scion_client.start_agent(inst_id)
self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))
sac = StreamingAgentClient(resource_id=inst_id,
process=self.scion_proc)
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_INITIALIZED)
sac.connect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_CONNECTED)
sac.start_streaming()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_STREAMING)
# Retrieve data
gevent.sleep(1.1)
self.assertTrue(os.path.exists(ds_filename))
inst_data_t1 = self.scion_client.get_asset_data(inst_id)
self.assertEquals(inst_data_t1["dataset_id"], ds_id)
# self.assertEquals(inst_data_t1["variables"], ['time', 'cpu_percent'])
self.assertIn("cpu_percent", inst_data_t1["data"])
num_rows_t1 = inst_data_t1["num_rows"]
self.assertGreaterEqual(num_rows_t1, 2)
gevent.sleep(1)
inst_data_t2 = self.scion_client.get_asset_data(inst_id)
num_rows_t2 = inst_data_t2["num_rows"]
self.assertGreater(num_rows_t2, num_rows_t1)
self.assertGreaterEqual(len(self.recv_packets), self.recv_rows)
self.assertLessEqual(abs(self.recv_rows - num_rows_t2), 2)
# Take down agent
sac.stop_streaming() # Not required to stop agent, just to test here
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_CONNECTED)
sac.disconnect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_INITIALIZED)
self.scion_client.stop_agent(inst_id)
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
