def create_dataset(self, name='', datastore_name='', view_name='', stream_id='', parameter_dict=None, spatial_domain=None, temporal_domain=None, parameter_dictionary_id='', description=''):
validate_true(parameter_dict or parameter_dictionary_id, 'A parameter dictionary must be supplied to register a new dataset.')
validate_is_not_none(spatial_domain, 'A spatial domain must be supplied to register a new dataset.')
validate_is_not_none(temporal_domain, 'A temporal domain must be supplied to register a new dataset.')
if parameter_dictionary_id:
pd = self.read_parameter_dictionary(parameter_dictionary_id)
pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False)
parameter_dict = self._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context)
parameter_dict = parameter_dict.dump()
dataset = Dataset()
dataset.description = description
dataset.name = name
dataset.primary_view_key = stream_id or None
dataset.datastore_name = datastore_name or self.DEFAULT_DATASTORE
dataset.view_name = view_name or self.DEFAULT_VIEW
dataset.parameter_dictionary = parameter_dict
dataset.temporal_domain = temporal_domain
dataset.spatial_domain = spatial_domain
dataset.registered = False
dataset_id, _ = self.clients.resource_registry.create(dataset)
if stream_id:
self.add_stream(dataset_id,stream_id)
log.debug('creating dataset: %s', dataset_id)
cov = self._create_coverage(dataset_id, description or dataset_id, parameter_dict, spatial_domain, temporal_domain)
self._save_coverage(cov)
cov.close()
return dataset_id
def get_cov(self):
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
t_ctxt = ParameterContext(
'time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt, is_temporal=True)
lat_ctxt = ParameterContext(
'lat', param_type=QuantityType(value_encoding=np.dtype('float32')))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext(
'lon', param_type=QuantityType(value_encoding=np.dtype('float32')))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
pdict.add_context(lon_ctxt)
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs,
MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial',
[0]), scrs, MutabilityEnum.IMMUTABLE
) # 0d spatial topology (station/trajectory)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
scov = SimplexCoverage(self.working_dir,
create_guid(),
'sample coverage_model',
parameter_dictionary=pdict,
temporal_domain=tdom,
spatial_domain=sdom,
inline_data_writes=True,
in_memory_storage=False)
# Insert some timesteps (automatically expands other arrays)
nt = 200000
scov.insert_timesteps(nt)
# Add data for each parameter
scov.set_parameter_values('time', value=np.arange(nt))
scov.set_parameter_values('lat', value=45)
scov.set_parameter_values('lon', value=-71)
dset = Dataset()
dset.parameter_dictionary = pdict.dump()
dset.spatial_domain = sdom.dump()
dset.temporal_domain = tdom.dump()
return scov, dset
def get_cov(self):
# Instantiate a ParameterDictionary
pdict = ParameterDictionary()
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt, is_temporal=True)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.dtype('float32')))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.dtype('float32')))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
pdict.add_context(lon_ctxt)
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 0d spatial topology (station/trajectory)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
scov = SimplexCoverage(self.working_dir, create_guid(), 'sample coverage_model', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom, inline_data_writes=True, in_memory_storage=False)
# Insert some timesteps (automatically expands other arrays)
nt = 200000
# Add data for each parameter
data = { scov.temporal_parameter_name: np.arange(nt),
'lat': np.empty(nt),
'lon': np.empty(nt)
}
data['lat'][:] = 45
data['lon'][:] = -71
scov.set_parameter_values(make_parameter_data_dict(data))
dset = Dataset()
dset.parameter_dictionary = pdict.dump()
dset.spatial_domain = sdom.dump()
dset.temporal_domain = tdom.dump()
return scov, dset
def _make_dataset(self):
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
dataset = Dataset(name='test_dataset')
dataset_id = self.dataset_management.create_dataset(
dataset, parameter_dictionary_id=parameter_dict_id)
return dataset_id
def create_dataset(self, parameter_dict_id=''):
'''
Creates a time-series dataset
'''
if not parameter_dict_id:
parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
dataset = Dataset('test_dataset_%i'%self.i)
dataset_id = self.dataset_management.create_dataset(dataset, parameter_dictionary_id=parameter_dict_id)
self.addCleanup(self.dataset_management.delete_dataset, dataset_id)
return dataset_id
def test_dataset_crud(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
dataset = Dataset(name='ctd_dataset')
dataset_id = self.dataset_management.create_dataset(
dataset, parameter_dictionary_id=pdict_id)
ds_obj = self.dataset_management.read_dataset(dataset_id)
self.assertEquals(ds_obj.name, 'ctd_dataset')
ds_obj.name = 'something different'
self.dataset_management.update_dataset(ds_obj)
ds_obj2 = self.dataset_management.read_dataset(dataset_id)
self.assertEquals(ds_obj.name, ds_obj2.name)
def create_dataset(self, parameter_dict_id=''):
'''
Creates a time-series dataset
'''
dataset_management = DatasetManagementServiceClient()
if not parameter_dict_id:
parameter_dict_id = dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
dataset = Dataset(name='test_dataset_')
dataset_id = dataset_management.create_dataset(
dataset, parameter_dictionary_id=parameter_dict_id)
self.addCleanup(dataset_management.delete_dataset, dataset_id)
return dataset_id
def create_dataset(self, name='', datastore_name='', view_name='', stream_id='', parameter_dict=None, parameter_dictionary_id='', description='', parent_dataset_id=''):
validate_true(parameter_dict or parameter_dictionary_id, 'A parameter dictionary must be supplied to register a new dataset.')
if parameter_dictionary_id:
pd = self.read_parameter_dictionary(parameter_dictionary_id)
pcs = self.read_parameter_contexts(parameter_dictionary_id, id_only=False)
parameter_dict = self._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context)
parameter_dict = parameter_dict.dump()
parameter_dict = self.numpy_walk(parameter_dict)
dataset = Dataset()
dataset.description = description
dataset.name = name
dataset.primary_view_key = stream_id or None
dataset.datastore_name = datastore_name or self.DEFAULT_DATASTORE
dataset.view_name = view_name or self.DEFAULT_VIEW
dataset.parameter_dictionary = parameter_dict
dataset.registered = False
dataset_id, _ = self.clients.resource_registry.create(dataset)
if stream_id:
self.add_stream(dataset_id, stream_id)
log.debug('creating dataset: %s', dataset_id)
if parent_dataset_id:
vcov = self._create_view_coverage(dataset_id, description or dataset_id, parent_dataset_id)
vcov.close()
return dataset_id
cov = self._create_coverage(dataset_id, description or dataset_id, parameter_dict)
self._save_coverage(cov)
cov.close()
#table loader create resource
if self._get_eoi_service_available():
log.debug('DM:create dataset: %s -- dataset_id: %s', name, dataset_id)
self._create_single_resource(dataset_id, parameter_dict)
return dataset_id
def create_dataset(self,
name='',
datastore_name='',
view_name='',
stream_id='',
parameter_dict=None,
spatial_domain=None,
temporal_domain=None,
parameter_dictionary_id='',
description='',
parent_dataset_id=''):
validate_true(
parameter_dict or parameter_dictionary_id,
'A parameter dictionary must be supplied to register a new dataset.'
)
validate_is_not_none(
spatial_domain,
'A spatial domain must be supplied to register a new dataset.')
validate_is_not_none(
temporal_domain,
'A temporal domain must be supplied to register a new dataset.')
if parameter_dictionary_id:
pd = self.read_parameter_dictionary(parameter_dictionary_id)
pcs = self.read_parameter_contexts(parameter_dictionary_id,
id_only=False)
parameter_dict = self._merge_contexts(
[ParameterContext.load(i.parameter_context) for i in pcs],
pd.temporal_context)
parameter_dict = parameter_dict.dump()
parameter_dict = self.numpy_walk(parameter_dict)
dataset = Dataset()
dataset.description = description
dataset.name = name
dataset.primary_view_key = stream_id or None
dataset.datastore_name = datastore_name or self.DEFAULT_DATASTORE
dataset.view_name = view_name or self.DEFAULT_VIEW
dataset.parameter_dictionary = parameter_dict
dataset.temporal_domain = temporal_domain
dataset.spatial_domain = spatial_domain
dataset.registered = False
dataset_id, _ = self.clients.resource_registry.create(dataset)
if stream_id:
self.add_stream(dataset_id, stream_id)
log.debug('creating dataset: %s', dataset_id)
if parent_dataset_id:
vcov = self._create_view_coverage(dataset_id, description
or dataset_id, parent_dataset_id)
vcov.close()
return dataset_id
cov = self._create_coverage(dataset_id, description or dataset_id,
parameter_dict, spatial_domain,
temporal_domain)
self._save_coverage(cov)
cov.close()
return dataset_id
def test_scion_agent(self):
# Create user
actor_id = self.scion_client.define_user(first_name="John",
last_name="Doe",
username="******",
password="******",
email="*****@*****.**")
inst_ids = []
ds_ids = []
for source in sources:
# Instrument
agent_info = [
dict(
agent_type="data_agent",
config=dict(
plugin=
"scion.agent.model.orb.orb_plugin.Orb_DataAgentPlugin",
sampling_interval=0.5,
stream_name="basic_streams",
auto_streaming=False))
]
inst_obj = Instrument(
name=source,
description="Multiplexed generic compressed data frame packet",
location=GeospatialLocation(latitude=42.867079,
longitude=-127.257324),
agent_info=agent_info)
inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)
inst_ids.append(inst_id)
# Dataset
schema_def = DataSchemaParser.parse_schema_ref("ds_orb_mgenc_m40")
ds_obj = Dataset(name=source, schema_definition=schema_def)
ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)
self.rr.create_association(inst_id, PRED.hasDataset, ds_id)
ds_filename = self.container.file_system.get(
"%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
self.assertFalse(os.path.exists(ds_filename))
inst_data_t0 = self.scion_client.get_asset_data(inst_id)
self.assertEquals(inst_data_t0["dataset_id"], ds_id)
self.assertEquals(inst_data_t0["num_rows"], 0)
ds_ids.append(ds_id)
# Install a data packet catcher
# TODO
# Start agent
sacs = []
for idx, source in enumerate(sources):
self.assertFalse(
StreamingAgentClient.is_agent_active(inst_ids[idx]))
agent_pid = self.scion_client.start_agent(inst_ids[idx])
self.assertTrue(StreamingAgentClient.is_agent_active(
inst_ids[idx]))
sac = StreamingAgentClient(resource_id=inst_ids[idx],
process=self.scion_proc)
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_INITIALIZED)
sac.connect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_CONNECTED)
# Coming in from the agent config.
streaming_args = {
#'orb_name' : 'taexport.ucsd.edu:usarrayTA',
'orb_name': 'ceusnexport.ucsd.edu:usarray',
'select': source,
'--timeout': 5,
'sample_interval': 5
}
sac.start_streaming(streaming_args)
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_STREAMING)
sacs.append(sac)
gevent.sleep(120)
# Take down agent
for idx, sac in enumerate(sacs):
#sac.stop_streaming() # Not required to stop agent, just to test here
#agent_status = sac.get_status()
#self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)
#sac.disconnect()
#agent_status = sac.get_status()
#self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)
self.scion_client.stop_agent(inst_ids[idx])
self.assertFalse(
StreamingAgentClient.is_agent_active(inst_ids[idx]))
def test_scion_agent(self):
# Create user
actor_id = self.scion_client.define_user(first_name="John",
last_name="Doe",
username="******",
password="******",
email="*****@*****.**")
# Create instrument
agent_info = [
dict(agent_type="data_agent",
config=dict(
plugin=
"scion.agent.model.cdip.cdip_plugin.CDIP_DataAgentPlugin",
sampling_interval=10,
stream_name="basic_streams",
auto_streaming=False))
]
inst_obj = Instrument(name="TA_121A/MGENC/M40",
description="CDIP buoy data",
location=GeospatialLocation(
latitude=37.94831666666667,
longitude=-123.4675),
agent_info=agent_info)
inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)
# Create dataset
schema_def = DataSchemaParser.parse_schema_ref("ds_cdip01_main")
ds_obj = Dataset(name="Dataset Sensor", schema_definition=schema_def)
ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)
self.rr.create_association(inst_id, PRED.hasDataset, ds_id)
ds_filename = self.container.file_system.get(
"%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
self.assertFalse(os.path.exists(ds_filename))
inst_data_t0 = self.scion_client.get_asset_data(inst_id)
self.assertEquals(inst_data_t0["dataset_id"], ds_id)
self.assertEquals(inst_data_t0["num_rows"], 0)
# Install a data packet catcher
self.recv_packets, self.recv_rows = [], 0
def process_packet_cb(packet, route, stream):
if not isinstance(packet, DataPacket):
log.warn("Received a non DataPacket message")
self.recv_packets.append(packet)
self.recv_rows += len(packet.data["data"])
log.info("Received data packet #%s: rows=%s, cols=%s",
len(self.recv_packets), len(packet.data["data"]),
packet.data["cols"])
#log.info('Packet data: ' + str(packet.data))
def cleanup_stream_sub():
if self.stream_sub:
self.stream_sub.stop()
self.stream_sub = None
self.stream_sub = StreamSubscriber(process=self.scion_proc,
stream="basic_streams",
callback=process_packet_cb)
self.stream_sub.start()
self.addCleanup(cleanup_stream_sub)
# Start agent
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
agent_pid = self.scion_client.start_agent(inst_id)
self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))
sac = StreamingAgentClient(resource_id=inst_id,
process=self.scion_proc)
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_INITIALIZED)
sac.connect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_CONNECTED)
# Coming in from the agent config.
streaming_args = {
'url': 'http://cdip.ucsd.edu/data_access/justdar.cdip?029+pm',
'sampling_interval': 10
}
sac.start_streaming(streaming_args)
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_STREAMING)
# Set to progressively high values for real data stream tests.
gevent.sleep(20)
# Retrieve data
self.assertTrue(os.path.exists(ds_filename))
inst_data = self.scion_client.get_asset_data(inst_id)
"""
{'data': {'Dp': [[1465682100000, 325]],
'Hs': [[1465682100000, 3.03]],
'Ta': [[1465682100000, 6.92]],
'Temp': [[1465682100000, 12.2]],
'Tp': [[1465682100000, 9.09]]},
'dataset_id': '08bc829159e6401182462b713b180dbe',
'num_rows': 1,
'ts_generated': '1465685467675',
'var_def': [{'base_type': 'ntp_time',
'description': 'NTPv4 timestamp',
'name': 'time',
'storage_dtype': 'i8',
'unit': ''},
{'base_type': 'float',
'description': 'Significant wave height',
'name': 'Hs',
'storage_dtype': 'f8',
'unit': 'meters'},
{'base_type': 'float',
'description': 'Peak wave period',
'name': 'Tp',
'storage_dtype': 'f8',
'unit': 'seconds'},
{'base_type': 'int',
'description': 'Peak wave direction',
'name': 'Dp',
'storage_dtype': 'i4',
'unit': 'degrees'},
{'base_type': 'float',
'description': 'Average wave period',
'name': 'Ta',
'storage_dtype': 'f8',
'unit': 'seconds'},
{'base_type': 'float',
'description': 'Surface temperature',
'name': 'Temp',
'storage_dtype': 'f8',
'unit': 'celcius'}],
'variables': ['time', 'Hs', 'Tp', 'Dp', 'Ta', 'Temp']}
"""
num_rows = inst_data["num_rows"]
log.info('CDIP test produced %i data rows.' % num_rows)
# Take down agent
sac.stop_streaming() # Not required to stop agent, just to test here
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_CONNECTED)
sac.disconnect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_INITIALIZED)
self.scion_client.stop_agent(inst_id)
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
def test_scion_agent(self):
# Create user
actor_id = self.scion_client.define_user(
first_name="John", last_name="Doe",
username="******", password="******", email="*****@*****.**")
# Create instrument
agent_info=[dict(agent_type="data_agent",
config=dict(plugin="scion.agent.model.orb.orb_plugin.Orb_DataAgentPlugin",
sampling_interval=0.5, stream_name="basic_streams",
auto_streaming=False))]
inst_obj = Instrument(name="TA_121A/MGENC/M40", description="Multiplexed generic compressed data frame packet",
location=GeospatialLocation(latitude=42.867079, longitude=-127.257324),
agent_info=agent_info)
inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)
# Create dataset
schema_def = DataSchemaParser.parse_schema_ref("ds_orb_mgenc_m40")
ds_obj = Dataset(name="Dataset Sensor 3",
schema_definition=schema_def)
ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)
self.rr.create_association(inst_id, PRED.hasDataset, ds_id)
ds_filename = self.container.file_system.get("%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
self.assertFalse(os.path.exists(ds_filename))
inst_data_t0 = self.scion_client.get_asset_data(inst_id)
self.assertEquals(inst_data_t0["dataset_id"], ds_id)
self.assertEquals(inst_data_t0["num_rows"], 0)
# Install a data packet catcher
self.recv_packets, self.recv_rows = [], 0
def process_packet_cb(packet, route, stream):
if not isinstance(packet, DataPacket):
log.warn("Received a non DataPacket message")
self.recv_packets.append(packet)
self.recv_rows += len(packet.data["data"])
log.info("Received data packet #%s: rows=%s, cols=%s", len(self.recv_packets), len(packet.data["data"]),
packet.data["cols"])
#log.info('Packet data: ' + str(packet.data))
def cleanup_stream_sub():
if self.stream_sub:
self.stream_sub.stop()
self.stream_sub = None
self.stream_sub = StreamSubscriber(process=self.scion_proc, stream="basic_streams", callback=process_packet_cb)
self.stream_sub.start()
self.addCleanup(cleanup_stream_sub)
# Start agent
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
agent_pid = self.scion_client.start_agent(inst_id)
self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))
sac = StreamingAgentClient(resource_id=inst_id, process=self.scion_proc)
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)
sac.connect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)
# Coming in from the agent config.
streaming_args = {
'orb_name' : 'taexport.ucsd.edu:usarrayTA',
#'select' : 'TA_109C/MGENC/M40',
'select' : 'TA_121A/MGENC/M40',
'--timeout' : 5,
'sample_interval' : 5
}
sac.start_streaming(streaming_args)
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_STREAMING)
gevent.sleep(180)
# Retrieve data
"""
gevent.sleep(1)
self.assertTrue(os.path.exists(ds_filename))
inst_data_t1 = self.scion_client.get_asset_data(inst_id)
self.assertEquals(inst_data_t1["dataset_id"], ds_id)
self.assertEquals(inst_data_t1["variables"], ['time', 'cpu_percent'])
self.assertIn("cpu_percent", inst_data_t1["data"])
num_rows_t1 = inst_data_t1["num_rows"]
self.assertGreaterEqual(num_rows_t1, 2)
gevent.sleep(1)
inst_data_t2 = self.scion_client.get_asset_data(inst_id)
num_rows_t2 = inst_data_t2["num_rows"]
self.assertGreater(num_rows_t2, num_rows_t1)
self.assertGreaterEqual(len(self.recv_packets), self.recv_rows)
self.assertLessEqual(abs(self.recv_rows - num_rows_t2), 2)
"""
# Take down agent
sac.stop_streaming() # Not required to stop agent, just to test here
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_CONNECTED)
sac.disconnect()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"], StreamingAgent.AGENTSTATE_INITIALIZED)
self.scion_client.stop_agent(inst_id)
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
def test_scion_agent(self):
# Create user
actor_id = self.scion_client.define_user(first_name="John",
last_name="Doe",
username="******",
password="******",
email="*****@*****.**")
# Create instrument
data_filename = os.path.join(
os.path.split(__file__)[0], "testdata/orb_replay_data.yml")
agent_info = [
dict(
agent_type="data_agent",
config=dict(
plugin=
"scion.agent.model.orb.orb_replay_plugin.ORBReplay_DataAgentPlugin",
sampling_interval=0.2,
stream_name="basic_streams",
replay_file=data_filename,
replay_scenario="basic",
auto_streaming=False))
]
inst_obj = Instrument(name="ORB Sensor 1",
description="Seismic stream",
location=GeospatialLocation(
latitude=32.867079, longitude=-117.257324),
agent_info=agent_info)
inst_id, _ = self.rr.create(inst_obj, actor_id=actor_id)
# Create dataset
schema_def = DataSchemaParser.parse_schema_ref("ds_orb01_main")
ds_obj = Dataset(name="ORB Dataset 1", schema_definition=schema_def)
ds_id, _ = self.rr.create(ds_obj, actor_id=actor_id)
self.rr.create_association(inst_id, PRED.hasDataset, ds_id)
ds_filename = self.container.file_system.get(
"%s/%s%s.hdf5" % (DS_BASE_PATH, DS_FILE_PREFIX, ds_id))
# Install a data packet catcher
self.recv_packets, self.recv_rows = [], 0
def process_packet_cb(packet, route, stream):
if not isinstance(packet, DataPacket):
log.warn("Received a non DataPacket message")
self.recv_packets.append(packet)
self.recv_rows += len(packet.data["data"])
log.info("Received data packet #%s: rows=%s, cols=%s",
len(self.recv_packets), len(packet.data["data"]),
packet.data["cols"])
def cleanup_stream_sub():
if self.stream_sub:
self.stream_sub.stop()
self.stream_sub = None
self.stream_sub = StreamSubscriber(process=self.scion_proc,
stream="basic_streams",
callback=process_packet_cb)
self.stream_sub.start()
self.addCleanup(cleanup_stream_sub)
# Start agent
agent_pid = self.scion_client.start_agent(inst_id)
self.assertTrue(StreamingAgentClient.is_agent_active(inst_id))
sac = StreamingAgentClient(resource_id=inst_id,
process=self.scion_proc)
sac.connect()
sac.start_streaming()
agent_status = sac.get_status()
self.assertEquals(agent_status["current_state"],
StreamingAgent.AGENTSTATE_STREAMING)
# Retrieve data
gevent.sleep(1.5)
self.assertTrue(os.path.exists(ds_filename))
inst_data_t2 = self.scion_client.get_asset_data(inst_id)
print "T2", inst_data_t2
num_rows_t2 = inst_data_t2["num_rows"]
self.assertEquals(num_rows_t2, 40)
sample_data = inst_data_t2["data"]["sample_vector"]
self.assertEquals(sample_data[0][1], 100)
self.assertEquals(sample_data[39][1], 409)
self.assertEquals(sample_data[1][0] - sample_data[0][0], 100)
self.assertEquals(sample_data[39][0] - sample_data[0][0], 3900)
# Take down agent
sac.stop_streaming() # Not required to stop agent, just to test here
sac.disconnect()
self.scion_client.stop_agent(inst_id)
self.assertFalse(StreamingAgentClient.is_agent_active(inst_id))
inst_data = self.scion_client.get_asset_data(
inst_id, data_filter=dict(max_rows=20))
self.assertEquals(len(inst_data["data"]["sample_vector"]), 20)
self.assertEquals(inst_data["data"]["sample_vector"][0][1], 300)
inst_data = self.scion_client.get_asset_data(
inst_id, data_filter=dict(start_time="1460694001000"))
self.assertEquals(len(inst_data["data"]["sample_vector"]), 30)
self.assertEquals(inst_data["data"]["sample_vector"][0][1], 200)
# Note: Time filter applies before expansion
inst_data = self.scion_client.get_asset_data(
inst_id,
data_filter=dict(start_time="1460694001000",
start_time_include=False))
self.assertEquals(len(inst_data["data"]["sample_vector"]), 20)
self.assertEquals(inst_data["data"]["sample_vector"][0][1], 300)
inst_data = self.scion_client.get_asset_data(
inst_id, data_filter=dict(start_time="1460694001000", max_rows=10))
self.assertEquals(len(inst_data["data"]["sample_vector"]), 10)
self.assertEquals(inst_data["data"]["sample_vector"][0][1], 400)
inst_data = self.scion_client.get_asset_data(
inst_id,
data_filter=dict(start_time="1460694001000",
end_time="1460694002000"))
print inst_data
self.assertEquals(len(inst_data["data"]["sample_vector"]), 20)
self.assertEquals(inst_data["data"]["sample_vector"][0][1], 200)
