def test_dca_ingestion_pause_resume(self):
data_product_id, dataset_id = self.make_ctd_data_product()
streamer = Streamer(data_product_id, interval=1)
self.addCleanup(streamer.stop)
# Let a couple samples accumulate
self.use_monitor(dataset_id, samples=2)
# Go into DCA and get an editable handle to the coverage
with DirectCoverageAccess() as dca:
with dca.get_editable_coverage(dataset_id) as cov: # <-- This pauses ingestion
monitor = DatasetMonitor(dataset_id)
monitor.event.wait(7) # <-- ~7 Samples should accumulate on the ingestion queue
self.assertFalse(monitor.event.is_set()) # Verifies that nothing was processed (i.e. ingestion is actually paused)
monitor.stop()
# Stop the streamer
streamer.stop()
cont = True
while cont:
monitor = DatasetMonitor(dataset_id)
if not monitor.event.wait(10):
cont = False
monitor.stop()
with DirectCoverageAccess() as dca:
with dca.get_read_only_coverage(dataset_id) as cov:
self.assertGreaterEqual(cov.num_timesteps, 8)
def test_logger(self):
data_product_id = self.make_ctd_data_product()
# Clone the data product so we have an output
clone_id = self.clone_data_product(data_product_id)
data_process_id = self.create_data_process_logger(
data_product_id, clone_id, {'x': 'temp'})
dataset_monitor = DatasetMonitor(data_product_id=data_product_id)
self.addCleanup(dataset_monitor.stop)
# Put some data into the the data product
rdt = self.ph.rdt_for_data_product(data_product_id)
rdt['time'] = np.arange(40)
rdt['temp'] = np.arange(40)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(dataset_monitor.wait())
# Watch the output
dataset_monitor = DatasetMonitor(data_product_id=clone_id)
self.addCleanup(dataset_monitor.stop)
# Run the replay
self.data_process_management.activate_data_process(data_process_id)
# Make sure data came out
self.assertTrue(dataset_monitor.wait())
def test_instrument_simple(self):
instrument_model_id = self.create_instrument_model()
instrument_agent_id = self.create_instrument_agent(instrument_model_id)
instrument_device_id = self.create_instrument_device(instrument_model_id)
instrument_agent_instance_id = self.create_instrument_agent_instance(instrument_agent_id, instrument_device_id)
raw_dp_id, parsed_dp_id = self.create_instrument_data_products(instrument_device_id)
self.start_instrument_agent_instance(instrument_agent_instance_id)
agent_process_id = self.poll_instrument_agent_instance(instrument_agent_instance_id, instrument_device_id)
agent_client = ResourceAgentClient(instrument_device_id,
to_name=agent_process_id,
process=FakeProcess())
self.agent_state_transition(agent_client, ResourceAgentEvent.INITIALIZE, ResourceAgentState.INACTIVE)
self.agent_state_transition(agent_client, ResourceAgentEvent.GO_ACTIVE, ResourceAgentState.IDLE)
self.agent_state_transition(agent_client, ResourceAgentEvent.RUN, ResourceAgentState.COMMAND)
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(parsed_dp_id)
for i in xrange(10):
monitor = DatasetMonitor(dataset_id=dataset_id)
agent_client.execute_resource(AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE))
if not monitor.event.wait(30):
raise AssertionError('Failed on the %ith granule' % i)
monitor.stop()
rdt = RecordDictionaryTool.load_from_granule(self.data_retriever.retrieve(dataset_id))
self.assertEquals(len(rdt), 10)
def test_coefficient_compatibility(self):
data_product_id = self.create_data_product(
name='Calibration Coefficient Test Data product',
stream_def_id=self.stream_def_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = np.arange(10)
rdt['temp'] = [10] * 10
rdt['cc_coefficient'] = [2] * 10
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(
data_product_id)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(dataset_monitor.wait())
rdt2 = RecordDictionaryTool.load_from_granule(
self.data_retriever.retrieve(dataset_id))
np.testing.assert_array_equal(rdt2['offset'], [12] * 10)
def test_append_parameter(self):
# Make a CTDBP Data Product
data_product_id = self.make_ctd_data_product()
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(
data_product_id)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
# Throw some data in it
rdt = self.ph.rdt_for_data_product(data_product_id)
rdt['time'] = np.arange(30)
rdt['temp'] = np.arange(30)
rdt['pressure'] = np.arange(30)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(dataset_monitor.wait())
dataset_monitor.event.clear()
# Grab the egg
egg_url = self.egg_url
egg_path = TransformWorker.download_egg(egg_url)
import pkg_resources
pkg_resources.working_set.add_entry(egg_path)
self.addCleanup(os.remove, egg_path)
# Make a parameter function
owner = 'ion_example.add_arrays'
func = 'add_arrays'
arglist = ['a', 'b']
pf = ParameterFunction(name='add_arrays',
function_type=PFT.PYTHON,
owner=owner,
function=func,
args=arglist)
pfunc_id = self.dataset_management.create_parameter_function(pf)
self.addCleanup(self.dataset_management.delete_parameter_function,
pfunc_id)
# Make a context (instance of the function)
context = ParameterContext(name='array_sum',
units="1",
fill_value="-9999",
parameter_function_id=pfunc_id,
parameter_type="function",
value_encoding="float32",
display_name="Array Summation",
parameter_function_map={
'a': 'temp',
'b': 'pressure'
})
#pfunc = DatasetManagementService.get_coverage_function(pf)
#pfunc.param_map = {'a':'temp', 'b':'pressure'}
#ctxt = ParameterContext('array_sum', param_type=ParameterFunctionType(pfunc))
#ctxt_dump = ctxt.dump()
#ctxt_id = self.dataset_management.create_parameter_context('array_sum', ctxt_dump)
ctxt_id = self.dataset_management.create_parameter(context)
self.dataset_management.add_parameter_to_dataset(ctxt_id, dataset_id)
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['array_sum'], np.arange(0, 60, 2))
def init_check(self):
self.dp_id = self.create_data_product(name=uuid4().hex, stream_def_id=self.stream_def_id)
self.data_product_management.activate_data_product_persistence(self.dp_id)
self.addCleanup(self.data_product_management.suspend_data_product_persistence, self.dp_id)
self.dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(self.dp_id)
self.dataset_monitor = DatasetMonitor(self.dataset_id)
self.addCleanup(self.dataset_monitor.stop)
def test_array_flow_paths(self):
data_product_id, stream_def_id = self.make_array_data_product()
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id)
dm = DatasetMonitor(dataset_id)
self.addCleanup(dm.stop)
# I need to make sure that we can fill the RDT with its values
# Test for one timestep
# Test for multiple timesteps
# Publishes
# Ingests correctly
# Retrieves correctly
#--------------------------------------------------------------------------------
# Ensure that the RDT can be filled with ArrayType values
#--------------------------------------------------------------------------------
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [0]
rdt['temp_sample'] = [[0,1,2,3,4]]
np.testing.assert_array_equal(rdt['temp_sample'], np.array([[0,1,2,3,4]]))
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(dm.event.wait(10))
dm.event.clear()
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['temp_sample'], np.array([[0,1,2,3,4]]))
#--------------------------------------------------------------------------------
# Ensure that it deals with multiple values
#--------------------------------------------------------------------------------
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [1,2,3]
rdt['temp_sample'] = [[0,1,2,3,4],[1],[5,5,5,5,5]]
m = rdt.fill_value('temp_sample') or np.finfo(np.float32).max
np.testing.assert_equal(m,np.finfo(np.float32).max)
np.testing.assert_array_equal(rdt['temp_sample'], [[0,1,2,3,4],[1,m,m,m,m],[5,5,5,5,5]])
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(dm.event.wait(10))
dm.event.clear()
#--------------------------------------------------------------------------------
# Retrieve and Verify
#--------------------------------------------------------------------------------
retrieved_granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(retrieved_granule)
np.testing.assert_array_equal(rdt['time'], np.array([0,1,2,3]))
np.testing.assert_array_equal(rdt['temp_sample'], np.array([[0,1,2,3,4],[0,1,2,3,4],[1,m,m,m,m],[5,5,5,5,5]]))
def test_get_data_from_FDW(self):
# generate a data product and check that the FDW can get data
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(
1) # Yield to other greenlets, had an issue with connectivity
print "--------------------------------"
print dataset_id
coverage_path = DatasetManagementService()._get_coverage_path(
dataset_id)
print coverage_path
print "--------------------------------"
#verify table exists in the DB (similar to above)
# ....code...
# check that the geoserver layer exists as above
# ... code ....
# make a WMS/WFS request...somet like this (or both)
url = self.gs_host + '/geoserver/geonode/ows?service=WFS&version=1.0.0&request=GetFeature&typeName=geonode:ooi_' + dataset_id + '_ooi&maxFeatures=1&outputFormat=csv'
r = requests.get(url)
assertTrue(r.status_code == 200)
def test_create_dataset_verify_geoserver_layer(self):
#generate layer and check that the service created it in geoserver
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(
1) # Yield to other greenlets, had an issue with connectivity
log.debug("--------------------------------")
log.debug(dataset_id)
coverage_path = DatasetManagementService()._get_coverage_path(
dataset_id)
log.debug(coverage_path)
log.debug("--------------------------------")
# verify that the layer exists in geoserver
try:
r = requests.get(self.gs_rest_url + '/layers/ooi_' + dataset_id +
'_ooi.xml',
auth=(self.username, self.PASSWORD))
self.assertTrue(r.status_code == 200)
except Exception as e:
log.error("check service and layer exist...%s", e)
self.assertTrue(False)
def test_array_visualization(self):
data_product_id, stream_def_id = self.make_array_data_product()
# Make a granule with an array type, give it a few values
# Send it to google_dt transform, verify output
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(2208988800, 2208988810)
rdt['temp_sample'] = np.arange(10*4).reshape(10,4)
rdt['cond_sample'] = np.arange(10*4).reshape(10,4)
granule = rdt.to_granule()
dataset_monitor = DatasetMonitor(self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id))
self.addCleanup(dataset_monitor.stop)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
dataset_monitor.event.wait(10)
gdt_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('google_dt',id_only=True)
gdt_stream_def = self.create_stream_definition('gdt', parameter_dictionary_id=gdt_pdict_id)
gdt_data_granule = VizTransformGoogleDTAlgorithm.execute(granule, params=gdt_stream_def)
rdt = RecordDictionaryTool.load_from_granule(gdt_data_granule)
testval = {'data_content': [
[0.0 , 0.0 , 1.0 , 2.0 , 3.0 , 0.0 , 2.0 , 4.0 , 6.0 , 0.0 , 1.0 , 2.0 , 3.0] ,
[1.0 , 4.0 , 5.0 , 6.0 , 7.0 , 8.0 , 10.0 , 12.0 , 14.0 , 4.0 , 5.0 , 6.0 , 7.0] ,
[2.0 , 8.0 , 9.0 , 10.0 , 11.0 , 16.0 , 18.0 , 20.0 , 22.0 , 8.0 , 9.0 , 10.0 , 11.0] ,
[3.0 , 12.0 , 13.0 , 14.0 , 15.0 , 24.0 , 26.0 , 28.0 , 30.0 , 12.0 , 13.0 , 14.0 , 15.0] ,
[4.0 , 16.0 , 17.0 , 18.0 , 19.0 , 32.0 , 34.0 , 36.0 , 38.0 , 16.0 , 17.0 , 18.0 , 19.0] ,
[5.0 , 20.0 , 21.0 , 22.0 , 23.0 , 40.0 , 42.0 , 44.0 , 46.0 , 20.0 , 21.0 , 22.0 , 23.0] ,
[6.0 , 24.0 , 25.0 , 26.0 , 27.0 , 48.0 , 50.0 , 52.0 , 54.0 , 24.0 , 25.0 , 26.0 , 27.0] ,
[7.0 , 28.0 , 29.0 , 30.0 , 31.0 , 56.0 , 58.0 , 60.0 , 62.0 , 28.0 , 29.0 , 30.0 , 31.0] ,
[8.0 , 32.0 , 33.0 , 34.0 , 35.0 , 64.0 , 66.0 , 68.0 , 70.0 , 32.0 , 33.0 , 34.0 , 35.0] ,
[9.0 , 36.0 , 37.0 , 38.0 , 39.0 , 72.0 , 74.0 , 76.0 , 78.0 , 36.0 , 37.0 , 38.0 , 39.0]] ,
'data_description': [('time', 'number', 'time'),
('temp_sample[0]', 'number', 'temp_sample[0]', {'precision': '5'}),
('temp_sample[1]', 'number', 'temp_sample[1]', {'precision': '5'}),
('temp_sample[2]', 'number', 'temp_sample[2]', {'precision': '5'}),
('temp_sample[3]', 'number', 'temp_sample[3]', {'precision': '5'}),
('temp_offset[0]', 'number', 'temp_offset[0]', {'precision': '5'}),
('temp_offset[1]', 'number', 'temp_offset[1]', {'precision': '5'}),
('temp_offset[2]', 'number', 'temp_offset[2]', {'precision': '5'}),
('temp_offset[3]', 'number', 'temp_offset[3]', {'precision': '5'}),
('cond_sample[0]', 'number', 'cond_sample[0]', {'precision': '5'}),
('cond_sample[1]', 'number', 'cond_sample[1]', {'precision': '5'}),
('cond_sample[2]', 'number', 'cond_sample[2]', {'precision': '5'}),
('cond_sample[3]', 'number', 'cond_sample[3]', {'precision': '5'})],
'viz_product_type': 'google_dt'}
self.assertEquals(rdt['google_dt_components'][0], testval)
def push_granule(self, data_product_id):
'''
Publishes and monitors that the granule arrived
'''
datasets, _ = self.rrclient.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)
dataset_monitor = DatasetMonitor(datasets[0])
rdt = self.ph.rdt_for_data_product(data_product_id)
self.ph.fill_parsed_rdt(rdt)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
assert dataset_monitor.wait()
dataset_monitor.stop()
def test_retrieve_process(self):
data_product_id = self.make_ctd_data_product()
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(
data_product_id)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
rdt = self.ph.rdt_for_data_product(data_product_id)
date0 = datetime(2014, 1, 1, 0, 0) # 2014-01-01T00:00Z
time0 = calendar.timegm(date0.timetuple()) + 2208988800 # NTP
rdt['time'] = np.arange(time0, time0 + 30)
rdt['temp'] = np.arange(30)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(dataset_monitor.wait())
dataset_monitor.event.clear()
retrieve_process = RetrieveProcess(dataset_id)
rdt = retrieve_process.retrieve(date0, date0 + timedelta(hours=1))
np.testing.assert_array_equal(rdt['temp'], np.arange(30))
def test_create_dataset(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
# Yield to other greenlets, had an issue with connectivity
gevent.sleep(1)
log.debug("--------------------------------")
log.debug(dataset_id)
coverage_path = DatasetManagementService()._get_coverage_path(
dataset_id)
log.debug(coverage_path)
log.debug("--------------------------------")
breakpoint(locals(), globals())
def check_presta_instrument_data_products(self, reference_designator):
# Check the parsed data product make sure it's got everything it needs and can be published persisted etc.
# Absolute Pressure (SFLPRES_L0) is what comes off the instrumnet, SFLPRES_L1 is a pfunc
# Let's go ahead and publish some fake data!!!
# According to https://alfresco.oceanobservatories.org/alfresco/d/d/workspace/SpacesStore/63e16865-9d9e-4b11-b0b3-d5658faa5080/1341-00230_Data_Product_Spec_SFLPRES_OOI.pdf
# Appendix A. Example 1.
# p_psia_tide = 14.8670
# the tide should be 10.2504
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator,
info_list)
if not passing:
return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
now = time.time()
ntp_now = now + 2208988800.
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [ntp_now]
rdt['absolute_pressure'] = [14.8670]
passing &= self.assert_array_almost_equal(rdt['seafloor_pressure'],
[10.2504], 4)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(
dataset_monitor.wait()) # Bumped to 20 to keep buildbot happy
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['time'], [ntp_now])
passing &= self.assert_array_almost_equal(rdt['seafloor_pressure'],
[10.2504], 4)
passing &= self.assert_array_almost_equal(rdt['absolute_pressure'],
[14.8670], 4)
return passing
def test_fdt_created_during(self):
# generate a data product and check that the FDT exists
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(
1) # Yield to other greenlets, had an issue with connectivity
print "--------------------------------"
print dataset_id
coverage_path = DatasetManagementService()._get_coverage_path(
dataset_id)
print coverage_path
print "--------------------------------"
def test_creation_args(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
stream_def_id = self.create_stream_definition('ctd', parameter_dictionary_id=pdict_id)
data_product_id = self.create_data_product('ctd', stream_def_id=stream_def_id)
self.activate_data_product(data_product_id)
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id)
breakpoint(locals())
rdt = self.ph.get_rdt(stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
self.ph.publish_rdt_to_data_product(data_product_id,rdt)
dataset_monitor.event.wait(10)
breakpoint(locals())
granule = self.data_retriever.retrieve(dataset_id)
breakpoint(locals())
def test_pydap_handlers(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
stream_def_id = self.create_stream_definition('ctd', parameter_dictionary_id=pdict_id)
data_product_id = self.create_data_product('ctd', stream_def_id=stream_def_id)
self.activate_data_product(data_product_id)
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(data_product_id)
rdt = self.ph.get_rdt(stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
self.ph.publish_rdt_to_data_product(data_product_id,rdt)
dataset_monitor.event.wait(10)
from pydap.client import open_url
pydap_host = CFG.get_safe('server.pydap.host','localhost')
pydap_port = CFG.get_safe('server.pydap.port',8001)
url = 'http://%s:%s/%s' %(pydap_host, pydap_port, dataset_id)
ds = open_url(url)
ds['temp']['temp'][:]
def setup_resource(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
# Yield to other greenlets, had an issue with connectivity
gevent.sleep(1)
self.offering_id = dataset_id
def check_tempsf_instrument_data_product(self, reference_designator):
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator,
info_list)
if not passing: return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
now = time.time()
ntp_now = now + 2208988800
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [ntp_now]
rdt['temperature'] = [[
25.3884, 26.9384, 24.3394, 23.3401, 22.9832, 29.4434, 26.9873,
15.2883, 16.3374, 14.5883, 15.7253, 18.4383, 15.3488, 17.2993,
10.2111, 11.5993, 10.9345, 9.4444, 9.9876, 10.9834, 11.0098,
5.3456, 4.2994, 4.3009
]]
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
passing &= self.assertTrue(dataset_monitor.wait())
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['time'], [ntp_now])
passing &= self.assert_array_almost_equal(rdt['temperature'], [[
25.3884, 26.9384, 24.3394, 23.3401, 22.9832, 29.4434, 26.9873,
15.2883, 16.3374, 14.5883, 15.7253, 18.4383, 15.3488, 17.2993,
10.2111, 11.5993, 10.9345, 9.4444, 9.9876, 10.9834, 11.0098,
5.3456, 4.2994, 4.3009
]])
return passing
def create_device_site_deployment(self,
dep_name="Deployment",
starting=''):
from interface.objects import StreamConfiguration, StreamConfigurationType, InstrumentDevice
from interface.objects import InstrumentModel, PlatformAgent, InstrumentSite, TemporalBounds, Deployment
from interface.objects import RemotePlatformDeploymentContext
stream_conf = StreamConfiguration(
stream_name="CTD 1 Parsed Stream",
parameter_dictionary_name='ctd_parsed_param_dict',
stream_type=StreamConfigurationType.PARSED)
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict')
stream_def_id = self.create_stream_definition(
name='CTD 1', parameter_dictionary_id=pdict_id)
data_product_id = self.create_data_product(
name="DDP_1",
stream_def_id=stream_def_id,
stream_configuration=stream_conf)
self.activate_data_product(data_product_id)
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(
data_product_id)
stream_def = self.resource_registry.find_objects(
data_product_id, PRED.hasStreamDefinition)[0][0]
param_dict = self.resource_registry.find_objects(
stream_def._id, PRED.hasParameterDictionary)[0][0]
# Add data to the DataProduct
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
rdt = self.ph.get_rdt(stream_def._id)
rdt_ = self.ph.rdt_for_data_product(data_product_id)
self.assertEquals(rdt.fields, rdt_.fields)
rdt['time'] = [0, 1, 2, 3]
rdt['temp'] = [10, 11, 12, 13]
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(dataset_monitor.wait())
# Create Device
device = InstrumentDevice(name='Device 1')
device_id = self.instrument_management.create_instrument_device(device)
self.data_acquisition_management.register_instrument(device_id)
self.data_acquisition_management.assign_data_product(
device_id, data_product_id)
# Create Model
model = InstrumentModel(name='Model 1')
model_id = self.instrument_management.create_instrument_model(model)
self.instrument_management.assign_instrument_model_to_instrument_device(
model_id, device_id)
# Create AgentDefinition
ad = PlatformAgent(stream_configurations=[stream_conf])
ad_id, _ = self.resource_registry.create(ad)
# Create Site
site = InstrumentSite(name='Site 1',
stream_configurations=[stream_conf])
site_id, _ = self.resource_registry.create(site)
self.resource_registry.create_association(site_id, PRED.hasModel,
model_id)
self.resource_registry.create_association(site_id,
PRED.hasAgentDefinition,
ad_id)
# TemporalBounds of the Deployment
temp_bounds = TemporalBounds(start_datetime=starting, end_datetime='')
# Create Deployment
deployment = Deployment(name=dep_name,
type="RemotePlatform",
context=RemotePlatformDeploymentContext(),
constraint_list=[temp_bounds])
deployment_id = self.observatory_management.create_deployment(
deployment=deployment, site_id=site_id, device_id=device_id)
return site_id, device_id, dataset_id, deployment_id, param_dict, data_product_id
def test_qc_interval_integration(self):
# 1 need to make a dataset that only has one discrete qc violation
# 2 Launch the process
# 3 Setup the scheduler to run it say three times
# 4 Get the Events and verify the data
#--------------------------------------------------------------------------------
# Make a dataset that has only one discrete qc violation
#--------------------------------------------------------------------------------
dp_id, dataset_id, stream_def_id = self.make_data_product()
ph = ParameterHelper(self.dataset_management, self.addCleanup)
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
for rdt in self.populate_vectors(stream_def_id, 1,
lambda x: [41] + [39] * (x - 1)):
ph.publish_rdt_to_data_product(dp_id, rdt)
self.assertTrue(
monitor.event.wait(10)) # Give it 10 seconds to populate
#--------------------------------------------------------------------------------
# Launch the process
#--------------------------------------------------------------------------------
interval_key = uuid4().hex
config = DotDict()
config.process.interval_key = interval_key
config.process.qc_params = [
'glblrng_qc'
] # The others are tested in other tests for completeness
self.sync_launch(config)
async_queue = Queue()
def callback(event, *args, **kwargs):
times = event.temporal_values
self.assertEquals(len(times), 1)
async_queue.put(1)
es = EventSubscriber(event_type=OT.ParameterQCEvent,
origin=dp_id,
callback=callback,
auto_delete=True)
es.start()
self.addCleanup(es.stop)
#--------------------------------------------------------------------------------
# Setup the scheduler
#--------------------------------------------------------------------------------
timer_id = self.scheduler_service.create_interval_timer(
start_time=time.time(),
end_time=time.time() + 13,
interval=5,
event_origin=interval_key)
#--------------------------------------------------------------------------------
# Get the events and verify them
#--------------------------------------------------------------------------------
try:
for i in xrange(2):
async_queue.get(timeout=10)
except Empty:
raise AssertionError('QC Events not raised')
def test_add_parameter_function(self):
# req-tag: NEW SA - 31
# Make a CTDBP Data Product
data_product_id = self.make_ctd_data_product()
self.data_product_id = data_product_id
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(
data_product_id)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
# Throw some data in it
rdt = self.ph.rdt_for_data_product(data_product_id)
rdt['time'] = np.arange(30)
rdt['temp'] = np.arange(30)
rdt['pressure'] = np.arange(30)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(dataset_monitor.wait())
dataset_monitor.event.clear()
#--------------------------------------------------------------------------------
# This is what the user defines either via preload or through the UI
#--------------------------------------------------------------------------------
# Where the egg is
egg_url = self.egg_url
# Make a parameter function
owner = 'ion_example.add_arrays'
func = 'add_arrays'
arglist = ['a', 'b']
pf = ParameterFunction(name='add_arrays',
function_type=PFT.PYTHON,
owner=owner,
function=func,
args=arglist,
egg_uri=egg_url)
pfunc_id = self.dataset_management.create_parameter_function(pf)
#--------------------------------------------------------------------------------
self.addCleanup(self.dataset_management.delete_parameter_function,
pfunc_id)
# Make a data process definition
dpd = DataProcessDefinition(name='add_arrays',
description='Sums two arrays')
dpd_id = self.data_process_management.create_data_process_definition(
dpd, pfunc_id)
# TODO: assert assoc exists
argmap = {'a': 'temp', 'b': 'pressure'}
dp_id = self.data_process_management.create_data_process(
dpd_id, [data_product_id],
argument_map=argmap,
out_param_name='array_sum')
# Verify that the function worked!
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['array_sum'], np.arange(0, 60, 2))
# Verify that we can inspect it as well
source_code = self.data_process_management.inspect_data_process_definition(
dpd_id)
self.assertEquals(source_code,
'def add_arrays(a, b):\n return a+b\n')
url = self.data_process_management.get_data_process_definition_url(
dpd_id)
self.assertEquals(
url,
'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
)
dpd_ids, _ = self.resource_registry.find_resources(
name='dataqc_spiketest',
restype=RT.DataProcessDefinition,
id_only=True)
dpd_id = dpd_ids[0]
url = self.data_process_management.get_data_process_definition_url(
dpd_id)
self.assertEquals(
url,
'https://github.com/ooici/ion-functions/blob/master/ion_functions/qc/qc_functions.py'
)
def test_lookup_values(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_lookups()
stream_def_id = self.pubsubcli.create_stream_definition(
'lookup', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition, stream_def_id)
data_product = DataProduct(name='lookup data product')
tdom, sdom = time_series_domain()
data_product.temporal_domain = tdom.dump()
data_product.spatial_domain = sdom.dump()
data_product_id = self.dpsc_cli.create_data_product(
data_product, stream_definition_id=stream_def_id)
self.addCleanup(self.dpsc_cli.delete_data_product, data_product_id)
data_producer = DataProducer(name='producer')
data_producer.producer_context = DataProcessProducerContext()
data_producer.producer_context.configuration['qc_keys'] = [
'offset_document'
]
data_producer_id, _ = self.rrclient.create(data_producer)
self.addCleanup(self.rrclient.delete, data_producer_id)
assoc, _ = self.rrclient.create_association(
subject=data_product_id,
object=data_producer_id,
predicate=PRED.hasDataProducer)
self.addCleanup(self.rrclient.delete_association, assoc)
document_keys = self.damsclient.list_qc_references(data_product_id)
self.assertEquals(document_keys, ['offset_document'])
svm = StoredValueManager(self.container)
svm.stored_value_cas('offset_document', {'offset_a': 2.0})
self.dpsc_cli.activate_data_product_persistence(data_product_id)
dataset_ids, _ = self.rrclient.find_objects(subject=data_product_id,
predicate=PRED.hasDataset,
id_only=True)
dataset_id = dataset_ids[0]
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [0]
rdt['temp'] = [20.]
granule = rdt.to_granule()
stream_ids, _ = self.rrclient.find_objects(subject=data_product_id,
predicate=PRED.hasStream,
id_only=True)
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
publisher = StandaloneStreamPublisher(stream_id, route)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['temp'], rdt2['temp'])
np.testing.assert_array_almost_equal(rdt2['calibrated'],
np.array([22.0]))
svm.stored_value_cas('updated_document', {'offset_a': 3.0})
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ep = EventPublisher(event_type=OT.ExternalReferencesUpdatedEvent)
ep.publish_event(origin=data_product_id,
reference_keys=['updated_document'])
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [1]
rdt['temp'] = [20.]
granule = rdt.to_granule()
gevent.sleep(2) # Yield so that the event goes through
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt2['temp'], np.array([20., 20.]))
np.testing.assert_array_almost_equal(rdt2['calibrated'],
np.array([22.0, 23.0]))
def use_monitor(self, dataset_id, samples=10, wait=10):
for x in xrange(samples):
monitor = DatasetMonitor(dataset_id)
monitor.event.wait(wait)
monitor.stop()
def test_pydap(self):
if not CFG.get_safe('bootstrap.use_pydap', False):
raise unittest.SkipTest('PyDAP is off (bootstrap.use_pydap)')
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='example')
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 10)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(
1) # Yield to other greenlets, had an issue with connectivity
pydap_host = CFG.get_safe('server.pydap.host', 'localhost')
pydap_port = CFG.get_safe('server.pydap.port', 8001)
url = 'http://%s:%s/%s' % (pydap_host, pydap_port, dataset_id)
for i in xrange(
3
): # Do it three times to test that the cache doesn't corrupt the requests/responses
ds = open_url(url)
np.testing.assert_array_equal(ds['time'][:], np.arange(10))
untested = []
for k, v in rdt.iteritems():
if k == rdt.temporal_parameter:
continue
context = rdt.context(k)
if isinstance(context.param_type, QuantityType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
elif isinstance(context.param_type, ArrayType):
if context.param_type.inner_encoding is None:
values = np.empty(rdt[k].shape, dtype='O')
for i, obj in enumerate(rdt[k]):
values[i] = str(obj)
np.testing.assert_array_equal(ds[k][k][:][0], values)
elif len(rdt[k].shape) > 1:
values = np.empty(rdt[k].shape[0], dtype='O')
for i in xrange(rdt[k].shape[0]):
values[i] = ','.join(
map(lambda x: str(x), rdt[k][i].tolist()))
elif isinstance(context.param_type, ConstantType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
elif isinstance(context.param_type, CategoryType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
else:
untested.append('%s (%s)' % (k, context.param_type))
if untested:
raise AssertionError('Untested parameters: %s' % untested)
def check_vel3d_instrument_data_products(self, reference_designator):
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator,
info_list)
if not passing:
return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
pdict = self.RR2.find_parameter_dictionary_of_stream_definition_using_has_parameter_dictionary(
stream_def_id)
self.assertEquals(pdict.name, 'vel3d_b_sample')
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
lat = 14.6846
lon = -51.044
ts = np.array([
3319563600, 3319567200, 3319570800, 3319574400, 3319578000,
3319581600, 3319585200, 3319588800, 3319592400, 3319596000
],
dtype=np.float)
ve = np.array([-3.2, 0.1, 0., 2.3, -0.1, 5.6, 5.1, 5.8, 8.8, 10.3])
vn = np.array([18.2, 9.9, 12., 6.6, 7.4, 3.4, -2.6, 0.2, -1.5, 4.1])
vu = np.array([-1.1, -0.6, -1.4, -2, -1.7, -2, 1.3, -1.6, -1.1, -4.5])
ve_expected = np.array([
-0.085136, -0.028752, -0.036007, 0.002136, -0.023158, 0.043218,
0.056451, 0.054727, 0.088446, 0.085952
])
vn_expected = np.array([
0.164012, 0.094738, 0.114471, 0.06986, 0.07029, 0.049237,
-0.009499, 0.019311, 0.012096, 0.070017
])
vu_expected = np.array([
-0.011, -0.006, -0.014, -0.02, -0.017, -0.02, 0.013, -0.016,
-0.011, -0.045
])
rdt['time'] = ts
rdt['lat'] = [lat] * 10
rdt['lon'] = [lon] * 10
rdt['turbulent_velocity_east'] = ve
rdt['turbulent_velocity_north'] = vn
rdt['turbulent_velocity_up'] = vu
passing &= self.assert_array_almost_equal(
rdt['eastward_turbulent_velocity'], ve_expected)
passing &= self.assert_array_almost_equal(
rdt['northward_turbulent_velocity'], vn_expected)
passing &= self.assert_array_almost_equal(
rdt['upward_turbulent_velocity'], vu_expected)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
passing &= self.assertTrue(dataset_monitor.wait())
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(
rdt['eastward_turbulent_velocity'], ve_expected)
passing &= self.assert_array_almost_equal(
rdt['northward_turbulent_velocity'], vn_expected)
passing &= self.assert_array_almost_equal(
rdt['upward_turbulent_velocity'], vu_expected)
return passing
def check_trhph_instrument_data_products(self, reference_designator):
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator,
info_list)
if not passing:
return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
pdict = self.RR2.find_parameter_dictionary_of_stream_definition_using_has_parameter_dictionary(
stream_def_id)
passing &= self.assertEquals(pdict.name, 'trhph_sample')
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
# calibration constants
a = 1.98e-9
b = -2.45e-6
c = 9.28e-4
d = -0.0888
e = 0.731
V_s = 1.506
V_c = 0.
T = 11.8
r1 = 0.906
r2 = 4.095
r3 = 4.095
ORP_V = 1.806
Cl = np.nan
offset = 2008
gain = 4.0
# Normally this would be 50 per the DPS but the precision is %4.0f which truncates the values to the nearest 1...
ORP = ((ORP_V * 1000.) - offset) / gain
ntp_now = time.time() + 2208988800
rdt['cc_a'] = [a]
rdt['cc_b'] = [b]
rdt['cc_c'] = [c]
rdt['cc_d'] = [d]
rdt['cc_e'] = [e]
rdt['ref_temp_volts'] = [V_s]
rdt['resistivity_temp_volts'] = [V_c]
rdt['eh_sensor'] = [ORP_V]
rdt['resistivity_5'] = [r1]
rdt['resistivity_x1'] = [r2]
rdt['resistivity_x5'] = [r3]
rdt['cc_offset'] = [offset]
rdt['cc_gain'] = [gain]
rdt['time'] = [ntp_now]
passing &= self.assert_array_almost_equal(rdt['vent_fluid_temperaure'],
[T], 2)
passing &= self.assert_array_almost_equal(
rdt['vent_fluid_chloride_conc'], [Cl], 4)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_orp'], [ORP],
4)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
passing &= self.assertTrue(dataset_monitor.wait())
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_temperaure'],
[T], 2)
passing &= self.assert_array_almost_equal(
rdt['vent_fluid_chloride_conc'], [Cl], 4)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_orp'], [ORP],
4)
return passing