def test_dca_ingestion_pause_resume(self):
data_product_id, dataset_id = self.make_ctd_data_product()
streamer = Streamer(data_product_id, interval=1)
self.addCleanup(streamer.stop)
# Let a couple samples accumulate
self.use_monitor(dataset_id, samples=2)
# Go into DCA and get an editable handle to the coverage
with DirectCoverageAccess() as dca:
with dca.get_editable_coverage(dataset_id) as cov: # <-- This pauses ingestion
monitor = DatasetMonitor(dataset_id)
monitor.event.wait(7) # <-- ~7 Samples should accumulate on the ingestion queue
self.assertFalse(monitor.event.is_set()) # Verifies that nothing was processed (i.e. ingestion is actually paused)
monitor.stop()
# Stop the streamer
streamer.stop()
cont = True
while cont:
monitor = DatasetMonitor(dataset_id)
if not monitor.event.wait(10):
cont = False
monitor.stop()
with DirectCoverageAccess() as dca:
with dca.get_read_only_coverage(dataset_id) as cov:
self.assertGreaterEqual(cov.num_timesteps, 8)
def test_manual_data_upload(self):
data_product_id, dataset_id = self.make_manual_upload_data_product()
streamer = Streamer(data_product_id, interval=0.5, simple_time=True)
self.addCleanup(streamer.stop)
# Let at least 10 samples accumulate
self.use_monitor(dataset_id, samples=10)
# Verify that the HITL parameters are fill value
with DirectCoverageAccess() as dca:
with dca.get_read_only_coverage(dataset_id) as cov:
fillarr = np.array([False]*10)
for p in [p for p in cov.list_parameters() if p.endswith('_hitl_qc')]:
np.testing.assert_equal(cov.get_parameter_values(p, slice(None, 10)), fillarr)
# Upload the data - this pauses ingestion, performs the upload, and resumes ingestion
with DirectCoverageAccess() as dca:
dca.manual_upload(dataset_id, 'test_data/testmanualupload.csv', 'test_data/testmanualupload.yml')
streamer.stop()
# Wait a moment for ingestion to catch up
self.use_monitor(dataset_id, samples=2)
# Verify that the HITL parameters now have the correct values
want_vals = {
'temp_hitl_qc': np.array([0, 0, 0, 0, 1, 0, 0, 1, 0, 0], dtype=bool),
'cond_hitl_qc': np.array([1, 0, 1, 0, 0, 0, 1, 1, 0, 0], dtype=bool)
}
with DirectCoverageAccess() as dca:
with dca.get_read_only_coverage(dataset_id) as cov:
for p in [p for p in cov.list_parameters() if p.endswith('_hitl_qc')]:
np.testing.assert_equal(cov.get_parameter_values(p, slice(None, 10)), want_vals[p])
def test_repair_temporal_geometry(self):
data_product_id, dataset_id = self.make_ctd_data_product()
streamer = Streamer(data_product_id, interval=0.5, simple_time=True)
self.addCleanup(streamer.stop)
# Let at least 10 samples accumulate
self.use_monitor(dataset_id, samples=10)
# Stop the streamer, reset i, restart the streamer - this simulates duplicate data
streamer.stop()
streamer.i = 0
streamer.start()
# Let at least 20 more samples accumulate
self.use_monitor(dataset_id, samples=20)
#Stop the streamer
streamer.stop()
# Open the coverage and mess with the times
with DirectCoverageAccess() as dca:
with dca.get_read_only_coverage(dataset_id) as cov:
self.assertEqual(cov.num_timesteps, 30)
t = cov.get_time_values()
self.assertEqual(len(t), 30)
self.assertFalse(np.array_equal(np.sort(t), t))
dca.repair_temporal_geometry(dataset_id)
with dca.get_read_only_coverage(dataset_id) as cov:
self.assertGreaterEqual(cov.num_timesteps, 19)
t = cov.get_time_values()
self.assertGreaterEqual(len(t), 19)
np.testing.assert_array_equal(np.sort(t), t)
def test_dca_ingestion_pause_resume(self):
data_product_id, dataset_id = self.make_ctd_data_product()
streamer = Streamer(data_product_id, interval=1)
self.addCleanup(streamer.stop)
# Let a couple samples accumulate
self.use_monitor(dataset_id, samples=2)
# Go into DCA and get an editable handle to the coverage
with DirectCoverageAccess() as dca:
with dca.get_editable_coverage(dataset_id) as cov: # <-- This pauses ingestion
monitor = DatasetMonitor(dataset_id)
monitor.event.wait(7) # <-- ~7 Samples should accumulate on the ingestion queue
self.assertFalse(monitor.event.is_set()) # Verifies that nothing was processed (i.e. ingestion is actually paused)
monitor.stop()
# Stop the streamer
streamer.stop()
cont = True
while cont:
monitor = DatasetMonitor(dataset_id)
if not monitor.event.wait(10):
cont = False
monitor.stop()
with DirectCoverageAccess() as dca:
with dca.get_read_only_coverage(dataset_id) as cov:
self.assertGreaterEqual(cov.num_timesteps, 8)
def test_repair_temporal_geometry(self):
data_product_id, dataset_id = self.make_ctd_data_product()
streamer = Streamer(data_product_id, interval=0.5, simple_time=True)
self.addCleanup(streamer.stop)
# Let at least 10 samples accumulate
self.use_monitor(dataset_id, samples=10)
# Stop the streamer, reset i, restart the streamer - this simulates duplicate data
streamer.stop()
streamer.i = 0
streamer.start()
# Let at least 20 more samples accumulate
self.use_monitor(dataset_id, samples=20)
#Stop the streamer
streamer.stop()
# Open the coverage and mess with the times
with DirectCoverageAccess() as dca:
with dca.get_read_only_coverage(dataset_id) as cov:
self.assertEqual(cov.num_timesteps, 30)
t = cov.get_time_values()
self.assertEqual(len(t), 30)
self.assertFalse(np.array_equal(np.sort(t), t))
dca.repair_temporal_geometry(dataset_id)
with dca.get_read_only_coverage(dataset_id) as cov:
self.assertGreaterEqual(cov.num_timesteps, 19)
t = cov.get_time_values()
self.assertGreaterEqual(len(t), 19)
np.testing.assert_array_equal(np.sort(t), t)
def test_manual_data_upload(self):
data_product_id, dataset_id = self.make_manual_upload_data_product()
streamer = Streamer(data_product_id, interval=0.5, simple_time=True)
self.addCleanup(streamer.stop)
# Let at least 10 samples accumulate
self.use_monitor(dataset_id, samples=10)
# Verify that the HITL parameters are fill value
with DirectCoverageAccess() as dca:
with dca.get_read_only_coverage(dataset_id) as cov:
fillarr = np.array([False] * 10)
for p in [
p for p in cov.list_parameters()
if p.endswith('_hitl_qc')
]:
np.testing.assert_equal(
cov.get_parameter_values(p, slice(None, 10)), fillarr)
# Upload the data - this pauses ingestion, performs the upload, and resumes ingestion
with DirectCoverageAccess() as dca:
dca.manual_upload(dataset_id, 'test_data/testmanualupload.csv',
'test_data/testmanualupload.yml')
streamer.stop()
# Wait a moment for ingestion to catch up
self.use_monitor(dataset_id, samples=2)
# Verify that the HITL parameters now have the correct values
want_vals = {
'temp_hitl_qc': np.array([0, 0, 0, 0, 1, 0, 0, 1, 0, 0],
dtype=bool),
'cond_hitl_qc': np.array([1, 0, 1, 0, 0, 0, 1, 1, 0, 0],
dtype=bool)
}
with DirectCoverageAccess() as dca:
with dca.get_read_only_coverage(dataset_id) as cov:
for p in [
p for p in cov.list_parameters()
if p.endswith('_hitl_qc')
]:
np.testing.assert_equal(
cov.get_parameter_values(p, slice(None, 10)),
want_vals[p])
def test_fill_temporal_gap(self):
from ion.services.dm.inventory.dataset_management_service import DatasetManagementService
data_product_id, dataset_id = self.make_ctd_data_product()
pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict')
streamer = Streamer(data_product_id, interval=0.5)
self.addCleanup(streamer.stop)
self.use_monitor(dataset_id, samples=10)
streamer.stop()
gap_times = []
waiter = Event()
while not waiter.wait(1):
gap_times.append(time.time() + 2208988800)
if len(gap_times) == 10:
waiter.set()
# Simulate a gap by appending a new SimplexCoverage with times after the above gap
with DirectCoverageAccess() as dca:
dca.pause_ingestion(dataset_id)
with dca.get_read_only_coverage(dataset_id) as cov:
beforecovtimes = cov.get_time_values()
with DatasetManagementService._create_simplex_coverage(dataset_id, pdict, None, None) as scov:
scov.insert_timesteps(3)
now = time.time() + 2208988800
ts = [now, now + 1, now + 2]
scov.set_time_values(ts)
aftercovtimes = scov.get_time_values()
DatasetManagementService._splice_coverage(dataset_id, scov)
# Start streaming data again
streamer.start()
# Create the gap-fill coverage
with DatasetManagementService._create_simplex_coverage(dataset_id, pdict, None, None) as scov:
scov.insert_timesteps(len(gap_times))
scov.set_time_values(gap_times)
gap_cov_path = scov.persistence_dir
gapcovtimes = scov.get_time_values()
# Fill the gap and capture times to do some assertions
with DirectCoverageAccess() as dca:
with dca.get_read_only_coverage(dataset_id) as cov:
otimes = cov.get_time_values()
dca.fill_temporal_gap(dataset_id, gap_coverage_path=gap_cov_path)
with dca.get_read_only_coverage(dataset_id) as cov:
agtimes = cov.get_time_values()
self.use_monitor(dataset_id, samples=5)
with DirectCoverageAccess() as dca:
with dca.get_read_only_coverage(dataset_id) as cov:
ntimes = cov.get_time_values()
self.assertLess(len(otimes), len(agtimes))
self.assertLess(len(agtimes), len(ntimes))
bctl = len(beforecovtimes)
gctl = len(gapcovtimes)
actl = len(aftercovtimes)
np.testing.assert_array_equal(beforecovtimes, ntimes[:bctl])
np.testing.assert_array_equal(gapcovtimes, ntimes[bctl+1:bctl+gctl+1])
np.testing.assert_array_equal(aftercovtimes, ntimes[bctl+gctl+1:bctl+gctl+actl+1])
np.testing.assert_array_equal(agtimes, ntimes[:len(agtimes)])
def test_fill_temporal_gap(self):
from ion.services.dm.inventory.dataset_management_service import DatasetManagementService
data_product_id, dataset_id = self.make_ctd_data_product()
pdict = DatasetManagementService.get_parameter_dictionary_by_name(
'ctd_parsed_param_dict')
streamer = Streamer(data_product_id, interval=0.5)
self.addCleanup(streamer.stop)
self.use_monitor(dataset_id, samples=10)
streamer.stop()
gap_times = []
waiter = Event()
while not waiter.wait(1):
gap_times.append(time.time() + 2208988800)
if len(gap_times) == 10:
waiter.set()
# Simulate a gap by appending a new SimplexCoverage with times after the above gap
with DirectCoverageAccess() as dca:
dca.pause_ingestion(dataset_id)
with dca.get_read_only_coverage(dataset_id) as cov:
beforecovtimes = cov.get_time_values()
with DatasetManagementService._create_simplex_coverage(
dataset_id, pdict, None, None) as scov:
scov.insert_timesteps(3)
now = time.time() + 2208988800
ts = [now, now + 1, now + 2]
scov.set_time_values(ts)
aftercovtimes = scov.get_time_values()
DatasetManagementService._splice_coverage(dataset_id, scov)
# Start streaming data again
streamer.start()
# Create the gap-fill coverage
with DatasetManagementService._create_simplex_coverage(
dataset_id, pdict, None, None) as scov:
scov.insert_timesteps(len(gap_times))
scov.set_time_values(gap_times)
gap_cov_path = scov.persistence_dir
gapcovtimes = scov.get_time_values()
# Fill the gap and capture times to do some assertions
with DirectCoverageAccess() as dca:
with dca.get_read_only_coverage(dataset_id) as cov:
otimes = cov.get_time_values()
dca.fill_temporal_gap(dataset_id, gap_coverage_path=gap_cov_path)
with dca.get_read_only_coverage(dataset_id) as cov:
agtimes = cov.get_time_values()
self.use_monitor(dataset_id, samples=5)
with DirectCoverageAccess() as dca:
with dca.get_read_only_coverage(dataset_id) as cov:
ntimes = cov.get_time_values()
self.assertLess(len(otimes), len(agtimes))
self.assertLess(len(agtimes), len(ntimes))
bctl = len(beforecovtimes)
gctl = len(gapcovtimes)
actl = len(aftercovtimes)
np.testing.assert_array_equal(beforecovtimes, ntimes[:bctl])
np.testing.assert_array_equal(gapcovtimes,
ntimes[bctl + 1:bctl + gctl + 1])
np.testing.assert_array_equal(
aftercovtimes, ntimes[bctl + gctl + 1:bctl + gctl + actl + 1])
np.testing.assert_array_equal(agtimes, ntimes[:len(agtimes)])
