def gap_coverage(self,stream_id): try: old_cov = self._coverages.pop(stream_id) dataset_id = self.get_dataset(stream_id) sdom, tdom = time_series_domain() new_cov = DatasetManagementService._create_simplex_coverage(dataset_id, old_cov.parameter_dictionary, sdom, tdom, old_cov._persistence_layer.inline_data_writes) old_cov.close() result = new_cov except KeyError: result = self.get_coverage(stream_id) self._coverages[stream_id] = result return result
def gap_coverage(self, stream_id): try: old_cov = self._coverages.pop(stream_id) dataset_id = self.get_dataset(stream_id) sdom, tdom = time_series_domain() new_cov = DatasetManagementService._create_simplex_coverage( dataset_id, old_cov.parameter_dictionary, sdom, tdom, old_cov._persistence_layer.inline_data_writes) old_cov.close() result = new_cov except KeyError: result = self.get_coverage(stream_id) self._coverages[stream_id] = result return result
def test_fill_temporal_gap(self): from ion.services.dm.inventory.dataset_management_service import DatasetManagementService data_product_id, dataset_id = self.make_ctd_data_product() pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict') streamer = Streamer(data_product_id, interval=0.5) self.addCleanup(streamer.stop) self.use_monitor(dataset_id, samples=10) streamer.stop() gap_times = [] waiter = Event() while not waiter.wait(1): gap_times.append(time.time() + 2208988800) if len(gap_times) == 10: waiter.set() # Simulate a gap by appending a new SimplexCoverage with times after the above gap with DirectCoverageAccess() as dca: dca.pause_ingestion(dataset_id) with dca.get_read_only_coverage(dataset_id) as cov: beforecovtimes = cov.get_time_values() with DatasetManagementService._create_simplex_coverage(dataset_id, pdict, None, None) as scov: scov.insert_timesteps(3) now = time.time() + 2208988800 ts = [now, now + 1, now + 2] scov.set_time_values(ts) aftercovtimes = scov.get_time_values() DatasetManagementService._splice_coverage(dataset_id, scov) # Start streaming data again streamer.start() # Create the gap-fill coverage with DatasetManagementService._create_simplex_coverage(dataset_id, pdict, None, None) as scov: scov.insert_timesteps(len(gap_times)) scov.set_time_values(gap_times) gap_cov_path = scov.persistence_dir gapcovtimes = scov.get_time_values() # Fill the gap and capture times to do some assertions with DirectCoverageAccess() as dca: with dca.get_read_only_coverage(dataset_id) as cov: otimes = cov.get_time_values() dca.fill_temporal_gap(dataset_id, gap_coverage_path=gap_cov_path) with dca.get_read_only_coverage(dataset_id) as cov: agtimes = cov.get_time_values() self.use_monitor(dataset_id, samples=5) with DirectCoverageAccess() as dca: with dca.get_read_only_coverage(dataset_id) as cov: ntimes = cov.get_time_values() self.assertLess(len(otimes), len(agtimes)) self.assertLess(len(agtimes), len(ntimes)) bctl = len(beforecovtimes) gctl = len(gapcovtimes) actl = len(aftercovtimes) np.testing.assert_array_equal(beforecovtimes, ntimes[:bctl]) np.testing.assert_array_equal(gapcovtimes, ntimes[bctl+1:bctl+gctl+1]) np.testing.assert_array_equal(aftercovtimes, ntimes[bctl+gctl+1:bctl+gctl+actl+1]) np.testing.assert_array_equal(agtimes, ntimes[:len(agtimes)])
def test_fill_temporal_gap(self): from ion.services.dm.inventory.dataset_management_service import DatasetManagementService data_product_id, dataset_id = self.make_ctd_data_product() pdict = DatasetManagementService.get_parameter_dictionary_by_name( 'ctd_parsed_param_dict') streamer = Streamer(data_product_id, interval=0.5) self.addCleanup(streamer.stop) self.use_monitor(dataset_id, samples=10) streamer.stop() gap_times = [] waiter = Event() while not waiter.wait(1): gap_times.append(time.time() + 2208988800) if len(gap_times) == 10: waiter.set() # Simulate a gap by appending a new SimplexCoverage with times after the above gap with DirectCoverageAccess() as dca: dca.pause_ingestion(dataset_id) with dca.get_read_only_coverage(dataset_id) as cov: beforecovtimes = cov.get_time_values() with DatasetManagementService._create_simplex_coverage( dataset_id, pdict, None, None) as scov: scov.insert_timesteps(3) now = time.time() + 2208988800 ts = [now, now + 1, now + 2] scov.set_time_values(ts) aftercovtimes = scov.get_time_values() DatasetManagementService._splice_coverage(dataset_id, scov) # Start streaming data again streamer.start() # Create the gap-fill coverage with DatasetManagementService._create_simplex_coverage( dataset_id, pdict, None, None) as scov: scov.insert_timesteps(len(gap_times)) scov.set_time_values(gap_times) gap_cov_path = scov.persistence_dir gapcovtimes = scov.get_time_values() # Fill the gap and capture times to do some assertions with DirectCoverageAccess() as dca: with dca.get_read_only_coverage(dataset_id) as cov: otimes = cov.get_time_values() dca.fill_temporal_gap(dataset_id, gap_coverage_path=gap_cov_path) with dca.get_read_only_coverage(dataset_id) as cov: agtimes = cov.get_time_values() self.use_monitor(dataset_id, samples=5) with DirectCoverageAccess() as dca: with dca.get_read_only_coverage(dataset_id) as cov: ntimes = cov.get_time_values() self.assertLess(len(otimes), len(agtimes)) self.assertLess(len(agtimes), len(ntimes)) bctl = len(beforecovtimes) gctl = len(gapcovtimes) actl = len(aftercovtimes) np.testing.assert_array_equal(beforecovtimes, ntimes[:bctl]) np.testing.assert_array_equal(gapcovtimes, ntimes[bctl + 1:bctl + gctl + 1]) np.testing.assert_array_equal( aftercovtimes, ntimes[bctl + gctl + 1:bctl + gctl + actl + 1]) np.testing.assert_array_equal(agtimes, ntimes[:len(agtimes)])