def fill_temporal_gap(self, dataset_id, gap_coverage_path=None, gap_coverage_id=None): if gap_coverage_path is None and gap_coverage_id is None: raise ValueError('Must specify either \'gap_coverage_path\' or \'gap_coverage_id\'') if gap_coverage_path is None: gap_coverage_path = self.get_coverage_path(gap_coverage_id) from coverage_model import AbstractCoverage gap_cov = AbstractCoverage.load(gap_coverage_path) self.pause_ingestion(self.get_stream_id(dataset_id)) DatasetManagementService._splice_coverage(dataset_id, gap_cov)
def splice_coverage(self, dataset_id, coverage): log.info('Splicing new coverage') DatasetManagementService._splice_coverage(dataset_id, coverage)
def splice_coverage(self, dataset_id, coverage): log.info('Splicing new coverage') DatasetManagementService._splice_coverage(dataset_id, coverage)
def test_fill_temporal_gap(self): from ion.services.dm.inventory.dataset_management_service import DatasetManagementService data_product_id, dataset_id = self.make_ctd_data_product() pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict') streamer = Streamer(data_product_id, interval=0.5) self.addCleanup(streamer.stop) self.use_monitor(dataset_id, samples=10) streamer.stop() gap_times = [] waiter = Event() while not waiter.wait(1): gap_times.append(time.time() + 2208988800) if len(gap_times) == 10: waiter.set() # Simulate a gap by appending a new SimplexCoverage with times after the above gap with DirectCoverageAccess() as dca: dca.pause_ingestion(dataset_id) with dca.get_read_only_coverage(dataset_id) as cov: beforecovtimes = cov.get_time_values() with DatasetManagementService._create_simplex_coverage(dataset_id, pdict, None, None) as scov: scov.insert_timesteps(3) now = time.time() + 2208988800 ts = [now, now + 1, now + 2] scov.set_time_values(ts) aftercovtimes = scov.get_time_values() DatasetManagementService._splice_coverage(dataset_id, scov) # Start streaming data again streamer.start() # Create the gap-fill coverage with DatasetManagementService._create_simplex_coverage(dataset_id, pdict, None, None) as scov: scov.insert_timesteps(len(gap_times)) scov.set_time_values(gap_times) gap_cov_path = scov.persistence_dir gapcovtimes = scov.get_time_values() # Fill the gap and capture times to do some assertions with DirectCoverageAccess() as dca: with dca.get_read_only_coverage(dataset_id) as cov: otimes = cov.get_time_values() dca.fill_temporal_gap(dataset_id, gap_coverage_path=gap_cov_path) with dca.get_read_only_coverage(dataset_id) as cov: agtimes = cov.get_time_values() self.use_monitor(dataset_id, samples=5) with DirectCoverageAccess() as dca: with dca.get_read_only_coverage(dataset_id) as cov: ntimes = cov.get_time_values() self.assertLess(len(otimes), len(agtimes)) self.assertLess(len(agtimes), len(ntimes)) bctl = len(beforecovtimes) gctl = len(gapcovtimes) actl = len(aftercovtimes) np.testing.assert_array_equal(beforecovtimes, ntimes[:bctl]) np.testing.assert_array_equal(gapcovtimes, ntimes[bctl+1:bctl+gctl+1]) np.testing.assert_array_equal(aftercovtimes, ntimes[bctl+gctl+1:bctl+gctl+actl+1]) np.testing.assert_array_equal(agtimes, ntimes[:len(agtimes)])
def test_fill_temporal_gap(self): from ion.services.dm.inventory.dataset_management_service import DatasetManagementService data_product_id, dataset_id = self.make_ctd_data_product() pdict = DatasetManagementService.get_parameter_dictionary_by_name( 'ctd_parsed_param_dict') streamer = Streamer(data_product_id, interval=0.5) self.addCleanup(streamer.stop) self.use_monitor(dataset_id, samples=10) streamer.stop() gap_times = [] waiter = Event() while not waiter.wait(1): gap_times.append(time.time() + 2208988800) if len(gap_times) == 10: waiter.set() # Simulate a gap by appending a new SimplexCoverage with times after the above gap with DirectCoverageAccess() as dca: dca.pause_ingestion(dataset_id) with dca.get_read_only_coverage(dataset_id) as cov: beforecovtimes = cov.get_time_values() with DatasetManagementService._create_simplex_coverage( dataset_id, pdict, None, None) as scov: scov.insert_timesteps(3) now = time.time() + 2208988800 ts = [now, now + 1, now + 2] scov.set_time_values(ts) aftercovtimes = scov.get_time_values() DatasetManagementService._splice_coverage(dataset_id, scov) # Start streaming data again streamer.start() # Create the gap-fill coverage with DatasetManagementService._create_simplex_coverage( dataset_id, pdict, None, None) as scov: scov.insert_timesteps(len(gap_times)) scov.set_time_values(gap_times) gap_cov_path = scov.persistence_dir gapcovtimes = scov.get_time_values() # Fill the gap and capture times to do some assertions with DirectCoverageAccess() as dca: with dca.get_read_only_coverage(dataset_id) as cov: otimes = cov.get_time_values() dca.fill_temporal_gap(dataset_id, gap_coverage_path=gap_cov_path) with dca.get_read_only_coverage(dataset_id) as cov: agtimes = cov.get_time_values() self.use_monitor(dataset_id, samples=5) with DirectCoverageAccess() as dca: with dca.get_read_only_coverage(dataset_id) as cov: ntimes = cov.get_time_values() self.assertLess(len(otimes), len(agtimes)) self.assertLess(len(agtimes), len(ntimes)) bctl = len(beforecovtimes) gctl = len(gapcovtimes) actl = len(aftercovtimes) np.testing.assert_array_equal(beforecovtimes, ntimes[:bctl]) np.testing.assert_array_equal(gapcovtimes, ntimes[bctl + 1:bctl + gctl + 1]) np.testing.assert_array_equal( aftercovtimes, ntimes[bctl + gctl + 1:bctl + gctl + actl + 1]) np.testing.assert_array_equal(agtimes, ntimes[:len(agtimes)])