def retrieve(self, dataset_id='', query=None, delivery_format=None, module='', cls='', kwargs=None): if query is None: query = {} if delivery_format is None: delivery_format = {} validate_is_instance(query,dict,'Query was improperly formatted.') validate_true(dataset_id, 'No dataset provided') replay_instance = ReplayProcess() replay_instance.dataset = self.clients.dataset_management.read_dataset(dataset_id) replay_instance.dataset_id = dataset_id replay_instance.start_time = query.get('start_time', None) replay_instance.end_time = query.get('end_time', None) replay_instance.parameters = query.get('parameters',None) replay_instance.container = self.container retrieve_data = replay_instance.execute_retrieve() if module and cls: return self._transform_data(retrieve_data, module, cls, kwargs or {}) return retrieve_data
def retrieve_oob(cls, dataset_id='', query=None, delivery_format=''): query = query or {} coverage = None try: coverage = cls._get_coverage(dataset_id) if coverage is None: raise BadRequest('no such coverage') if coverage.num_timesteps == 0: log.info('Reading from an empty coverage') rdt = RecordDictionaryTool( param_dictionary=coverage.parameter_dictionary) else: rdt = ReplayProcess._coverage_to_granule( coverage=coverage, start_time=query.get('start_time', None), end_time=query.get('end_time', None), stride_time=query.get('stride_time', None), parameters=query.get('parameters', None), stream_def_id=delivery_format, tdoa=query.get('tdoa', None)) except: cls._eject_cache(dataset_id) log.exception('Problems reading from the coverage') raise BadRequest('Problems reading from the coverage') return rdt.to_granule()
def retrieve_oob(cls, dataset_id='', query=None, delivery_format=''): query = query or {} coverage = None try: coverage = cls._get_coverage(dataset_id) if coverage is None: raise BadRequest('no such coverage') if isinstance(coverage, SimplexCoverage) and coverage.is_empty(): log.info('Reading from an empty coverage') rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary) else: args = { 'start_time' : query.get('start_time', None), 'end_time' : query.get('end_time', None), 'stride_time' : query.get('stride_time', None), 'parameters' : query.get('parameters', None), 'stream_def_id' : delivery_format, 'tdoa' : query.get('tdoa', None), 'sort_parameter' : query.get('sort_parameter', None) } rdt = ReplayProcess._cov2granule(coverage=coverage, **args) except Exception as e: cls._eject_cache(dataset_id) data_products, _ = Container.instance.resource_registry.find_subjects(object=dataset_id, predicate=PRED.hasDataset, subject_type=RT.DataProduct) for data_product in data_products: log.error("Data Product %s (%s) had issues reading from the coverage model\nretrieve_oob(dataset_id='%s', query=%s, delivery_format=%s)", data_product.name, data_product._id, dataset_id, query, delivery_format) log.error("Problems reading from the coverage", exc_info=True) raise BadRequest('Problems reading from the coverage') return rdt.to_granule()
def retrieve_last_data_points(self, dataset_id='', number_of_points=100, delivery_format=''): rdt = ReplayProcess.get_last_values(dataset_id, number_of_points, delivery_format) return rdt.to_granule()
def setUp(self): self.replay = ReplayProcess() self.replay.dataset_id = 'dataset' self.replay.dataset = DotDict() self.replay.dataset.datastore_name='datasets' self.replay.dataset.primary_view_key = 'stream_id' self.replay.deliver_format = {} self.replay.start_time = None self.replay.end_time = None self.replay.output = DotDict() self.replay.output.publish = Mock()
def retrieve_oob(cls, dataset_id='', query=None, delivery_format=''): query = query or {} coverage = None try: coverage = cls._get_coverage(dataset_id) if coverage is None: raise BadRequest('no such coverage') if coverage.num_timesteps == 0: log.info('Reading from an empty coverage') rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary) else: rdt = ReplayProcess._coverage_to_granule(coverage=coverage, start_time=query.get('start_time', None), end_time=query.get('end_time',None), stride_time=query.get('stride_time',None), parameters=query.get('parameters',None), stream_def_id=delivery_format, tdoa=query.get('tdoa',None)) except: cls._eject_cache(dataset_id) log.exception('Problems reading from the coverage') raise BadRequest('Problems reading from the coverage') return rdt.to_granule()
def retrieve_oob(cls, dataset_id='', query=None, delivery_format=''): query = query or {} coverage = None try: coverage = cls._get_coverage(dataset_id) if coverage is None: raise BadRequest('no such coverage') if coverage.num_timesteps == 0: log.info('Reading from an empty coverage') rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary) else: rdt = ReplayProcess._cov2granule(coverage=coverage, start_time=query.get('start_time', None), end_time=query.get('end_time',None), stride_time=query.get('stride_time',None), parameters=query.get('parameters',None), stream_def_id=delivery_format, tdoa=query.get('tdoa',None)) except: cls._eject_cache(dataset_id) data_products, _ = Container.instance.resource_registry.find_subjects(object=dataset_id, predicate=PRED.hasDataset, subject_type=RT.DataProduct) for data_product in data_products: log.exception("Data Product %s (%s) had issues reading from the coverage model\nretrieve_oob(dataset_id='%s', query=%s, delivery_format=%s)", data_product.name, data_product._id, dataset_id, query, delivery_format) raise BadRequest('Problems reading from the coverage') return rdt.to_granule()
def retrieve_oob(cls, dataset_id='', query=None, delivery_format=''): query = query or {} coverage = None try: coverage = cls._get_coverage(dataset_id) if coverage is None: raise BadRequest('no such coverage') if isinstance(coverage, SimplexCoverage) and coverage.is_empty(): log.info('Reading from an empty coverage') rdt = RecordDictionaryTool( param_dictionary=coverage.parameter_dictionary) else: args = { 'start_time': query.get('start_time', None), 'end_time': query.get('end_time', None), 'stride_time': query.get('stride_time', None), 'parameters': query.get('parameters', None), 'stream_def_id': delivery_format, 'tdoa': query.get('tdoa', None), 'sort_parameter': query.get('sort_parameter', None) } rdt = ReplayProcess._cov2granule(coverage=coverage, **args) except Exception as e: cls._eject_cache(dataset_id) data_products, _ = Container.instance.resource_registry.find_subjects( object=dataset_id, predicate=PRED.hasDataset, subject_type=RT.DataProduct) for data_product in data_products: log.error( "Data Product %s (%s) had issues reading from the coverage model\nretrieve_oob(dataset_id='%s', query=%s, delivery_format=%s)", data_product.name, data_product._id, dataset_id, query, delivery_format) log.error("Problems reading from the coverage", exc_info=True) raise BadRequest('Problems reading from the coverage') return rdt.to_granule()
def retrieve_last_data_points(self, dataset_id='', number_of_points=100): return ReplayProcess.get_last_values(dataset_id, number_of_points)
def retrieve_last_data_points(self, dataset_id='', number_of_points=100, delivery_format=''): return ReplayProcess.get_last_values(dataset_id, number_of_points, delivery_format)
def retrieve_last_granule(self, dataset_id=""): return ReplayProcess.get_last_granule(self.container, dataset_id)
def retrieve(self, dataset_id="", query=None, delivery_format=None, module="", cls="", kwargs=None): """ Retrieves a dataset. @param dataset_id Dataset identifier @param query Query parameters (start_time, end_time, stride_time, parameters) @param delivery_format The stream definition identifier for the outgoing granule (stream_defintinition_id) @param module Module to chain a transform into @param cls Class of the transform @param kwargs Keyword Arguments to pass into the transform. """ if query is None: query = {} if delivery_format is None: delivery_format = {} validate_is_instance(query, dict, "Query was improperly formatted.") validate_true(dataset_id, "No dataset provided") replay_instance = ReplayProcess() replay_instance.dataset = self.clients.dataset_management.read_dataset(dataset_id) replay_instance.dataset_id = dataset_id replay_instance.start_time = query.get("start_time", None) replay_instance.end_time = query.get("end_time", None) replay_instance.stride_time = query.get("stride_time", None) replay_instance.parameters = query.get("parameters", None) replay_instance.tdoa = query.get("tdoa", None) replay_instance.stream_def_id = delivery_format replay_instance.container = self.container if replay_instance.tdoa is not None: validate_is_instance(replay_instance.tdoa, slice) retrieve_data = replay_instance.execute_retrieve() if module and cls: return self._transform_data(retrieve_data, module, cls, kwargs or {}) return retrieve_data
def retrieve_last_data_point(self, dataset_id=''): return ReplayProcess.get_last_values(dataset_id)
class ReplayProcessUnitTest(PyonTestCase): def setUp(self): self.replay = ReplayProcess() self.replay.dataset_id = 'dataset' self.replay.dataset = DotDict() self.replay.dataset.datastore_name='datasets' self.replay.dataset.primary_view_key = 'stream_id' self.replay.deliver_format = {} self.replay.start_time = None self.replay.end_time = None self.replay.output = DotDict() self.replay.output.publish = Mock() @patch('ion.processes.data.replay.replay_process.DatasetManagementService') @patch('ion.processes.data.replay.replay_process.CoverageCraft') def test_execute_retrieve(self,mock_bb, mock_dsm): mock_dsm._get_coverage = Mock() mock_bb.sync_rdt_with_coverage = Mock() mock_bb().to_granule = Mock() mock_bb().to_granule.return_value = {'test':1} self.replay.start_time = 0 self.replay.end_time = 0 self.replay.parameters = None retval = self.replay.execute_retrieve() self.assertEquals(retval,{'test':1}) @patch('ion.processes.data.replay.replay_process.gevent') def test_execute_replay(self,mock_gevent): mock_gevent.spawn = Mock() self.replay.publishing = DotDict() self.replay.publishing.is_set = Mock() self.replay.publishing.return_value = False retval = self.replay.execute_replay self.assertTrue(retval) def test_replay(self): self.replay.publishing = DotDict() self.replay.publishing.set = Mock() self.replay.publishing['clear'] = Mock() self.replay.execute_retrieve = Mock() self.replay.execute_retrieve.return_value = {'test':True} self.replay.output = DotDict() self.replay.output.publish = Mock() retval = self.replay.replay() self.assertTrue(retval) @patch('ion.processes.data.replay.replay_process.DatasetManagementService') @patch('ion.processes.data.replay.replay_process.DatasetManagementServiceClient') @patch('ion.processes.data.replay.replay_process.CoverageCraft') def test_get_last_granule(self, mock_bb, dsm_cli, dsm): mock_bb().sync_rdt_with_coverage = Mock() mock_bb().to_granule.return_value = {'test':True} dsm_cli().read_dataset = Mock() dataset = DotDict() dataset.datastore_name = 'test' dataset.view_name = 'bogus/view' dsm._get_coverage = Mock() dsm._get_coverage.return_value = {} datastore = DotDict() datastore.query_view = Mock() datastore.query_view.return_value = [{'doc':{'ts_create':0}}] container = DotDict() container.datastore_manager.get_datastore = Mock() container.datastore_manager.get_datastore.return_value = datastore retval = self.replay.get_last_granule(container,'dataset_id') self.assertEquals(retval,{'test':True}) @patch('ion.processes.data.replay.replay_process.DatasetManagementService') @patch('ion.processes.data.replay.replay_process.CoverageCraft') def test_get_last_values(self, craft, dsm): dsm._get_coverage = Mock() craft().sync_rdt_with_coverage = Mock() craft().to_granule = Mock() craft().to_granule.return_value = {'test':True} retval = self.replay.get_last_values('dataset_id') self.assertEquals(retval,{'test':True})