def retrieve(self, dataset_id='', query=None, delivery_format=None, module='', cls='', kwargs=None):
if query is None:
query = {}
if delivery_format is None:
delivery_format = {}
validate_is_instance(query,dict,'Query was improperly formatted.')
validate_true(dataset_id, 'No dataset provided')
replay_instance = ReplayProcess()
replay_instance.dataset = self.clients.dataset_management.read_dataset(dataset_id)
replay_instance.dataset_id = dataset_id
replay_instance.start_time = query.get('start_time', None)
replay_instance.end_time = query.get('end_time', None)
replay_instance.parameters = query.get('parameters',None)
replay_instance.container = self.container
retrieve_data = replay_instance.execute_retrieve()
if module and cls:
return self._transform_data(retrieve_data, module, cls, kwargs or {})
return retrieve_data
def retrieve_oob(cls, dataset_id='', query=None, delivery_format=''):
query = query or {}
coverage = None
try:
coverage = cls._get_coverage(dataset_id)
if coverage is None:
raise BadRequest('no such coverage')
if coverage.num_timesteps == 0:
log.info('Reading from an empty coverage')
rdt = RecordDictionaryTool(
param_dictionary=coverage.parameter_dictionary)
else:
rdt = ReplayProcess._coverage_to_granule(
coverage=coverage,
start_time=query.get('start_time', None),
end_time=query.get('end_time', None),
stride_time=query.get('stride_time', None),
parameters=query.get('parameters', None),
stream_def_id=delivery_format,
tdoa=query.get('tdoa', None))
except:
cls._eject_cache(dataset_id)
log.exception('Problems reading from the coverage')
raise BadRequest('Problems reading from the coverage')
return rdt.to_granule()
def retrieve_oob(cls, dataset_id='', query=None, delivery_format=''):
query = query or {}
coverage = None
try:
coverage = cls._get_coverage(dataset_id)
if coverage is None:
raise BadRequest('no such coverage')
if isinstance(coverage, SimplexCoverage) and coverage.is_empty():
log.info('Reading from an empty coverage')
rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary)
else:
args = {
'start_time' : query.get('start_time', None),
'end_time' : query.get('end_time', None),
'stride_time' : query.get('stride_time', None),
'parameters' : query.get('parameters', None),
'stream_def_id' : delivery_format,
'tdoa' : query.get('tdoa', None),
'sort_parameter' : query.get('sort_parameter', None)
}
rdt = ReplayProcess._cov2granule(coverage=coverage, **args)
except Exception as e:
cls._eject_cache(dataset_id)
data_products, _ = Container.instance.resource_registry.find_subjects(object=dataset_id, predicate=PRED.hasDataset, subject_type=RT.DataProduct)
for data_product in data_products:
log.error("Data Product %s (%s) had issues reading from the coverage model\nretrieve_oob(dataset_id='%s', query=%s, delivery_format=%s)", data_product.name, data_product._id, dataset_id, query, delivery_format)
log.error("Problems reading from the coverage", exc_info=True)
raise BadRequest('Problems reading from the coverage')
return rdt.to_granule()
def retrieve_last_data_points(self,
dataset_id='',
number_of_points=100,
delivery_format=''):
rdt = ReplayProcess.get_last_values(dataset_id, number_of_points,
delivery_format)
return rdt.to_granule()
def setUp(self):
self.replay = ReplayProcess()
self.replay.dataset_id = 'dataset'
self.replay.dataset = DotDict()
self.replay.dataset.datastore_name='datasets'
self.replay.dataset.primary_view_key = 'stream_id'
self.replay.deliver_format = {}
self.replay.start_time = None
self.replay.end_time = None
self.replay.output = DotDict()
self.replay.output.publish = Mock()
def retrieve_oob(cls, dataset_id='', query=None, delivery_format=''):
query = query or {}
coverage = None
try:
coverage = cls._get_coverage(dataset_id)
if coverage is None:
raise BadRequest('no such coverage')
if coverage.num_timesteps == 0:
log.info('Reading from an empty coverage')
rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary)
else:
rdt = ReplayProcess._coverage_to_granule(coverage=coverage, start_time=query.get('start_time', None), end_time=query.get('end_time',None), stride_time=query.get('stride_time',None), parameters=query.get('parameters',None), stream_def_id=delivery_format, tdoa=query.get('tdoa',None))
except:
cls._eject_cache(dataset_id)
log.exception('Problems reading from the coverage')
raise BadRequest('Problems reading from the coverage')
return rdt.to_granule()
def retrieve_oob(cls, dataset_id='', query=None, delivery_format=''):
query = query or {}
coverage = None
try:
coverage = cls._get_coverage(dataset_id)
if coverage is None:
raise BadRequest('no such coverage')
if coverage.num_timesteps == 0:
log.info('Reading from an empty coverage')
rdt = RecordDictionaryTool(param_dictionary=coverage.parameter_dictionary)
else:
rdt = ReplayProcess._cov2granule(coverage=coverage, start_time=query.get('start_time', None), end_time=query.get('end_time',None), stride_time=query.get('stride_time',None), parameters=query.get('parameters',None), stream_def_id=delivery_format, tdoa=query.get('tdoa',None))
except:
cls._eject_cache(dataset_id)
data_products, _ = Container.instance.resource_registry.find_subjects(object=dataset_id, predicate=PRED.hasDataset, subject_type=RT.DataProduct)
for data_product in data_products:
log.exception("Data Product %s (%s) had issues reading from the coverage model\nretrieve_oob(dataset_id='%s', query=%s, delivery_format=%s)", data_product.name, data_product._id, dataset_id, query, delivery_format)
raise BadRequest('Problems reading from the coverage')
return rdt.to_granule()
def retrieve_oob(cls, dataset_id='', query=None, delivery_format=''):
query = query or {}
coverage = None
try:
coverage = cls._get_coverage(dataset_id)
if coverage is None:
raise BadRequest('no such coverage')
if isinstance(coverage, SimplexCoverage) and coverage.is_empty():
log.info('Reading from an empty coverage')
rdt = RecordDictionaryTool(
param_dictionary=coverage.parameter_dictionary)
else:
args = {
'start_time': query.get('start_time', None),
'end_time': query.get('end_time', None),
'stride_time': query.get('stride_time', None),
'parameters': query.get('parameters', None),
'stream_def_id': delivery_format,
'tdoa': query.get('tdoa', None),
'sort_parameter': query.get('sort_parameter', None)
}
rdt = ReplayProcess._cov2granule(coverage=coverage, **args)
except Exception as e:
cls._eject_cache(dataset_id)
data_products, _ = Container.instance.resource_registry.find_subjects(
object=dataset_id,
predicate=PRED.hasDataset,
subject_type=RT.DataProduct)
for data_product in data_products:
log.error(
"Data Product %s (%s) had issues reading from the coverage model\nretrieve_oob(dataset_id='%s', query=%s, delivery_format=%s)",
data_product.name, data_product._id, dataset_id, query,
delivery_format)
log.error("Problems reading from the coverage", exc_info=True)
raise BadRequest('Problems reading from the coverage')
return rdt.to_granule()
def retrieve_last_data_points(self, dataset_id='', number_of_points=100):
return ReplayProcess.get_last_values(dataset_id, number_of_points)
def retrieve_last_data_points(self, dataset_id='', number_of_points=100):
return ReplayProcess.get_last_values(dataset_id, number_of_points)
def retrieve_last_data_points(self, dataset_id='', number_of_points=100, delivery_format=''):
return ReplayProcess.get_last_values(dataset_id, number_of_points, delivery_format)
def retrieve_last_granule(self, dataset_id=""):
return ReplayProcess.get_last_granule(self.container, dataset_id)
def retrieve(self, dataset_id="", query=None, delivery_format=None, module="", cls="", kwargs=None):
"""
Retrieves a dataset.
@param dataset_id Dataset identifier
@param query Query parameters (start_time, end_time, stride_time, parameters)
@param delivery_format The stream definition identifier for the outgoing granule (stream_defintinition_id)
@param module Module to chain a transform into
@param cls Class of the transform
@param kwargs Keyword Arguments to pass into the transform.
"""
if query is None:
query = {}
if delivery_format is None:
delivery_format = {}
validate_is_instance(query, dict, "Query was improperly formatted.")
validate_true(dataset_id, "No dataset provided")
replay_instance = ReplayProcess()
replay_instance.dataset = self.clients.dataset_management.read_dataset(dataset_id)
replay_instance.dataset_id = dataset_id
replay_instance.start_time = query.get("start_time", None)
replay_instance.end_time = query.get("end_time", None)
replay_instance.stride_time = query.get("stride_time", None)
replay_instance.parameters = query.get("parameters", None)
replay_instance.tdoa = query.get("tdoa", None)
replay_instance.stream_def_id = delivery_format
replay_instance.container = self.container
if replay_instance.tdoa is not None:
validate_is_instance(replay_instance.tdoa, slice)
retrieve_data = replay_instance.execute_retrieve()
if module and cls:
return self._transform_data(retrieve_data, module, cls, kwargs or {})
return retrieve_data
def retrieve_last_data_point(self, dataset_id=''):
return ReplayProcess.get_last_values(dataset_id)
class ReplayProcessUnitTest(PyonTestCase):
def setUp(self):
self.replay = ReplayProcess()
self.replay.dataset_id = 'dataset'
self.replay.dataset = DotDict()
self.replay.dataset.datastore_name='datasets'
self.replay.dataset.primary_view_key = 'stream_id'
self.replay.deliver_format = {}
self.replay.start_time = None
self.replay.end_time = None
self.replay.output = DotDict()
self.replay.output.publish = Mock()
@patch('ion.processes.data.replay.replay_process.DatasetManagementService')
@patch('ion.processes.data.replay.replay_process.CoverageCraft')
def test_execute_retrieve(self,mock_bb, mock_dsm):
mock_dsm._get_coverage = Mock()
mock_bb.sync_rdt_with_coverage = Mock()
mock_bb().to_granule = Mock()
mock_bb().to_granule.return_value = {'test':1}
self.replay.start_time = 0
self.replay.end_time = 0
self.replay.parameters = None
retval = self.replay.execute_retrieve()
self.assertEquals(retval,{'test':1})
@patch('ion.processes.data.replay.replay_process.gevent')
def test_execute_replay(self,mock_gevent):
mock_gevent.spawn = Mock()
self.replay.publishing = DotDict()
self.replay.publishing.is_set = Mock()
self.replay.publishing.return_value = False
retval = self.replay.execute_replay
self.assertTrue(retval)
def test_replay(self):
self.replay.publishing = DotDict()
self.replay.publishing.set = Mock()
self.replay.publishing['clear'] = Mock()
self.replay.execute_retrieve = Mock()
self.replay.execute_retrieve.return_value = {'test':True}
self.replay.output = DotDict()
self.replay.output.publish = Mock()
retval = self.replay.replay()
self.assertTrue(retval)
@patch('ion.processes.data.replay.replay_process.DatasetManagementService')
@patch('ion.processes.data.replay.replay_process.DatasetManagementServiceClient')
@patch('ion.processes.data.replay.replay_process.CoverageCraft')
def test_get_last_granule(self, mock_bb, dsm_cli, dsm):
mock_bb().sync_rdt_with_coverage = Mock()
mock_bb().to_granule.return_value = {'test':True}
dsm_cli().read_dataset = Mock()
dataset = DotDict()
dataset.datastore_name = 'test'
dataset.view_name = 'bogus/view'
dsm._get_coverage = Mock()
dsm._get_coverage.return_value = {}
datastore = DotDict()
datastore.query_view = Mock()
datastore.query_view.return_value = [{'doc':{'ts_create':0}}]
container = DotDict()
container.datastore_manager.get_datastore = Mock()
container.datastore_manager.get_datastore.return_value = datastore
retval = self.replay.get_last_granule(container,'dataset_id')
self.assertEquals(retval,{'test':True})
@patch('ion.processes.data.replay.replay_process.DatasetManagementService')
@patch('ion.processes.data.replay.replay_process.CoverageCraft')
def test_get_last_values(self, craft, dsm):
dsm._get_coverage = Mock()
craft().sync_rdt_with_coverage = Mock()
craft().to_granule = Mock()
craft().to_granule.return_value = {'test':True}
retval = self.replay.get_last_values('dataset_id')
self.assertEquals(retval,{'test':True})
