def test_coverage_ingest(self):
stream_id = self.pubsub_management.create_stream()
dataset_id = self.create_dataset()
# I freaking hate this bug
self.get_datastore(dataset_id)
ingestion_config_id = self.get_ingestion_config()
self.ingestion_management.persist_data_stream(
stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id
)
black_box = CoverageCraft()
black_box.rdt["time"] = np.arange(20)
black_box.rdt["temp"] = np.random.random(20) * 10
black_box.sync_with_granule()
granule = black_box.to_granule()
publisher = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point_name, stream_id)
publisher.publish(granule)
self.wait_until_we_have_enough_granules(dataset_id, 1)
coverage = DatasetManagementService._get_coverage(dataset_id)
black_box = CoverageCraft(coverage)
black_box.sync_rdt_with_coverage()
comp = black_box.rdt["time"] == np.arange(20)
self.assertTrue(comp.all())
black_box = CoverageCraft()
black_box.rdt["time"] = np.arange(20) + 20
black_box.rdt["temp"] = np.random.random(20) * 10
black_box.sync_with_granule()
granule = black_box.to_granule()
publisher.publish(granule)
self.wait_until_we_have_enough_granules(dataset_id, 2)
coverage = DatasetManagementService._get_coverage(dataset_id)
black_box = CoverageCraft(coverage)
black_box.sync_rdt_with_coverage()
comp = black_box.rdt["time"] == np.arange(40)
self.assertTrue(comp.all())
granule = self.data_retriever.retrieve(dataset_id)
black_box = CoverageCraft()
black_box.sync_rdt_with_granule(granule)
comp = black_box.rdt["time"] == np.arange(40)
self.assertTrue(comp.all())
def get_last_values(cls, dataset_id):
coverage = DatasetManagementService._get_coverage(dataset_id)
black_box = CoverageCraft(coverage)
black_box.sync_rdt_with_coverage(tdoa=slice(-1,None))
granule = black_box.to_granule()
return granule
def get_last_granule(cls, container, dataset_id):
dsm_cli = DatasetManagementServiceClient()
dataset = dsm_cli.read_dataset(dataset_id)
cc = container
datastore_name = dataset.datastore_name
view_name = dataset.view_name
datastore = cc.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
opts = dict(
start_key = [dataset_id, {}],
end_key = [dataset_id, 0],
descending = True,
limit = 1,
include_docs = True
)
results = datastore.query_view(view_name,opts=opts)
if not results:
raise NotFound('A granule could not be located.')
if results[0] is None:
raise NotFound('A granule could not be located.')
doc = results[0].get('doc')
if doc is None:
return None
ts = float(doc.get('ts_create',0))
coverage = DatasetManagementService._get_coverage(dataset_id)
black_box = CoverageCraft(coverage)
black_box.sync_rdt_with_coverage(start_time=ts,end_time=None)
granule = black_box.to_granule()
return granule
def publish_hifi(self, stream_id, offset=0):
pub = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point_name, stream_id)
black_box = CoverageCraft()
black_box.rdt["time"] = np.arange(10) + (offset * 10)
black_box.rdt["temp"] = (np.arange(10) + (offset * 10)) * 2
granule = black_box.to_granule()
pub.publish(granule)
def execute_retrieve(self):
'''
execute_retrieve Executes a retrieval and returns the result
as a value in lieu of publishing it on a stream
'''
coverage = DatasetManagementService._get_coverage(self.dataset_id)
crafter = CoverageCraft(coverage)
#@todo: add bounds checking to ensure the dataset being retrieved is not too large
crafter.sync_rdt_with_coverage(start_time=self.start_time,end_time=self.end_time,parameters=self.parameters)
granule = crafter.to_granule()
return granule
def _trigger_func(self, stream_id):
t_i = 0
while not self.finished.is_set():
length = 10
black_box = CoverageCraft()
black_box.rdt['time'] = numpy.arange(10) + t_i*10
black_box.rdt['temp'] = numpy.random.random(10) * 10
black_box.rdt['lat'] = numpy.array([0] * 10)
black_box.rdt['lon'] = numpy.array([0] * 10)
black_box.rdt['depth'] = numpy.array([0] * 10)
black_box.rdt['conductivity'] = numpy.random.random(10) * 10
black_box.rdt['data'] = numpy.random.randint(0,255,10) # Simulates random bytes
black_box.sync_with_granule()
granule = black_box.to_granule()
self.publish(granule)
gevent.sleep(self.interval)
t_i += 1
