def test_coverage_ingest(self): stream_id = self.pubsub_management.create_stream() dataset_id = self.create_dataset() # I freaking hate this bug self.get_datastore(dataset_id) ingestion_config_id = self.get_ingestion_config() self.ingestion_management.persist_data_stream( stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id ) black_box = CoverageCraft() black_box.rdt["time"] = np.arange(20) black_box.rdt["temp"] = np.random.random(20) * 10 black_box.sync_with_granule() granule = black_box.to_granule() publisher = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point_name, stream_id) publisher.publish(granule) self.wait_until_we_have_enough_granules(dataset_id, 1) coverage = DatasetManagementService._get_coverage(dataset_id) black_box = CoverageCraft(coverage) black_box.sync_rdt_with_coverage() comp = black_box.rdt["time"] == np.arange(20) self.assertTrue(comp.all()) black_box = CoverageCraft() black_box.rdt["time"] = np.arange(20) + 20 black_box.rdt["temp"] = np.random.random(20) * 10 black_box.sync_with_granule() granule = black_box.to_granule() publisher.publish(granule) self.wait_until_we_have_enough_granules(dataset_id, 2) coverage = DatasetManagementService._get_coverage(dataset_id) black_box = CoverageCraft(coverage) black_box.sync_rdt_with_coverage() comp = black_box.rdt["time"] == np.arange(40) self.assertTrue(comp.all()) granule = self.data_retriever.retrieve(dataset_id) black_box = CoverageCraft() black_box.sync_rdt_with_granule(granule) comp = black_box.rdt["time"] == np.arange(40) self.assertTrue(comp.all())
def get_last_values(cls, dataset_id): coverage = DatasetManagementService._get_coverage(dataset_id) black_box = CoverageCraft(coverage) black_box.sync_rdt_with_coverage(tdoa=slice(-1,None)) granule = black_box.to_granule() return granule
def get_last_granule(cls, container, dataset_id): dsm_cli = DatasetManagementServiceClient() dataset = dsm_cli.read_dataset(dataset_id) cc = container datastore_name = dataset.datastore_name view_name = dataset.view_name datastore = cc.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA) opts = dict( start_key = [dataset_id, {}], end_key = [dataset_id, 0], descending = True, limit = 1, include_docs = True ) results = datastore.query_view(view_name,opts=opts) if not results: raise NotFound('A granule could not be located.') if results[0] is None: raise NotFound('A granule could not be located.') doc = results[0].get('doc') if doc is None: return None ts = float(doc.get('ts_create',0)) coverage = DatasetManagementService._get_coverage(dataset_id) black_box = CoverageCraft(coverage) black_box.sync_rdt_with_coverage(start_time=ts,end_time=None) granule = black_box.to_granule() return granule
def publish_hifi(self, stream_id, offset=0): pub = SimpleStreamPublisher.new_publisher(self.container, self.exchange_point_name, stream_id) black_box = CoverageCraft() black_box.rdt["time"] = np.arange(10) + (offset * 10) black_box.rdt["temp"] = (np.arange(10) + (offset * 10)) * 2 granule = black_box.to_granule() pub.publish(granule)
def execute_retrieve(self): ''' execute_retrieve Executes a retrieval and returns the result as a value in lieu of publishing it on a stream ''' coverage = DatasetManagementService._get_coverage(self.dataset_id) crafter = CoverageCraft(coverage) #@todo: add bounds checking to ensure the dataset being retrieved is not too large crafter.sync_rdt_with_coverage(start_time=self.start_time,end_time=self.end_time,parameters=self.parameters) granule = crafter.to_granule() return granule
def _trigger_func(self, stream_id): t_i = 0 while not self.finished.is_set(): length = 10 black_box = CoverageCraft() black_box.rdt['time'] = numpy.arange(10) + t_i*10 black_box.rdt['temp'] = numpy.random.random(10) * 10 black_box.rdt['lat'] = numpy.array([0] * 10) black_box.rdt['lon'] = numpy.array([0] * 10) black_box.rdt['depth'] = numpy.array([0] * 10) black_box.rdt['conductivity'] = numpy.random.random(10) * 10 black_box.rdt['data'] = numpy.random.randint(0,255,10) # Simulates random bytes black_box.sync_with_granule() granule = black_box.to_granule() self.publish(granule) gevent.sleep(self.interval) t_i += 1