def testPush(self): """This tests the push ingest by creating a test dir, populating it, then forcing the ingester to run """ staging = tempfile.mkdtemp() self.todelete.append(staging) # Create a temp file to ingest f_name = str(int(time.time())) f = open(os.path.join(staging, f_name), "a") f.close() # Check there is only 1 file here self.assertEquals(1, len(os.listdir(staging))) dataset = Dataset() dataset.id = 1 dataset.data_source = PushDataSource(path=staging, pattern='^(?P<timestamp>[0-9]+)$') self.ingester.enqueue_ingress(dataset) self.ingester.process_ingress_queue(True) self.assertEquals(1, self.ingester._archive_queue.qsize()) # Check there are now no files self.assertEquals(0, len(os.listdir(staging)))
def setup_datasets(self): ds = Dataset() ds.data_source = _DataSource() ds.id = 1 ds.data_source.__xmlrpc_class__ = "mock_data_source" ds.data_source.sampling = PeriodicSampling(10000) self.datasets.append(ds)
def testBasicIngest(self): """This test performs a simple data ingest""" # Capture the ingests data_entries = DataEntryListener() self.ingester.service.register_observation_listener(data_entries) dataset = Dataset() dataset.id = 1 datasource = _DataSource() datasource.__xmlrpc_class__ = "csv1" dataset.data_source = datasource self.ingester.enqueue_ingress(dataset) self.assertEquals(1, self.ingester._ingress_queue.qsize()) self.ingester.process_ingress_queue(True) self.assertEquals(1, self.ingester._archive_queue.qsize()) self.ingester.process_archive_queue(True) self.assertEquals(1, data_entries.count())