def testPush(self):
"""This tests the push ingest by creating a test dir, populating it, then forcing the ingester to run
"""
staging = tempfile.mkdtemp()
self.todelete.append(staging)
# Create a temp file to ingest
f_name = str(int(time.time()))
f = open(os.path.join(staging, f_name), "a")
f.close()
# Check there is only 1 file here
self.assertEquals(1, len(os.listdir(staging)))
dataset = Dataset()
dataset.id = 1
dataset.data_source = PushDataSource(path=staging,
pattern='^(?P<timestamp>[0-9]+)$')
self.ingester.enqueue_ingress(dataset)
self.ingester.process_ingress_queue(True)
self.assertEquals(1, self.ingester._archive_queue.qsize())
# Check there are now no files
self.assertEquals(0, len(os.listdir(staging)))
def setup_datasets(self):
ds = Dataset()
ds.data_source = _DataSource()
ds.id = 1
ds.data_source.__xmlrpc_class__ = "mock_data_source"
ds.data_source.sampling = PeriodicSampling(10000)
self.datasets.append(ds)
def testBasicIngest(self):
"""This test performs a simple data ingest"""
# Capture the ingests
data_entries = DataEntryListener()
self.ingester.service.register_observation_listener(data_entries)
dataset = Dataset()
dataset.id = 1
datasource = _DataSource()
datasource.__xmlrpc_class__ = "csv1"
dataset.data_source = datasource
self.ingester.enqueue_ingress(dataset)
self.assertEquals(1, self.ingester._ingress_queue.qsize())
self.ingester.process_ingress_queue(True)
self.assertEquals(1, self.ingester._archive_queue.qsize())
self.ingester.process_archive_queue(True)
self.assertEquals(1, data_entries.count())