def test_job_queue(self):
job = Job.create(self.conn, self.dataset)
stage = job.get_stage("ingest")
status = stage.get_status()
assert status["pending"] == 0
assert status["finished"] == 0
assert job.is_done()
stage.queue({"test": "foo"}, {})
status = job.get_status()
assert status["pending"] == 1
assert status["finished"] == 0
assert status["running"] == 0
assert not job.is_done()
task = Stage.get_task(self.conn, "ingest", timeout=None)
assert task.job.dataset.name == job.dataset.name
assert task.payload["test"] == "foo"
status = job.get_status()
assert status["pending"] == 0
assert status["running"] == 1
assert status["finished"] == 0
assert not job.is_done()
task.done()
status = job.get_status()
assert status["pending"] == 0
assert status["running"] == 0
assert status["finished"] == 1
assert job.is_done()
def test_job_queue(self):
job = Job.create(self.conn, self.dataset)
stage = job.get_stage(Stage.INGEST)
status = stage.get_status()
assert status['pending'] == 0
assert status['finished'] == 0
assert job.is_done()
stage.queue({'test': 'foo'}, {})
status = job.get_status()
assert status['pending'] == 1
assert status['finished'] == 0
assert status['running'] == 0
assert not job.is_done()
task = Stage.get_task(self.conn, Stage.INGEST,
timeout=None)
assert task.job.dataset.name == job.dataset.name
assert task.payload['test'] == 'foo'
status = job.get_status()
assert status['pending'] == 0
assert status['running'] == 1
assert status['finished'] == 0
assert not job.is_done()
task.done()
status = job.get_status()
assert status['pending'] == 0
assert status['running'] == 0
assert status['finished'] == 1
assert job.is_done()
def sync(self):
"""Process only the tasks already in the job queue, but do not
go into an infinte loop waiting for new ones."""
self.init_internal()
while True:
stages = self.get_stages()
task = Stage.get_task(self.conn, stages, timeout=None)
if task is None:
return
self.handle_safe(task)
def process(self, interval=5):
while True:
if self._shutdown:
return
self.periodic()
stages = self.get_stages()
task = Stage.get_task(self.conn, stages, timeout=interval)
if task is None:
continue
self.handle_safe(task)
def process(self, blocking=True, interval=INTERVAL):
while True:
if self.exit_code > 0:
return self.exit_code
self.periodic()
stages = self.get_stages()
task = Stage.get_task(self.conn, stages, timeout=interval)
if task is None:
if not blocking:
return self.exit_code
continue
self.handle_safe(task)
def queue_task(dataset, stage, job_id=None, context=None, **payload):
stage = get_stage(dataset, stage, job_id=job_id)
stage.queue(payload or {}, context or {})
if settings.TESTING:
from aleph.worker import get_worker
worker = get_worker()
while True:
stages = worker.get_stages()
task = Stage.get_task(worker.conn, stages, timeout=None)
if task is None:
break
worker.dispatch_task(task)
def setUp(self):
# Force tests to use fake configuration
service_settings.REDIS_URL = None
service_settings.ARCHIVE_TYPE = 'file'
service_settings.ARCHIVE_PATH = mkdtemp()
balkhash_settings.BACKEND = 'LEVELDB'
balkhash_settings.LEVELDB_PATH = mkdtemp()
conn = get_fakeredis()
job = Job.create(conn, 'test')
stage = Stage(job, Stage.INGEST)
self.manager = Manager(stage, {})
self.manager.entities = []
self.manager.emit_entity = types.MethodType(emit_entity, self.manager)
self.manager.queue_entity = types.MethodType(queue_entity,
self.manager) # noqa
self.archive = init_archive()
self.manager._archive = self.archive
def setUp(self):
# Force tests to use fake configuration
ingestors_settings.TESTING = True
service_settings.REDIS_URL = None
service_settings.ARCHIVE_TYPE = 'file'
service_settings.ARCHIVE_PATH = mkdtemp()
ftmstore_settings.DATABASE_URI = 'sqlite://'
conn = get_fakeredis()
job = Job.create(conn, 'test')
stage = Stage(job, OP_INGEST)
dataset = get_dataset(job.dataset.name, OP_INGEST)
self.manager = Manager(dataset, stage, {})
self.manager.entities = []
self.manager.emit_entity = types.MethodType(emit_entity, self.manager)
self.manager.queue_entity = types.MethodType(queue_entity,
self.manager) # noqa
self.archive = init_archive()
self.manager._archive = self.archive
def _get_task(self):
"""return the newly created reporting task"""
return Stage.get_task(self.conn, [OP_REPORT])