def test_job_execution(self):
"""Job is removed from StartedJobRegistry after execution."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
self.assertTrue(job.is_queued)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
self.assertTrue(job.is_started)
worker.perform_job(job, queue)
self.assertNotIn(job.id, registry.get_job_ids())
self.assertTrue(job.is_finished)
# Job that fails
job = queue.enqueue(div_by_zero)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job, queue)
self.assertNotIn(job.id, registry.get_job_ids())
def test_get_queue(self):
"""registry.get_queue() returns the right Queue object."""
registry = StartedJobRegistry(connection=self.testconn)
self.assertEqual(registry.get_queue(), Queue(connection=self.testconn))
registry = StartedJobRegistry('foo', connection=self.testconn)
self.assertEqual(registry.get_queue(),
Queue('foo', connection=self.testconn))
def test_contains(self):
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
self.assertFalse(job in registry)
self.assertFalse(job.id in registry)
registry.add(job, 5)
self.assertTrue(job in registry)
self.assertTrue(job.id in registry)
def test_started_jobs(self):
"""Ensure that active jobs page works properly."""
queue = get_queue('django_rq_test')
queue_index = get_queue_index('django_rq_test')
job = queue.enqueue(access_self)
registry = StartedJobRegistry(queue.name, queue.connection)
registry.add(job, 2)
response = self.client.get(
reverse('rq_started_jobs', args=[queue_index])
)
self.assertEqual(response.context['jobs'], [job])
def test_job_deletion(self):
"""Ensure job is removed from StartedJobRegistry when deleted."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
self.assertTrue(job.is_queued)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
job.delete()
self.assertNotIn(job.id, registry.get_job_ids())
def test_prepare_job_execution(self):
"""Prepare job execution does the necessary bookkeeping."""
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
worker = Worker([queue])
worker.prepare_job_execution(job)
# Updates working queue
registry = StartedJobRegistry(connection=self.testconn)
self.assertEqual(registry.get_job_ids(), [job.id])
# Updates worker statuses
self.assertEqual(worker.get_state(), 'busy')
self.assertEqual(worker.get_current_job_id(), job.id)
def test_prepare_job_execution_inf_timeout(self):
"""Prepare job execution handles infinite job timeout"""
queue = Queue(connection=self.testconn)
job = queue.enqueue(long_running_job,
args=(1,),
job_timeout=-1)
worker = Worker([queue])
worker.prepare_job_execution(job)
# Updates working queue
registry = StartedJobRegistry(connection=self.testconn)
self.assertEqual(registry.get_job_ids(), [job.id])
# Score in queue is +inf
self.assertEqual(self.testconn.zscore(registry.key, job.id), float('Inf'))
def started_jobs(request, queue_index):
queue_index = int(queue_index)
queue = get_queue_by_index(queue_index)
registry = StartedJobRegistry(queue.name, queue.connection)
items_per_page = 100
num_jobs = len(registry)
page = int(request.GET.get('page', 1))
jobs = []
if num_jobs > 0:
last_page = int(ceil(num_jobs / items_per_page))
page_range = range(1, last_page + 1)
offset = items_per_page * (page - 1)
job_ids = registry.get_job_ids(offset, offset + items_per_page - 1)
for job_id in job_ids:
try:
jobs.append(Job.fetch(job_id, connection=queue.connection))
except NoSuchJobError:
pass
else:
page_range = []
context_data = {
'queue': queue,
'queue_index': queue_index,
'jobs': jobs,
'num_jobs': num_jobs,
'page': page,
'page_range': page_range,
'job_status': 'Started',
}
return render(request, 'django_rq/jobs.html', context_data)
class TestRegistry(RQTestCase):
def setUp(self):
super(TestRegistry, self).setUp()
self.registry = StartedJobRegistry(connection=self.testconn)
def test_add_and_remove(self):
"""Adding and removing job to StartedJobRegistry."""
timestamp = current_timestamp()
job = Job()
# Test that job is added with the right score
self.registry.add(job, 1000)
self.assertLess(self.testconn.zscore(self.registry.key, job.id),
timestamp + 1002)
# Ensure that a timeout of -1 results in a score of -1
self.registry.add(job, -1)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), -1)
# Ensure that job is properly removed from sorted set
self.registry.remove(job)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
def test_get_job_ids(self):
"""Getting job ids from StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, timestamp + 10, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 20, 'bar')
self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar'])
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 10, 'bar')
self.testconn.zadd(self.registry.key, timestamp + 30, 'baz')
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20),
['foo', 'bar'])
def test_cleanup(self):
"""Moving expired jobs to FailedQueue."""
failed_queue = FailedQueue(connection=self.testconn)
self.assertTrue(failed_queue.is_empty())
self.testconn.zadd(self.registry.key, 2, 'foo')
self.registry.cleanup(1)
self.assertNotIn('foo', failed_queue.job_ids)
self.assertEqual(self.testconn.zscore(self.registry.key, 'foo'), 2)
self.registry.cleanup()
self.assertIn('foo', failed_queue.job_ids)
self.assertEqual(self.testconn.zscore(self.registry.key, 'foo'), None)
def test_job_execution(self):
"""Job is removed from StartedJobRegistry after execution."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job)
self.assertNotIn(job.id, registry.get_job_ids())
# Job that fails
job = queue.enqueue(div_by_zero)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job)
self.assertNotIn(job.id, registry.get_job_ids())
def test_get_job_count(self):
"""StartedJobRegistry returns the right number of job count."""
timestamp = current_timestamp() + 10
self.testconn.zadd(self.registry.key, timestamp, 'foo')
self.testconn.zadd(self.registry.key, timestamp, 'bar')
self.assertEqual(self.registry.count, 2)
self.assertEqual(len(self.registry), 2)
def requeue_interrupted_jobs(queue_name=None):
# type: (str) -> None
"""
Requeues jobs found in the given queue's started job registry.
Only restarts those that aren't already queued or being run.
When rq starts a job, it records it in the queue's started job
registry. If the server is rebooted before the job completes, the
job is not automatically restarted from the information in the
registry. For tasks like secure deletion of files, this means that
information thought to be deleted is still present in the case of
seizure or compromise. We have manage.py tasks to clean such files
up, but this utility attempts to reduce the need for manual
intervention by automatically resuming interrupted jobs.
This function is predicated on a risky assumption: that all jobs
are idempotent. At time of writing, we use rq for securely
deleting submission files and hashing submissions for the ETag
header. Both of these can be safely repeated. If we add rq tasks
that cannot, this function should be improved to omit those.
"""
queue = create_queue(queue_name)
started_job_registry = StartedJobRegistry(queue=queue)
queued_job_ids = queue.get_job_ids()
logging.debug("queued jobs: {}".format(queued_job_ids))
started_job_ids = started_job_registry.get_job_ids()
logging.debug("started jobs: {}".format(started_job_ids))
job_ids = [j for j in started_job_ids if j not in queued_job_ids]
logging.debug("candidate job ids: {}".format(job_ids))
if not job_ids:
logging.debug("No interrupted jobs found in started job registry.")
for job_id in job_ids:
logging.debug("Considering job %s", job_id)
try:
job = started_job_registry.job_class.fetch(
job_id, started_job_registry.connection)
except NoSuchJobError as e:
logging.error("Could not find details for job %s: %s", job_id, e)
continue
logging.debug("Job %s enqueued at %s, started at %s", job_id,
job.enqueued_at, job.started_at)
worker = worker_for_job(job_id)
if worker:
logging.info(
"Skipping job %s, which is already being run by worker %s",
job_id, worker.key)
continue
logging.info("Requeuing job %s", job)
try:
started_job_registry.remove(job)
except InvalidJobOperation as e:
logging.error(
"Could not remove job %s from started job registry: %s", job,
e)
continue
try:
queue.enqueue_job(job)
logging.debug("Job now enqueued at %s, started at %s",
job.enqueued_at, job.started_at)
except Exception as e:
logging.error("Could not requeue job %s: %s", job, e)
continue
def poll_and_update_on_processing(n_intervals, session_id, jobs):
if n_intervals == 0:
return '#', '', {'display': 'none'}, '#', {'display': 'none'}, ''
res = None
dbconn = eng.connect() if os.environ.get(
'STACK') else sqlite3.connect('output.db')
query = f'select distinct table_name \
as name from information_schema.tables' \
if os.environ.get('STACK') \
else 'select distinct name from sqlite_master'
def _check_for_output(n_intervals, dbconn):
df = pd.read_sql(query, con=dbconn, columns=['name'])
# print(df.columns)
# print('NAMES', df)
if jobs and any(df.name.str.contains(f'output_{session_id}_{jobs[-1]}')):
print('found output')
return True
else:
print('didn\'t find output')
return False
if _check_for_output(n_intervals, dbconn):
# Allow user to download the results of the most recent job
df = pd.read_sql(f'select * from output_{session_id}_{jobs[-1]}',
con=dbconn)
df.rename({'index': 'filename'}, axis=1, inplace=True)
csv_string = df.to_csv(index=False)
output = join(BASE_DIR, 'output')
session = join(output, f'output_{session_id}')
analysis = join(session, 'analysis')
job = join(analysis, f'job_{jobs[-1]}')
images = join(job, 'images')
for dir_ in [output, session, analysis, job, images]:
if not isdir(dir_):
os.mkdir(dir_)
# don't download if imagedir already full
# download s3 images
if len(os.listdir(images)) > 0:
# print(os.listdir(imagedir))
if not isfile(join(job, 'images.zip')):
tmp = shutil.make_archive(images, 'zip', images)
print(f'made archive {tmp}')
# for f in os.listdir(imagedir):
# os.unlink(join(imagedir, f))
# {'display': 'none'}
res = ("data:text/csv;charset=utf-8," + quote(csv_string), 'output.csv', {},
f'/dash/download?session_id={session_id}&job_id={jobs[-1]}', {},
dbc.Alert('Your results are ready!', color='success'))
else:
if not os.environ.get('STACK'):
with open(join(BASE_DIR, '.aws'), 'r') as f:
print('getting aws creds')
creds = json.loads(f.read())
AWS_ACCESS = creds['access']
AWS_SECRET = creds['secret']
else:
AWS_ACCESS = os.environ['AWS_ACCESS']
AWS_SECRET = os.environ['AWS_SECRET']
s3 = get_s3(AWS_ACCESS, AWS_SECRET)
# select bucket
bucket = s3.Bucket('voigt')
# download file into current directory
print(f'found S3 buckets {bucket.objects.all()}')
for s3_object in bucket.objects.all():
print(f'found S3 object {s3_object.key}')
do_zip = False
# Need to split s3_object.key into path and file name, else it will
# give error file not found.
if f'output_{session_id}/job_{jobs[-1]}' not in s3_object.key:
continue
path, filename = os.path.split(s3_object.key)
fpth = join(images, filename)
print(f'checking for file {s3_object.key}')
if not isfile(fpth):
print(f'downloading file {s3_object.key}')
bucket.download_file(s3_object.key, fpth)
do_zip = True
if do_zip:
tmp = shutil.make_archive(images, 'zip', images)
print(f'made archive {tmp}')
res = ("data:text/csv;charset=utf-8," + quote(csv_string), 'output.csv', {},
f'/dash/download?session_id={session_id}&job_id={jobs[-1]}', {},
dbc.Alert('Your results are ready!', color='success'))
else:
registry = StartedJobRegistry('default', connection=conn)
input_dir = join(BASE_DIR, 'input', f'input_{session_id}', 'analysis')
# TODO concurrency? what if mutliple ppl use app at same time?
if (jobs and jobs[-1] not in registry.get_job_ids()) or not jobs:
msg = dbc.Alert('Ready.', color='primary') if len(os.listdir(input_dir)) > 0 \
else dbc.Alert('Upload some peak files first!', color='warning')
res = ('#', '', {'display': 'none'}, '#', {'display': 'none'}, msg)
elif jobs and jobs[-1] in registry.get_job_ids():
res = ('#', '', {'display': 'none'}, '#', {'display': 'none'},
dbc.Alert(
[
'Please wait while your request is processed.',
dbc.Spinner(type='grow')
],
color='danger')
)
dbconn.close()
return res
class TestRegistry(RQTestCase):
def setUp(self):
super(TestRegistry, self).setUp()
self.registry = StartedJobRegistry(connection=self.testconn)
def test_key(self):
self.assertEqual(self.registry.key, 'rq:wip:default')
def test_custom_job_class(self):
registry = StartedJobRegistry(job_class=CustomJob)
self.assertFalse(registry.job_class == self.registry.job_class)
def test_add_and_remove(self):
"""Adding and removing job to StartedJobRegistry."""
timestamp = current_timestamp()
job = Job()
# Test that job is added with the right score
self.registry.add(job, 1000)
self.assertLess(self.testconn.zscore(self.registry.key, job.id),
timestamp + 1002)
# Ensure that a timeout of -1 results in a score of -1
self.registry.add(job, -1)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), -1)
# Ensure that job is properly removed from sorted set
self.registry.remove(job)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
def test_get_job_ids(self):
"""Getting job ids from StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, timestamp + 10, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 20, 'bar')
self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar'])
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 10, 'bar')
self.testconn.zadd(self.registry.key, timestamp + 30, 'baz')
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20),
['foo', 'bar'])
def test_cleanup(self):
"""Moving expired jobs to FailedQueue."""
failed_queue = FailedQueue(connection=self.testconn)
self.assertTrue(failed_queue.is_empty())
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
self.testconn.zadd(self.registry.key, 2, job.id)
self.registry.cleanup(1)
self.assertNotIn(job.id, failed_queue.job_ids)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), 2)
self.registry.cleanup()
self.assertIn(job.id, failed_queue.job_ids)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), None)
job.refresh()
self.assertEqual(job.get_status(), JobStatus.FAILED)
def test_job_execution(self):
"""Job is removed from StartedJobRegistry after execution."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
self.assertTrue(job.is_queued)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
self.assertTrue(job.is_started)
worker.perform_job(job, queue)
self.assertNotIn(job.id, registry.get_job_ids())
self.assertTrue(job.is_finished)
# Job that fails
job = queue.enqueue(div_by_zero)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job, queue)
self.assertNotIn(job.id, registry.get_job_ids())
def test_job_deletion(self):
"""Ensure job is removed from StartedJobRegistry when deleted."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
self.assertTrue(job.is_queued)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
job.delete()
self.assertNotIn(job.id, registry.get_job_ids())
def test_get_job_count(self):
"""StartedJobRegistry returns the right number of job count."""
timestamp = current_timestamp() + 10
self.testconn.zadd(self.registry.key, timestamp, 'foo')
self.testconn.zadd(self.registry.key, timestamp, 'bar')
self.assertEqual(self.registry.count, 2)
self.assertEqual(len(self.registry), 2)
def test_clean_registries(self):
"""clean_registries() cleans Started and Finished job registries."""
queue = Queue(connection=self.testconn)
finished_job_registry = FinishedJobRegistry(connection=self.testconn)
self.testconn.zadd(finished_job_registry.key, 1, 'foo')
started_job_registry = StartedJobRegistry(connection=self.testconn)
self.testconn.zadd(started_job_registry.key, 1, 'foo')
clean_registries(queue)
self.assertEqual(self.testconn.zcard(finished_job_registry.key), 0)
self.assertEqual(self.testconn.zcard(started_job_registry.key), 0)
def get_current_rq_jobs():
registry = StartedJobRegistry('default', connection=get_redis_connection())
running_job_ids = registry.get_job_ids()
return len(running_job_ids)
class QueueManager:
retries_before_fail = 10
backoff = 30
def __init__(self, name: str, connection: redis.Redis):
self._name = name
self._logger = logger.bind(queue=name)
self._queue = rq.Queue("vivarium", connection=connection)
self._wip = StartedJobRegistry(
"vivarium", connection=connection, job_class=self._queue.job_class
)
self._finished = FinishedJobRegistry(
"vivarium", connection=connection, job_class=self._queue.job_class
)
self._status = {
"total": 0,
"pending": 0,
"running": 0,
"failed": 0,
"finished": 0,
"done": 0.0,
"workers": 0,
}
self._failed = False
self._completed = False
self._retries = QueueManager.retries_before_fail
self._backoff = QueueManager.backoff
@property
def name(self) -> str:
return self._name
@property
def failed(self) -> bool:
return self._failed
@property
def completed(self) -> bool:
return self._completed
@property
def jobs_to_finish(self) -> bool:
return not (self.failed or self.completed)
def enqueue(self, jobs: List[dict], workhorse_import_path: str) -> None:
self._logger.info(f"Enqueuing jobs in queue {self.name}")
for job in jobs:
# TODO: might be nice to have tighter ttls but it's hard to predict
# how long our jobs will take from model to model and the entire
# system is short lived anyway
self._queue.enqueue(
workhorse_import_path,
job_parameters=job,
ttl=60 * 60 * 24 * 2,
result_ttl=60 * 60,
job_timeout="7d",
)
def get_results(self) -> List:
self._logger.info(f"Checking queue {self.name}")
finished_jobs = self._get_finished_jobs()
start = time.time()
results = []
for job_id in finished_jobs:
result = self._get_result(job_id)
if result is not None:
results.append(result)
self._logger.info(
f"Retrieved {len(results)} results from queue {self.name} in {time.time() - start:.2f}s"
)
return results
def update_and_report(self) -> Dict[str, int]:
self._update_status()
template = (
f"Queue {self.name} - Total jobs: {{total}}, % Done: {{done:.2f}}% "
f"Pending: {{pending}}, Running: {{running}}, Failed: {{failed}}, Finished: {{finished}} "
f"Workers: {{workers}}."
)
if not (self.completed or self.failed):
self._logger.info(template.format(**self._status))
return self._status
def _update_status(self) -> None:
if self.jobs_to_finish and self._retries > 0:
self._logger.info(f"Updating status for queue {self.name}")
# TODO: Sometimes there are duplicate job_ids, why?
try:
# Account for timing discrepancies in accounting and the
# fact that a job might legit be in more than one of these.
finished_ids = set(self._finished.get_job_ids()) if self._queue else set()
running_ids = set(self._wip.get_job_ids()) if self._queue else set()
# "Finished" for our purposes means the job is done
# and the results have been retrieved, whereas for rq
# finished means the job is done and the results have
# not been retrieved. Once retrieved, jobs disappear
# from the finished job registry and we'd see all our
# account eventually just go to zero.
running_ids = running_ids | finished_ids
pending_ids = set(self._queue.job_ids) if self._queue else set()
pending_ids = pending_ids - running_ids
failed_ids = (
set(self._queue.failed_job_registry.get_job_ids())
if self._queue
else set()
)
failed_ids = failed_ids - (running_ids | pending_ids)
q_workers = len(rq.Worker.all(queue=self._queue)) if self._queue else 0
self._status["pending"] = len(pending_ids)
self._status["running"] = len(running_ids) + len(finished_ids)
self._status["failed"] = len(failed_ids)
self._status["finished"] = self._status["finished"]
self._status["total"] = (
len(pending_ids)
+ len(running_ids)
+ len(failed_ids)
+ self._status["finished"]
)
self._status["workers"] = q_workers
self._status["done"] = 100 * self._status["finished"] / self._status["total"]
if len(pending_ids) + len(running_ids) == 0:
self._mark_complete()
elif len(pending_ids) + len(finished_ids) + q_workers == 0:
self._logger.info(
f"Queue {self.name} ran out of workers with running jobs. Marking finished."
)
self._mark_complete()
except redis.connection.ConnectionError:
self._sleep_on_it()
self._update_status()
else:
self._mark_failed()
def _get_finished_jobs(self) -> List[str]:
if self._retries:
try:
return self._finished.get_job_ids()
except redis.exceptions.ConnectionError:
self._sleep_on_it()
return self._get_finished_jobs()
else:
self._mark_failed()
return []
def _get_result(self, job_id: str) -> Any:
job = self._get_job(job_id)
result = None
if job is not None:
start = time.time()
result = job.result
end = time.time()
self._logger.debug(
f"Deserialized {job_id} result from queue {self.name} in {end - start:.2f}s."
)
self._status["finished"] += 1
self._remove_finished_job(job)
return result
def _get_job(self, job_id: str) -> Union[None, Job]:
if self._retries:
try:
start = time.time()
job = self._queue.fetch_job(job_id)
end = time.time()
self._logger.debug(
f"Fetched job {job_id} from queue {self.name} in {end - start:.2f}s"
)
return job
except redis.exceptions.ConnectionError:
self._sleep_on_it()
return self._get_job(job_id)
else:
self._mark_failed()
return None
def _remove_finished_job(self, job: Job) -> None:
if self._retries:
try:
self._finished.remove(job)
except redis.exceptions.ConnectionError:
self._sleep_on_it()
self._remove_finished_job(job)
else:
self._mark_failed()
def _sleep_on_it(self) -> None:
backoff = self._backoff * random.random()
self._logger.warning(
f"Failed to connect to redis for queue {self.name}. Retrying in {backoff}s."
)
self._retries -= 1
time.sleep(backoff)
def _mark_failed(self) -> None:
# TODO: Probably should cleanup redis stuff, but not sure how yet.
if not (self.failed or self.completed):
self._logger.warning(
f"Queue {self.name} has run out of retries. Marking as failed."
)
self._status["failed"] += self._status["pending"] + self._status["running"]
self._status["pending"] = 0
self._status["running"] = 0
self._status["workers"] = 0
self._failed = True
def _mark_complete(self) -> None:
if not (self.failed or self.completed):
self._logger.info(f"All jobs on queue {self.name} complete.")
self._status["finished"] += self._status["pending"] + self._status["running"]
self._status["pending"] = 0
self._status["running"] = 0
self._status["workers"] = 0
self._completed = True
def GetJobProgress(app):
"""Infinite thread that sends job updates to client."""
with app.app_context():
redis = Redis()
queue = Queue(connection=redis)
running_registry = StartedJobRegistry(queue=queue)
finished_registry = FinishedJobRegistry(queue=queue)
showing_running_job_bar = False
toggle_on_running_job_bar = False
toggle_off_running_job_bar = False
while True:
time.sleep(2)
running_msg = "Running jobs:"
finished_msg = ""
running_job_id_list = running_registry.get_job_ids()
finished_job_id_list = finished_registry.get_job_ids()
if running_job_id_list:
# Get all job progress and emit update
for job_id in running_job_id_list:
running_msg += "<br>Job ID: {}".format(job_id)
running_job = Job.fetch(id=job_id, connection=redis)
if 'process' in running_job.meta:
running_msg += "<UL><LI>Process: {}".format(
running_job.meta['process'])
if 'progress' in running_job.meta:
running_msg += "<LI>Progress: {}</UL>".format(
running_job.meta['progress'])
socketio.emit('update job progress', {'data': running_msg})
toggle_on_running_job_bar = True
else:
toggle_off_running_job_bar = True
if finished_job_id_list:
# Display finished job notification and save to db
for job_id in finished_job_id_list:
finished_msg += "<br>Job ID: {}".format(job_id)
finished_job = Job.fetch(job_id, connection=redis)
if 'process' in finished_job.meta:
finished_msg += "<UL><LI>Process: {}".format(
finished_job.meta['process'])
rq_job = RQJob()
rq_job.save_job(finished_job)
finished_registry.remove(finished_job)
finished_msg += "<LI>Result: {}".format(
rq_job.get_result())
finished_msg += "<LI>Time elapsed: {} seconds</UL>".format(
rq_job.get_elapsed_time())
socketio.emit('update job finished', {'data': finished_msg})
socketio.emit('show finished job bar')
if showing_running_job_bar and toggle_off_running_job_bar:
toggle_off_running_job_bar = False
toggle_on_running_job_bar = False
showing_running_job_bar = False
print('---------Server: Hide running job bar')
socketio.emit('hide running job bar')
elif not showing_running_job_bar and toggle_on_running_job_bar:
toggle_off_running_job_bar = False
toggle_on_running_job_bar = False
showing_running_job_bar = True
print('---------Server: Show running job bar')
socketio.emit('show running job bar')
return
from words.datamuse_json import DatamuseWordNotRecognizedError
from words.forms import RelatedWordsForm, WordSetCreateForm, WordSetChoice, ScatterplotWordSetChoice
from words.models import WordSet, Word
# Get an instance of a logger
logger = logging.getLogger(__name__)
# se up django-rq queue
redis_url = os.getenv('REDISTOGO_URL', 'redis://localhost:6379')
redis_cursor = redis.from_url(redis_url) # singleton Redis client
# queue for running rq worker for WordSetCreateForm save
rq_queue = django_rq.get_queue('default', connection=redis_cursor)
# register to track running rq jobs
rq_started_job_registry = StartedJobRegistry(
queue=rq_queue) # connection=redis_cursor) #, queue=rq_queue)
def job_json(request, job_id):
"""Given the job_id for a django-rq job, exposes the job status, meta and result (if any) in json format"""
job = Job.fetch(job_id, connection=redis_cursor)
reply = {
'status': job.get_status(),
'meta': job.meta,
'result': job.result
}
return JsonResponse(reply)
def current_jobs(self):
""" Track currently active and queued jobs """
registry = StartedJobRegistry(queue=self.generate_queue)
jobs = registry.get_job_ids() + self.current_jobs()
return jobs
import logging
from rq import Queue
from rq.job import Job
from collections import deque
from rq import get_current_job
from django.conf import settings
import django_rq
from rq.registry import FinishedJobRegistry, StartedJobRegistry
logger = logging.getLogger('uprilogger')
q = django_rq.get_queue()
# rqq = Queue(connection=django_rq.get_connection())
finished_job_registry = FinishedJobRegistry(connection=django_rq.get_connection())
started_job_registry = StartedJobRegistry(connection=django_rq.get_connection())
failed_queue = django_rq.get_failed_queue()
#
# Job management
#
def job_message(message):
job = get_current_job(connection=django_rq.get_connection())
if not job.meta.get('messages'):
job.meta['messages'] = deque()
job.meta['messages'].append(message)
job.save()
from rq import Queue
from rq.job import Job
from rq.registry import StartedJobRegistry, FinishedJobRegistry
from redisQworker import conn
from model import train, predict
import config
import datetime
id_time_format = "%Y-%m-%d %H:%M:%S"
train_job_id = "train_job"
train_new_job_id = "new_train_job"
log = logging.getLogger(__name__)
q_train = Queue(connection=conn, name='train')
registr_train = StartedJobRegistry('train', connection=conn)
started_registry = StartedJobRegistry('train', connection=conn)
finished_registry = FinishedJobRegistry('train', connection=conn)
def init_time_id(id_time_format=id_time_format):
return datetime.datetime.utcnow().strftime(id_time_format)
def sort_id_time(list_id_time, id_time_format=id_time_format, reverse=False):
list_datetime = [
datetime.datetime.strptime(x, id_time_format) for x in list_id_time
]
list_datetime = sorted(list_datetime, reverse=reverse)
sorted_list = [x.strftime(id_time_format) for x in list_datetime]
class TestRegistry(RQTestCase):
def setUp(self):
super(TestRegistry, self).setUp()
self.registry = StartedJobRegistry(connection=self.testconn)
def test_init(self):
"""Registry can be instantiated with queue or name/Redis connection"""
queue = Queue('foo', connection=self.testconn)
registry = StartedJobRegistry(queue=queue)
self.assertEqual(registry.name, queue.name)
self.assertEqual(registry.connection, queue.connection)
registry = StartedJobRegistry('bar', self.testconn)
self.assertEqual(registry.name, 'bar')
self.assertEqual(registry.connection, self.testconn)
def test_key(self):
self.assertEqual(self.registry.key, 'rq:wip:default')
def test_custom_job_class(self):
registry = StartedJobRegistry(job_class=CustomJob)
self.assertFalse(registry.job_class == self.registry.job_class)
def test_contains(self):
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
self.assertFalse(job in registry)
self.assertFalse(job.id in registry)
registry.add(job, 5)
self.assertTrue(job in registry)
self.assertTrue(job.id in registry)
def test_get_expiration_time(self):
"""registry.get_expiration_time() returns correct datetime objects"""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
registry.add(job, 5)
self.assertEqual(
registry.get_expiration_time(job),
(datetime.utcnow() + timedelta(seconds=5)).replace(microsecond=0)
)
def test_add_and_remove(self):
"""Adding and removing job to StartedJobRegistry."""
timestamp = current_timestamp()
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
# Test that job is added with the right score
self.registry.add(job, 1000)
self.assertLess(self.testconn.zscore(self.registry.key, job.id),
timestamp + 1002)
# Ensure that a timeout of -1 results in a score of inf
self.registry.add(job, -1)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), float('inf'))
# Ensure that job is removed from sorted set, but job key is not deleted
self.registry.remove(job)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
self.assertTrue(self.testconn.exists(job.key))
self.registry.add(job, -1)
# registry.remove() also accepts job.id
self.registry.remove(job.id)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
self.registry.add(job, -1)
# delete_job = True deletes job key
self.registry.remove(job, delete_job=True)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
self.assertFalse(self.testconn.exists(job.key))
job = queue.enqueue(say_hello)
self.registry.add(job, -1)
# delete_job = True also works with job.id
self.registry.remove(job.id, delete_job=True)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
self.assertFalse(self.testconn.exists(job.key))
def test_get_job_ids(self):
"""Getting job ids from StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': timestamp + 10})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 20})
self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar'])
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': 1})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 10})
self.testconn.zadd(self.registry.key, {'baz': timestamp + 30})
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20),
['foo', 'bar'])
def test_cleanup_moves_jobs_to_failed_job_registry(self):
"""Moving expired jobs to FailedJobRegistry."""
queue = Queue(connection=self.testconn)
failed_job_registry = FailedJobRegistry(connection=self.testconn)
job = queue.enqueue(say_hello)
self.testconn.zadd(self.registry.key, {job.id: 2})
# Job has not been moved to FailedJobRegistry
self.registry.cleanup(1)
self.assertNotIn(job, failed_job_registry)
self.assertIn(job, self.registry)
self.registry.cleanup()
self.assertIn(job.id, failed_job_registry)
self.assertNotIn(job, self.registry)
job.refresh()
self.assertEqual(job.get_status(), JobStatus.FAILED)
self.assertTrue(job.exc_info) # explanation is written to exc_info
def test_job_execution(self):
"""Job is removed from StartedJobRegistry after execution."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
self.assertTrue(job.is_queued)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
self.assertTrue(job.is_started)
worker.perform_job(job, queue)
self.assertNotIn(job.id, registry.get_job_ids())
self.assertTrue(job.is_finished)
# Job that fails
job = queue.enqueue(div_by_zero)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job, queue)
self.assertNotIn(job.id, registry.get_job_ids())
def test_job_deletion(self):
"""Ensure job is removed from StartedJobRegistry when deleted."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
self.assertTrue(job.is_queued)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
job.delete()
self.assertNotIn(job.id, registry.get_job_ids())
def test_get_job_count(self):
"""StartedJobRegistry returns the right number of job count."""
timestamp = current_timestamp() + 10
self.testconn.zadd(self.registry.key, {'foo': timestamp})
self.testconn.zadd(self.registry.key, {'bar': timestamp})
self.assertEqual(self.registry.count, 2)
self.assertEqual(len(self.registry), 2)
def test_clean_registries(self):
"""clean_registries() cleans Started and Finished job registries."""
queue = Queue(connection=self.testconn)
finished_job_registry = FinishedJobRegistry(connection=self.testconn)
self.testconn.zadd(finished_job_registry.key, {'foo': 1})
started_job_registry = StartedJobRegistry(connection=self.testconn)
self.testconn.zadd(started_job_registry.key, {'foo': 1})
failed_job_registry = FailedJobRegistry(connection=self.testconn)
self.testconn.zadd(failed_job_registry.key, {'foo': 1})
clean_registries(queue)
self.assertEqual(self.testconn.zcard(finished_job_registry.key), 0)
self.assertEqual(self.testconn.zcard(started_job_registry.key), 0)
self.assertEqual(self.testconn.zcard(failed_job_registry.key), 0)
def test_get_queue(self):
"""registry.get_queue() returns the right Queue object."""
registry = StartedJobRegistry(connection=self.testconn)
self.assertEqual(registry.get_queue(), Queue(connection=self.testconn))
registry = StartedJobRegistry('foo', connection=self.testconn)
self.assertEqual(registry.get_queue(),
Queue('foo', connection=self.testconn))
def get_statistics():
queues = []
workers_collections = collect_workers_by_connection(QUEUES_LIST)
for index, config in enumerate(QUEUES_LIST):
queue = get_queue_by_index(index)
connection = queue.connection
connection_kwargs = connection.connection_pool.connection_kwargs
# Raw access to the first item from left of the redis list.
# This might not be accurate since new job can be added from the left
# with `at_front` parameters.
# Ideally rq should supports Queue.oldest_job
last_job_id = connection.lindex(queue.key, 0)
last_job = queue.fetch_job(
last_job_id.decode('utf-8')) if last_job_id else None
if last_job:
oldest_job_timestamp = to_localtime(last_job.enqueued_at)\
.strftime('%Y-%m-%d, %H:%M:%S')
else:
oldest_job_timestamp = "-"
# parse_class is not needed and not JSON serializable
try:
del (connection_kwargs['parser_class'])
except KeyError:
pass
queue_data = {
'name': queue.name,
'jobs': queue.count,
'oldest_job_timestamp': oldest_job_timestamp,
'index': index,
'connection_kwargs': connection_kwargs
}
if queue.name == 'failed':
queue_data['workers'] = '-'
queue_data['finished_jobs'] = '-'
queue_data['started_jobs'] = '-'
queue_data['deferred_jobs'] = '-'
elif RQ_SCHEDULER_INSTALLED and queue.name == 'scheduled':
scheduler = get_scheduler(connection=queue.connection)
queue_data['jobs'] = scheduler.count()
queue_data['workers'] = '-'
queue_data['finished_jobs'] = '-'
queue_data['started_jobs'] = '-'
queue_data['deferred_jobs'] = '-'
else:
connection = get_connection(queue.name)
all_workers = get_all_workers_by_configuration(
config['connection_config'], workers_collections)
queue_workers = [
worker for worker in all_workers if queue in worker.queues
]
queue_data['workers'] = len(queue_workers)
finished_job_registry = FinishedJobRegistry(queue.name, connection)
started_job_registry = StartedJobRegistry(queue.name, connection)
deferred_job_registry = DeferredJobRegistry(queue.name, connection)
queue_data['finished_jobs'] = len(finished_job_registry)
queue_data['started_jobs'] = len(started_job_registry)
queue_data['deferred_jobs'] = len(deferred_job_registry)
queues.append(queue_data)
return {'queues': queues, 'display_scheduled_jobs': RQ_SCHEDULER_INSTALLED}
sys.path.insert(0, "/home/christophe/workspace/velona/rq")
from time import sleep
from rqtest.functions import wait_a_bit
from redis import Redis
from rq import Queue
from rq.registry import StartedJobRegistry
"""
Test enqueing a job but deleting it before it has finished. The reason is to Test
that the job is correctly remove from the redis StartedJobRegistry
"""
conn = Redis('localhost', 6379)
q = Queue(connection=conn)
job = q.enqueue(wait_a_bit, 10)
registry = StartedJobRegistry(job.origin, conn)
print('Enqueued', job)
sleep(2)
print('StartedJobRegistry, before delete:')
print(registry.get_job_ids())
job.delete()
print('StartedJobRegistry, after delete:')
print(registry.get_job_ids())
request,
make_response
)
from redis import Redis
from stocks import (
StockDatabase,
)
queue_name = "trading-tasks"
db = StockDatabase('84WDI082Z0HOREL6')
tasks = []
app = Flask(__name__)
redis = Redis.from_url("redis://*****:*****@app.route('/api/trading/stocks/suggest/', methods=["POST"])
def suggest_stock():
return make_response(jsonify(None), 404)
@app.route('/api/trading/stocks/search/', methods=["GET"])
def search_stock():
"""
Describes a specific stock
def test_job_delete_removes_itself_from_registries(self):
"""job.delete() should remove itself from job registries"""
connection = self.testconn
job = Job.create(func=fixtures.say_hello,
status=JobStatus.FAILED,
connection=self.testconn,
origin='default')
job.save()
registry = FailedJobRegistry(connection=self.testconn)
registry.add(job, 500)
job.delete()
self.assertFalse(job in registry)
job = Job.create(func=fixtures.say_hello,
status=JobStatus.FINISHED,
connection=self.testconn,
origin='default')
job.save()
registry = FinishedJobRegistry(connection=self.testconn)
registry.add(job, 500)
job.delete()
self.assertFalse(job in registry)
job = Job.create(func=fixtures.say_hello,
status=JobStatus.STARTED,
connection=self.testconn,
origin='default')
job.save()
registry = StartedJobRegistry(connection=self.testconn)
registry.add(job, 500)
job.delete()
self.assertFalse(job in registry)
job = Job.create(func=fixtures.say_hello,
status=JobStatus.DEFERRED,
connection=self.testconn,
origin='default')
job.save()
registry = DeferredJobRegistry(connection=self.testconn)
registry.add(job, 500)
job.delete()
self.assertFalse(job in registry)
job = Job.create(func=fixtures.say_hello,
status=JobStatus.SCHEDULED,
connection=self.testconn,
origin='default')
job.save()
registry = ScheduledJobRegistry(connection=self.testconn)
registry.add(job, 500)
job.delete()
self.assertFalse(job in registry)
ParoptManager.start()
app.run(debug=True,
host="0.0.0.0",
port=8080,
use_reloader=False,
ssl_context='adhoc')
elif args.setupaws:
# run func to configure aws provider state
setupAWS()
else:
if args.workers <= 0:
print("Error: --workers must be an integer > 0")
sys.exit(1)
redis_url = app.config['REDIS_URL']
# clear previously started started jobs - if shut down while running a job, the job will remain in StartedJobsRegistry
# when it's restarted, which is a problem because it's not actually running anymore
with Connection(redis.from_url(redis_url)) as conn:
registry = StartedJobRegistry('default', connection=conn)
for job_id in registry.get_job_ids():
registry.remove(Job.fetch(job_id, connection=conn))
procs = []
for i in range(args.workers):
procs.append(
Process(target=startWorker,
args=(redis_url, app.config['QUEUES'])))
procs[i].start()
for proc in procs:
proc.join()
def _repr(q):
return "running:%d pending:%d finished:%d" % (StartedJobRegistry(
q.name, conn).count, q.count, FinishedJobRegistry(q.name,
conn).count)
from rq import Queue
from rq.job import Job
from rq.exceptions import NoSuchJobError
from rq.registry import StartedJobRegistry
from sitesearch.config import Config
from sitesearch.connections import get_search_connection, get_rq_redis_client
from sitesearch.tasks import JOB_ID, JOB_STARTED, JOB_NOT_QUEUED, index, INDEXING_TIMEOUT
from .resource import Resource
config = Config()
redis_client = get_rq_redis_client()
search_client = get_search_connection(config.default_search_site.index_name)
log = logging.getLogger(__name__)
queue = Queue(connection=redis_client)
registry = StartedJobRegistry('default', connection=redis_client)
API_KEY = os.environ['API_KEY']
class IndexerResource(Resource):
def on_get(self, req, resp):
"""Start an indexing job."""
try:
status = Job.fetch(JOB_ID, connection=redis_client).get_status()
except NoSuchJobError:
if JOB_ID in registry.get_job_ids():
status = JOB_STARTED
else:
status = JOB_NOT_QUEUED
def test_custom_job_class(self):
registry = StartedJobRegistry(job_class=CustomJob)
self.assertFalse(registry.job_class == self.registry.job_class)
def poll_and_update_on_processing(n_intervals, session_id, fit_jobs, file_format, min_, max_):
if n_intervals == 0:
return '#', {'display': 'none'}, '', '', True
res = None
def job_status():
if len(fit_jobs) == 0:
return 'ready'
try:
job = Job.fetch(fit_jobs[-1], connection=conn)
except NoSuchJobError:
return 'finished'
print(f'Job status: {job.get_status()}')
return job.get_status()
input_dir = join(BASE_DIR, 'input', f'input_{session_id}', 'fitting')
if job_status() == 'queued':
msg = dbc.Alert(['Waiting for compute resources...',
dbc.Spinner(type='grow')], color='warning')
res = ('#', '', {'display': 'none'}, msg, '', True)
elif job_status() == 'ready':
msg, flag = (dbc.Alert('Ready.', color='primary'), False) if len(os.listdir(input_dir)) > 0 \
else (dbc.Alert('Upload some TGA measurements first!', color='warning'), True)
res = ('#', '', {'display': 'none'}, msg, '', flag)
elif job_status() == 'finished':
fitting = join(BASE_DIR, 'output',
f'output_{session_id}', 'fitting')
outputdir = join(fitting, f'job_{fit_jobs[-1]}')
if not exists(outputdir):
os.mkdir(outputdir)
# already downloaded results
if len(os.listdir(outputdir)) > 0:
if not isfile(join(fitting, f'job_{fit_jobs[-1]}.zip')):
tmp = shutil.make_archive(outputdir, 'zip', outputdir)
print(f'made archive {tmp}')
res = (f'/dash/download-fit?session_id={session_id}&job_id={fit_jobs[-1]}', 'fits.csv', {},
dbc.Alert('Your results are ready!', color='success'), '', False)
# download results
else:
if not os.environ.get('STACK'):
with open(join(BASE_DIR, '.aws'), 'r') as f:
print('getting aws creds')
creds = json.loads(f.read())
AWS_ACCESS = creds['access']
AWS_SECRET = creds['secret']
else:
AWS_ACCESS = os.environ['AWS_ACCESS']
AWS_SECRET = os.environ['AWS_SECRET']
s3 = get_s3(AWS_ACCESS, AWS_SECRET)
# select bucket
bucket = s3.Bucket('voigt')
# download file into current directory
# print(f'found S3 buckets {bucket.objects.all()}')
for s3_object in bucket.objects.all():
do_zip = False
# Need to split s3_object.key into path and file name, else it will
# give error file not found.
if f'output_{session_id}/fitting/job_{fit_jobs[-1]}' not in s3_object.key:
continue
print(f'found S3 object {s3_object.key}')
path, filename = os.path.split(s3_object.key)
fpth = join(outputdir, filename)
print(f'checking for file {s3_object.key}')
if not isfile(fpth):
print(f'downloading file {s3_object.key}')
bucket.download_file(s3_object.key, fpth)
do_zip = True
if do_zip:
tmp = shutil.make_archive(outputdir, 'zip', outputdir)
print(f'made archive {tmp}')
res = (f'/dash/download-fit?session_id={session_id}&job_id={fit_jobs[-1]}', 'fits.csv', {},
dbc.Alert('Your results are ready!', color='success'), '', False)
elif job_status() == 'failed':
job = Job.fetch(fit_jobs[-1], connection=conn)
job_id = fit_jobs[-1]
if 'JobTimeoutException' in job.exc_info:
res = ('#', '', {'display': 'none'}, dbc.Alert(f'Job {session_id}:{job_id} failed due to timeout!', color='danger'), '', False)
else:
res = ('#', '', {'display': 'none'}, dbc.Alert(f'Job {session_id}:{job_id} failed!', color='danger'), '', False)
jobdir = join(BASE_DIR, 'output', f'output_{session_id}', 'fitting', f'job_{job_id}')
with open(join(jobdir, 'log.txt'), 'w') as f:
f.write(job.exc_info)
# upload log to S3
if os.environ.get('STACK'):
AWS_ACCESS = os.environ['AWS_ACCESS']
AWS_SECRET = os.environ['AWS_SECRET']
else:
with open(join(BASE_DIR, '.aws'), 'r') as f:
creds = json.loads(f.read())
AWS_ACCESS = creds['access']
AWS_SECRET = creds['secret']
from voigt.common.amazon import upload_file
s3_pth = join(f'output_{session_id}', 'fitting', f'job_{job_id}', 'log', 'log.txt')
upload_file(join(jobdir, 'log.txt'), object_name=s3_pth, aws_access_key_id=AWS_ACCESS,
aws_secret_access_key=AWS_SECRET)
elif job_status() == 'started':
registry = StartedJobRegistry('default', connection=conn)
# TODO concurrency? what if mutliple ppl use app at same time?
if (fit_jobs and fit_jobs[-1] not in registry.get_job_ids()) or not fit_jobs:
msg = dbc.Alert('Ready.', color='primary') if len(os.listdir(input_dir)) > 0 \
else dbc.Alert('Upload some TGA measurements first!', color='warning')
res = ('#', '', {'display': 'none'}, msg, '', True)
elif fit_jobs and fit_jobs[-1] in registry.get_job_ids():
res = ('#', '', {'display': 'none'},
dbc.Alert(
[
'Please wait while your request is processed.',
dbc.Spinner(type='grow')
],
color='warning'),
'',
True
)
res = list(res)
data = [read_data(join(input_dir, f), format=file_format)
for f in os.listdir(input_dir)]
if any([x[2][0] > min_ for x in data]):
msg = dbc.Alert(
'Run Start Temp outside of actual bounds!', color='warning')
disabled = True
elif any([x[2][-1] < max_ for x in data]):
msg = dbc.Alert(
'Run End Temp outside of actual bounds!', color='warning')
disabled = True
else:
msg = ''
disabled = False
res[-2] = msg
res[-1] = disabled
# print(res)
return tuple(res)
def jobs_running():
registry = StartedJobRegistry(name=sjs.get_sjs_config()['queue'], connection=sjs.get_redis_conn())
return registry.get_job_ids()
class TestQueue(RQTestCase):
def setUp(self):
super(TestQueue, self).setUp()
self.registry = StartedJobRegistry(connection=self.testconn)
def test_add_and_remove(self):
"""Adding and removing job to StartedJobRegistry."""
timestamp = current_timestamp()
job = Job()
# Test that job is added with the right score
self.registry.add(job, 1000)
self.assertLess(self.testconn.zscore(self.registry.key, job.id),
timestamp + 1002)
# Ensure that job is properly removed from sorted set
self.registry.remove(job)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
def test_get_job_ids(self):
"""Getting job ids from StartedJobRegistry."""
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, 10, 'bar')
self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar'])
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 10, 'bar')
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
def test_cleanup(self):
"""Moving expired jobs to FailedQueue."""
failed_queue = FailedQueue(connection=self.testconn)
self.assertTrue(failed_queue.is_empty())
self.testconn.zadd(self.registry.key, 1, 'foo')
self.registry.move_expired_jobs_to_failed_queue()
self.assertIn('foo', failed_queue.job_ids)
def test_job_execution(self):
"""Job is removed from StartedJobRegistry after execution."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job)
self.assertNotIn(job.id, registry.get_job_ids())
# Job that fails
job = queue.enqueue(div_by_zero)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job)
self.assertNotIn(job.id, registry.get_job_ids())
def test_get_job_count(self):
"""StartedJobRegistry returns the right number of job count."""
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, 10, 'bar')
self.assertEqual(self.registry.count, 2)
self.assertEqual(len(self.registry), 2)
#! /usr/bin/env python import sys from redis import StrictRedis from cvgmeasure.conf import REDIS_URL_RQ from rq.registry import StartedJobRegistry from rq.utils import current_timestamp r = StrictRedis.from_url(REDIS_URL_RQ) sjr = StartedJobRegistry(sys.argv[1], connection=r) #sjr.cleanup() sjr.cleanup(timestamp=current_timestamp()+100000000000)
def perform_job(self, job):
"""Performs the actual work of a job. Will/should only be called
inside the work horse's process.
"""
self.prepare_job_execution(job)
with self.connection._pipeline() as pipeline:
started_job_registry = StartedJobRegistry(job.origin,
self.connection)
try:
logging.debug('perform_job in sw')
job.matlab_engine = self.matlab_engine
logging.debug('pj engine:' + str(self.matlab_engine))
# logging.debug('pj args,kwargs:'+str(job._args)+','+str(job._kwargs))
if len(job._args) > 0:
new_args = (self.matlab_engine, ) + job._args
logging.debug('tg pj new args:' + str(new_args))
job._args = new_args
elif len(job._kwargs) > 0:
job._kwargs['matlab_engine'] = self.matlab_engine
logging.debug('tg pj new kwargs:' + str(job._kwargs))
with self.death_penalty_class(
job.timeout or self.queue_class.DEFAULT_TIMEOUT):
rv = job.perform()
# Pickle the result in the same try-except block since we need
# to use the same exc handling when pickling fails
job._result = rv
self.set_current_job_id(None, pipeline=pipeline)
result_ttl = job.get_result_ttl(self.default_result_ttl)
if result_ttl != 0:
job.ended_at = utcnow()
job._status = JobStatus.FINISHED
job.save(pipeline=pipeline)
finished_job_registry = FinishedJobRegistry(
job.origin, self.connection)
finished_job_registry.add(job, result_ttl, pipeline)
job.cleanup(result_ttl, pipeline=pipeline)
started_job_registry.remove(job, pipeline=pipeline)
pipeline.execute()
except Exception:
job.set_status(JobStatus.FAILED, pipeline=pipeline)
started_job_registry.remove(job, pipeline=pipeline)
try:
pipeline.execute()
except Exception:
pass
self.handle_exception(job, *sys.exc_info())
return False
if rv is None:
self.log.info('Job OK')
else:
self.log.info('Job OK, result = {0!r}'.format(yellow(
text_type(rv))))
if result_ttl == 0:
self.log.info('Result discarded immediately')
elif result_ttl > 0:
self.log.info('Result is kept for {0} seconds'.format(result_ttl))
else:
self.log.warning(
'Result will never expire, clean up result key manually')
return True
def started_job_registry(self):
"""Returns this queue's StartedJobRegistry."""
from rq.registry import StartedJobRegistry
return StartedJobRegistry(queue=self, job_class=self.job_class)
if 'REDIS_SERVER' in os.environ:
redis_server = os.environ["REDIS_SERVER"]
if 'REDIS_PORT' in os.environ:
redis_port = int(os.environ["REDIS_PORT"])
if 'REDIS_PASSWORD' in os.environ:
redis_password = os.environ["REDIS_PASSWORD"]
if 'ACCESS_KEY' in os.environ:
access_key = os.environ["ACCESS_KEY"]
if 'SECRET_KEY' in os.environ:
access_key = os.environ["SECRET_KEY"]
if 'S3_URL' in os.environ:
s3_url = os.environ["S3_URL"]
redis_con = Redis(redis_server, redis_port, password=redis_password)
train_queue = Queue('train', connection=redis_con)
registry = StartedJobRegistry('train', connection=redis_con)
fregistry = FinishedJobRegistry('train', connection=redis_con)
flregistry = train_queue.failed_job_registry
ns = Namespace('train',
description='Train a vision corpus on a prepared dataset')
train_job = ns.model(
'Train_Job', {
'name':
fields.String(required=True,
description='The name of the model that will be trained',
example='project0'),
'bucket':
fields.String(required=False,
description='The s3 bucket target',
example='bucket0'),
def setUp(self):
super(TestRegistry, self).setUp()
self.registry = StartedJobRegistry(connection=self.testconn)
def setUp(self):
super(TestRegistry, self).setUp()
self.registry = StartedJobRegistry(connection=self.testconn)
class TestRegistry(RQTestCase):
def setUp(self):
super(TestRegistry, self).setUp()
self.registry = StartedJobRegistry(connection=self.testconn)
def test_init(self):
"""Registry can be instantiated with queue or name/Redis connection"""
queue = Queue('foo', connection=self.testconn)
registry = StartedJobRegistry(queue=queue)
self.assertEqual(registry.name, queue.name)
self.assertEqual(registry.connection, queue.connection)
registry = StartedJobRegistry('bar', self.testconn)
self.assertEqual(registry.name, 'bar')
self.assertEqual(registry.connection, self.testconn)
def test_key(self):
self.assertEqual(self.registry.key, 'rq:wip:default')
def test_custom_job_class(self):
registry = StartedJobRegistry(job_class=CustomJob)
self.assertFalse(registry.job_class == self.registry.job_class)
def test_contains(self):
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
self.assertFalse(job in registry)
self.assertFalse(job.id in registry)
registry.add(job, 5)
self.assertTrue(job in registry)
self.assertTrue(job.id in registry)
def test_add_and_remove(self):
"""Adding and removing job to StartedJobRegistry."""
timestamp = current_timestamp()
job = Job()
# Test that job is added with the right score
self.registry.add(job, 1000)
self.assertLess(self.testconn.zscore(self.registry.key, job.id),
timestamp + 1002)
# Ensure that a timeout of -1 results in a score of inf
self.registry.add(job, -1)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), float('inf'))
# Ensure that job is properly removed from sorted set
self.registry.remove(job)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
def test_get_job_ids(self):
"""Getting job ids from StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': timestamp + 10})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 20})
self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar'])
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, {'foo': 1})
self.testconn.zadd(self.registry.key, {'bar': timestamp + 10})
self.testconn.zadd(self.registry.key, {'baz': timestamp + 30})
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20),
['foo', 'bar'])
def test_cleanup_moves_jobs_to_failed_job_registry(self):
"""Moving expired jobs to FailedJobRegistry."""
queue = Queue(connection=self.testconn)
failed_job_registry = FailedJobRegistry(connection=self.testconn)
job = queue.enqueue(say_hello)
self.testconn.zadd(self.registry.key, {job.id: 2})
# Job has not been moved to FailedJobRegistry
self.registry.cleanup(1)
self.assertNotIn(job, failed_job_registry)
self.assertIn(job, self.registry)
self.registry.cleanup()
self.assertIn(job.id, failed_job_registry)
self.assertNotIn(job, self.registry)
job.refresh()
self.assertEqual(job.get_status(), JobStatus.FAILED)
def test_job_execution(self):
"""Job is removed from StartedJobRegistry after execution."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
self.assertTrue(job.is_queued)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
self.assertTrue(job.is_started)
worker.perform_job(job, queue)
self.assertNotIn(job.id, registry.get_job_ids())
self.assertTrue(job.is_finished)
# Job that fails
job = queue.enqueue(div_by_zero)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job, queue)
self.assertNotIn(job.id, registry.get_job_ids())
def test_job_deletion(self):
"""Ensure job is removed from StartedJobRegistry when deleted."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
self.assertTrue(job.is_queued)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
job.delete()
self.assertNotIn(job.id, registry.get_job_ids())
def test_get_job_count(self):
"""StartedJobRegistry returns the right number of job count."""
timestamp = current_timestamp() + 10
self.testconn.zadd(self.registry.key, {'foo': timestamp})
self.testconn.zadd(self.registry.key, {'bar': timestamp})
self.assertEqual(self.registry.count, 2)
self.assertEqual(len(self.registry), 2)
def test_clean_registries(self):
"""clean_registries() cleans Started and Finished job registries."""
queue = Queue(connection=self.testconn)
finished_job_registry = FinishedJobRegistry(connection=self.testconn)
self.testconn.zadd(finished_job_registry.key, {'foo': 1})
started_job_registry = StartedJobRegistry(connection=self.testconn)
self.testconn.zadd(started_job_registry.key, {'foo': 1})
failed_job_registry = FailedJobRegistry(connection=self.testconn)
self.testconn.zadd(failed_job_registry.key, {'foo': 1})
clean_registries(queue)
self.assertEqual(self.testconn.zcard(finished_job_registry.key), 0)
self.assertEqual(self.testconn.zcard(started_job_registry.key), 0)
self.assertEqual(self.testconn.zcard(failed_job_registry.key), 0)
def test_get_queue(self):
"""registry.get_queue() returns the right Queue object."""
registry = StartedJobRegistry(connection=self.testconn)
self.assertEqual(registry.get_queue(), Queue(connection=self.testconn))
registry = StartedJobRegistry('foo', connection=self.testconn)
self.assertEqual(registry.get_queue(),
Queue('foo', connection=self.testconn))
def serialize_queues(instance_number, queues):
if scheduler_is_here:
result_list = [
dict(
name=q.name,
count=q.count,
queued_url=url_for(
".jobs_overview",
instance_number=instance_number,
queue_name=q.name,
registry_name="queued",
per_page="8",
page="1",
),
failed_job_registry_count=FailedJobRegistry(q.name).count,
failed_url=url_for(
".jobs_overview",
instance_number=instance_number,
queue_name=q.name,
registry_name="failed",
per_page="8",
page="1",
),
started_job_registry_count=StartedJobRegistry(q.name).count,
started_url=url_for(
".jobs_overview",
instance_number=instance_number,
queue_name=q.name,
registry_name="started",
per_page="8",
page="1",
),
deferred_job_registry_count=DeferredJobRegistry(q.name).count,
deferred_url=url_for(
".jobs_overview",
instance_number=instance_number,
queue_name=q.name,
registry_name="deferred",
per_page="8",
page="1",
),
finished_job_registry_count=FinishedJobRegistry(q.name).count,
finished_url=url_for(
".jobs_overview",
instance_number=instance_number,
queue_name=q.name,
registry_name="finished",
per_page="8",
page="1",
),
scheduled_job_registry_count=ScheduledJobRegistry(q.name).count,
scheduled_url=url_for(
".jobs_overview",
instance_number=instance_number,
queue_name=q.name,
registry_name="scheduled",
per_page="8",
page="1",
),
)
for q in queues
]
else:
result_list = [
dict(
name=q.name,
count=q.count,
queued_url=url_for(
".jobs_overview",
instance_number=instance_number,
queue_name=q.name,
registry_name="queued",
per_page="8",
page="1",
),
failed_job_registry_count=FailedJobRegistry(q.name).count,
failed_url=url_for(
".jobs_overview",
instance_number=instance_number,
queue_name=q.name,
registry_name="failed",
per_page="8",
page="1",
),
started_job_registry_count=StartedJobRegistry(q.name).count,
started_url=url_for(
".jobs_overview",
instance_number=instance_number,
queue_name=q.name,
registry_name="started",
per_page="8",
page="1",
),
deferred_job_registry_count=DeferredJobRegistry(q.name).count,
deferred_url=url_for(
".jobs_overview",
instance_number=instance_number,
queue_name=q.name,
registry_name="deferred",
per_page="8",
page="1",
),
finished_job_registry_count=FinishedJobRegistry(q.name).count,
finished_url=url_for(
".jobs_overview",
instance_number=instance_number,
queue_name=q.name,
registry_name="finished",
per_page="8",
page="1",
),
)
for q in queues
]
return result_list
class TestRegistry(RQTestCase):
def setUp(self):
super(TestRegistry, self).setUp()
self.registry = StartedJobRegistry(connection=self.testconn)
def test_add_and_remove(self):
"""Adding and removing job to StartedJobRegistry."""
timestamp = current_timestamp()
job = Job()
# Test that job is added with the right score
self.registry.add(job, 1000)
self.assertLess(self.testconn.zscore(self.registry.key, job.id),
timestamp + 1002)
# Ensure that a timeout of -1 results in a score of -1
self.registry.add(job, -1)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), -1)
# Ensure that job is properly removed from sorted set
self.registry.remove(job)
self.assertIsNone(self.testconn.zscore(self.registry.key, job.id))
def test_get_job_ids(self):
"""Getting job ids from StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, timestamp + 10, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 20, 'bar')
self.assertEqual(self.registry.get_job_ids(), ['foo', 'bar'])
def test_get_expired_job_ids(self):
"""Getting expired job ids form StartedJobRegistry."""
timestamp = current_timestamp()
self.testconn.zadd(self.registry.key, 1, 'foo')
self.testconn.zadd(self.registry.key, timestamp + 10, 'bar')
self.testconn.zadd(self.registry.key, timestamp + 30, 'baz')
self.assertEqual(self.registry.get_expired_job_ids(), ['foo'])
self.assertEqual(self.registry.get_expired_job_ids(timestamp + 20),
['foo', 'bar'])
def test_cleanup(self):
"""Moving expired jobs to FailedQueue."""
failed_queue = FailedQueue(connection=self.testconn)
self.assertTrue(failed_queue.is_empty())
queue = Queue(connection=self.testconn)
job = queue.enqueue(say_hello)
self.testconn.zadd(self.registry.key, 2, job.id)
self.registry.cleanup(1)
self.assertNotIn(job.id, failed_queue.job_ids)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), 2)
self.registry.cleanup()
self.assertIn(job.id, failed_queue.job_ids)
self.assertEqual(self.testconn.zscore(self.registry.key, job.id), None)
job.refresh()
self.assertEqual(job.status, JobStatus.FAILED)
def test_job_execution(self):
"""Job is removed from StartedJobRegistry after execution."""
registry = StartedJobRegistry(connection=self.testconn)
queue = Queue(connection=self.testconn)
worker = Worker([queue])
job = queue.enqueue(say_hello)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job)
self.assertNotIn(job.id, registry.get_job_ids())
# Job that fails
job = queue.enqueue(div_by_zero)
worker.prepare_job_execution(job)
self.assertIn(job.id, registry.get_job_ids())
worker.perform_job(job)
self.assertNotIn(job.id, registry.get_job_ids())
def test_get_job_count(self):
"""StartedJobRegistry returns the right number of job count."""
timestamp = current_timestamp() + 10
self.testconn.zadd(self.registry.key, timestamp, 'foo')
self.testconn.zadd(self.registry.key, timestamp, 'bar')
self.assertEqual(self.registry.count, 2)
self.assertEqual(len(self.registry), 2)
def test_clean_registries(self):
"""clean_registries() cleans Started and Finished job registries."""
queue = Queue(connection=self.testconn)
finished_job_registry = FinishedJobRegistry(connection=self.testconn)
self.testconn.zadd(finished_job_registry.key, 1, 'foo')
started_job_registry = StartedJobRegistry(connection=self.testconn)
self.testconn.zadd(started_job_registry.key, 1, 'foo')
clean_registries(queue)
self.assertEqual(self.testconn.zcard(finished_job_registry.key), 0)
self.assertEqual(self.testconn.zcard(started_job_registry.key), 0)
def getmetas():
with Connection(redis.from_url(current_app.config['REDIS_URL'])):
q = Queue('default')
registry = StartedJobRegistry('default')
f_registry = FinishedJobRegistry('default')
all_task_ids = getalltasksID()
# if request.method == 'GET':
response_text = all_task_ids.get_data(as_text=True)
response_json = all_task_ids.get_json()
# data = json.load(response_json)
running_tasks_ids = response_json["running"]["running_tasks_ids"]
finished_tasks_ids = response_json["finished"]["finished_tasks_ids"]
queued_tasks_ids = response_json["queued"]["queued_tasks_ids"]
data = {}
data['running_tasks'] = []
for task_id in running_tasks_ids:
d = {}
job = q.fetch_job(task_id)
job.refresh()
job.meta['result'] = 'null'
d[task_id] = job.meta
data['running_tasks'].append(d)
data['queued_tasks'] = []
for task_id in queued_tasks_ids:
d = {}
job = q.fetch_job(task_id)
job.refresh()
job.meta['result'] = 'null'
d[task_id] = job.meta
data['queued_tasks'].append(d)
data['finished_tasks'] = []
for task_id in finished_tasks_ids:
d = {}
job = q.fetch_job(task_id)
job.refresh()
job.meta['result'] = job.result
d[task_id] = job.meta
data['finished_tasks'].append(d)
with Connection(redis.from_url(current_app.config['REDIS_URL'])):
q = Queue('aggregator')
agg_fin_task_ids = get_all_finished_tasks_from('aggregator')
temp = agg_fin_task_ids.get_json()
agg_fin_tasks_ids = temp["finished"]["finished_tasks_ids"]
agg_que_task_ids = get_all_queued_tasks_from('aggregator')
temp = agg_que_task_ids.get_json()
agg_que_tasks_ids = temp["queued"]["queued_tasks_ids"]
agg_run_task_ids = get_all_running_tasks_from('aggregator')
temp = agg_run_task_ids.get_json()
agg_run_tasks_ids = temp["running"]["running_tasks_ids"]
data['agg_finished_tasks'] = []
for task_id in agg_fin_tasks_ids:
d = {}
job = q.fetch_job(task_id)
if job is not None:
job.refresh()
job.meta['result'] = job.result
d[task_id] = job.meta
data['agg_finished_tasks'].append(d)
data['agg_queued_tasks'] = []
for task_id in agg_que_tasks_ids:
d = {}
job = q.fetch_job(task_id)
if job is not None:
job.refresh()
job.meta['result'] = job.result
d[task_id] = job.meta
data['agg_queued_tasks'].append(d)
data['agg_running_tasks'] = []
for task_id in agg_run_tasks_ids:
d = {}
job = q.fetch_job(task_id)
if job is not None:
job.refresh()
job.meta['result'] = job.result
d[task_id] = job.meta
data['agg_running_tasks'].append(d)
return jsonify(data)
def count_builds(self, queue):
return queue.count + len(StartedJobRegistry(queue=queue))
from rq.job import Job
from rq.registry import StartedJobRegistry
from webapp.apis import query
from datetime import datetime
from functools import partial
import pandas as pd
jobs_api = Blueprint('jobs_api', __name__, url_prefix='/api/jobs')
redis_connection = Redis(host='redis', port=6379)
q = Queue('matching', connection=redis_connection)
registry = StartedJobRegistry('matching', connection=redis_connection)
@jobs_api.route('/get_current_jobs', methods=['GET'])
@login_required
def get_current_jobs():
queued_job_ids = q.job_ids
queued_jobs = q.jobs
q_time = [job.enqueued_at for job in queued_jobs]
q_ids = [job.meta['upload_id'] for job in queued_jobs]
try:
current_job_id = registry.get_job_ids()
current_job_created_at = [Job.fetch(job_id, connection=redis_connection).created_at for job_id in current_job_id]
current_job_upload_id = [Job.fetch(job_id, connection=redis_connection).meta['upload_id'] for job_id in current_job_id]
current_job = [
{
'job_id': job_id,
