def ret_results(task_id):
"""
Given a task id, return the dictionary result of all urls one level down
The AsyncResult object returned looks like the following:
Format: Async.groupresults[].Async.result.nextresult.groupresults[].Async.result
Args:
task_id: task_id of rabbitmq task
Returns:
List of Result objects of top level urls and all sub urls combined
"""
# initiate the results dictionary
ret_list = []
# list of result tuples to be processed
group_list = []
res = AsyncResult(task_id, app=app_celery)
if not res:
raise TaskNotFoundException("Task id %s not found or has expired." % task_id)
# remove the async wrapper
if res.ready():
group_res = res.get()
else:
raise TaskNotStartedException("Task id %s has not started. Please try again later." % task_id)
# process top urls and place them in results list
for group in group_res:
comb_res = process_top_urls(group)
ret_list.append(comb_res)
if comb_res.next_results:
group_list.append((comb_res, comb_res.next_results))
# if there are results to process
while group_list:
# res_tup is a (Result, AsyncResult)
res_tup = group_list.pop()
# comb_res represents the top level url result to be returned
comb_res = res_tup[0]
# res is AsyncResult
res = res_tup[1]
res = AsyncResult(res.id, app=app_celery)
# remove async wrapper
if res.ready():
res = res.get()
else:
comb_res.ready = False
# if there is one group that is not ready, set the result to not ready
# disabled for now since you can't check on the group in the docker vm
#if not res.ready():
# comb_res.ready = False
# if the top level url is ready to be processed
if comb_res.ready:
# group_list is an GroupResult
for group in res:
next_results = process_inner_urls(comb_res, group)
if next_results:
group_list.append((comb_res, next_results))
# make sure all urls collected in each group is unique
for comb_res in ret_list:
comb_res.results = list(set(comb_res.results))
return ret_list
def update_clusters(request):
"""
Given a task ID as part of the ``request`` (as ``request_id``), check on the
status of a job retrieving clusters' persistent data. Return a JSON with the
following fields:
``task_id``: return the job request ID
``ready``: ``True`` if the job has completed; ``False`` otherwise
``clusters_list``: a list of clusters' persistent data (if the job
has completed) or an empty list otherwise
"""
task_id = request.POST.get('task_id', '')
result = AsyncResult(task_id)
fetching_data_text_list = ['Fetching data... please wait', 'Fetching data...',
'Still fetching data...', 'Hopefully done soon!']
fdt = fetching_data_text_list[randint(0, len(fetching_data_text_list) - 1)]
r = {'task_id': task_id,
'ready': result.ready(),
'clusters_list': [],
'wait_text': fdt,
'error': None}
if task_id == 'missing_form_data': # See `fetch_clusters` method
r['error'] = "Missing form data. Please supply the data and try again."
elif result.ready():
clusters_pd = result.get()
r['clusters_list'] = clusters_pd.get('clusters', [])
if clusters_pd.get('error', None):
r['error'] = clusters_pd['error']
return HttpResponse(simplejson.dumps(r), mimetype="application/json")
def cleanup_tasks():
in_progress = QueryTaskTracker.all(QueryTaskTracker.IN_PROGRESS_LIST)
for tracker in in_progress:
result = AsyncResult(tracker.task_id)
if result.ready():
logging.info("in progress tracker %s finished", tracker.query_hash)
_unlock(tracker.query_hash, tracker.data_source_id)
tracker.update(state='finished')
# Maintain constant size of the finished tasks list:
removed = 1000
while removed > 0:
removed = QueryTaskTracker.prune(QueryTaskTracker.DONE_LIST, 1000)
waiting = QueryTaskTracker.all(QueryTaskTracker.WAITING_LIST)
for tracker in waiting:
if tracker is None:
continue
result = AsyncResult(tracker.task_id)
if result.ready():
logging.info("waiting tracker %s finished", tracker.query_hash)
_unlock(tracker.query_hash, tracker.data_source_id)
tracker.update(state='finished')
def twitter(**kwargs):
queries = fetch_queries()
if getattr(settings, 'KRAL_TWITTER_FIREHOSE', False) is not True:
for query in queries:
if '_' in query:
queries.append(query.replace('_',''))
cache_name = "facebookstream_%s" % query
if cache.get(cache_name):
previous_result = AsyncResult(cache.get(cache_name))
if previous_result.ready():
result = twitter_feed.delay(query)
cache.set(cache_name,result.task_id)
else:
result = twitter_feed.delay(query)
cache.set(cache_name,result.task_id)
if cache.get('twitterfeed'):
previous_queries = pickle.loads(cache.get('twitterfeed_queries'))
previous_result = AsyncResult(cache.get('twitterfeed'))
if previous_result.ready():
result = twitter_stream.delay(queries)
cache.set('twitterfeed',result.task_id)
if queries != previous_queries:
result = twitter_stream.delay(queries)
previous_result.revoke()
cache.set('twitterfeed_queries',pickle.dumps(queries))
cache.set('twitterfeed',result.task_id)
else:
result = twitter_stream.delay(queries)
cache.set('twitterfeed_queries',pickle.dumps(queries))
cache.set('twitterfeed',result.task_id)
return
def cleanup_tasks():
in_progress = QueryTaskTracker.all(QueryTaskTracker.IN_PROGRESS_LIST)
for tracker in in_progress:
result = AsyncResult(tracker.task_id)
# If the AsyncResult status is PENDING it means there is no celery task object for this tracker, and we can
# mark it as "dead":
if result.status == 'PENDING':
logging.info("In progress tracker for %s is no longer enqueued, cancelling (task: %s).",
tracker.query_hash, tracker.task_id)
_unlock(tracker.query_hash, tracker.data_source_id)
tracker.update(state='cancelled')
if result.ready():
logging.info("in progress tracker %s finished", tracker.query_hash)
_unlock(tracker.query_hash, tracker.data_source_id)
tracker.update(state='finished')
waiting = QueryTaskTracker.all(QueryTaskTracker.WAITING_LIST)
for tracker in waiting:
result = AsyncResult(tracker.task_id)
if result.ready():
logging.info("waiting tracker %s finished", tracker.query_hash)
_unlock(tracker.query_hash, tracker.data_source_id)
tracker.update(state='finished')
# Maintain constant size of the finished tasks list:
QueryTaskTracker.prune(QueryTaskTracker.DONE_LIST, 1000)
def update_clusters(request):
"""
Given a task ID as part of the ``request`` (as ``request_id``), check on the
status of a job retrieving clusters' persistent data. Return a JSON with the
following fields:
``task_id``: return the job request ID
``ready``: ``True`` if the job has completed; ``False`` otherwise
``clusters_list``: a list of clusters' persistent data (if the job
has completed) or an empty list otherwise
"""
task_id = request.POST.get("task_id", "")
result = AsyncResult(task_id)
fetching_data_text_list = [
"Fetching data... please wait",
"Fetching data...",
"Still fetching data...",
"Hopefully done soon!",
]
fdt = fetching_data_text_list[randint(0, len(fetching_data_text_list) - 1)]
r = {"task_id": task_id, "ready": result.ready(), "clusters_list": [], "wait_text": fdt, "error": None}
if result.ready():
clusters_pd = result.get()
r["clusters_list"] = clusters_pd.get("clusters", [])
if clusters_pd.get("error", None):
r["error"] = clusters_pd["error"]
return HttpResponse(simplejson.dumps(r), mimetype="application/json")
def get_result(self, request):
from celery.result import AsyncResult
res = AsyncResult(request.GET.get('async_id'))
if 'ajax' in self.request.GET:
data = {
'async_id': res.id,
'ready': res.ready()
}
if res.ready():
if res.successful():
smes = self.get_success_message(res.info)
if smes:
messages.success(self.request, smes)
# TODO: Do not store message if the ajax client stats that it will not redirect
# but handle the mssage itself
data.update({
'redirect': self.get_success_url(res.info),
'message': self.get_success_message(res.info)
})
else:
messages.error(self.request, self.get_error_message(res.info))
# TODO: Do not store message if the ajax client stats that it will not redirect
# but handle the mssage itself
data.update({
'redirect': self.get_error_url(),
'message': self.get_error_message(res.info)
})
return JsonResponse(data)
else:
if res.ready():
if res.successful():
return self.success(res.info)
else:
return self.error(res.info)
return render(request, 'pretixpresale/waiting.html')
def get(self, request, *args, **kwargs):
result = AsyncResult(kwargs['taskid'])
log_html = u''
if result.ready():
if result.successful():
rows, failed, remaining = result.result
log_html = []
has_error = False
for i, row in enumerate(rows):
if i:
has_error = has_error or not not row[-1]
row_tmpl = u'<tr><td>%s</td></tr>'
col_join = u'</td><td>'
else:
row_tmpl = u'<tr><th>%s</th></tr>'
col_join = u'</th><th>'
log_html.append(row_tmpl % col_join.join(escape(x) for x in row))
log_html = u'<table class="table">%s</table>' % u''.join(log_html)
if has_error:
log_html = u'<div class="alert alert-danger" role="alert">At least one row was not transferred. Please see log below for details.</div>' + log_html
else:
log_html = u'<div class="alert alert-success" role="alert">All rows successfully added.</div>' + log_html
else:
log_html = u'<div class="alert alert-danger" role="alert">%s</div>' % escape(unicode(result.result))
context_dict = {
'completed': result.ready(),
'log_html': log_html,
}
return self.render_json_response(context_dict)
def show_avg(steamid):
result = AsyncResult(steamid, app=celery)
print result
print result.ready()
if result.ready():
return render_template('avg.html', avg=result.get())
elif result.failed():
return result.traceback
else:
return render_template('processing.html')
def check_and_upload(support_upload_pk, auth_task, gen_task):
auth = AsyncResult(auth_task)
gen_csa = AsyncResult(gen_task)
if not (auth.ready() and gen_csa.ready()):
check_and_upload.retry(countdown=1)
elif auth.status == "SUCCESS" and gen_csa.status == "SUCCESS":
if auth.result:
upload_to_support(support_upload_pk)
else:
return
def get(self, request, *args, **kwargs):
"""Start or report on a cart-zipping task.
This method is expecting one of two possible query parameters:
task=[uuid]: Return information on the status of a
cart zipping task, including a path to download the zip
if it is done.
extensions[]: Start a new cart-zipping task for the requesting
user. Return a task_id, which can be used in the above query.
"""
if request.GET.get('task'):
task_id = request.GET['task']
task = AsyncResult(task_id)
if task.status == "PENDING":
return Response({'ready': task.ready(),
'status': task.status,
'progress': 0})
elif task.status == "SUCCESS":
return Response({'ready': task.ready(),
'status': "SUCCESS",
'progress': 100,
'path': task.result})
elif task.state == "FAILURE":
server_error = status.HTTP_500_INTERNAL_SERVER_ERROR
return Response({'ready': task.ready(),
'status': "FAILURE"}, status=server_error)
else:
meta = task._get_task_meta()
progress = meta.get('result', {}).get('progress', 0)
return Response({'ready': task.ready(),
'progress': progress,
'status': "PROGRESS"})
if request.GET.get('extensions[]'):
extensions = request.GET.getlist('extensions[]')
make_dirs = request.GET.get('make_dirs')
if make_dirs == 'false':
make_dirs = False
else:
make_dirs = True
cart = request.session.get("cart", {})
task_id = str(uuid.uuid4())
tasks.zip_files.apply_async(args=[cart, extensions, request.user.username, make_dirs], task_id=task_id)
return Response({"task": task_id}, status=status.HTTP_200_OK)
return Response(status=status.HTTP_200_OK)
def get(self, request, task_id, *args, **kwargs):
sync_task = AsyncResult(task_id)
data = {
'ready': sync_task.ready()
}
if sync_task.ready():
result = str(sync_task.result)
if result.strip():
data['error'] = result.strip()
else:
data['info'] = _('Completed.')
return HttpResponse(json.dumps(data), content_type='application/json')
def POST(self):
x = web.input(master_id='', project_id='', task_id='')
project = models.Project.get(id=x.project_id)
if not project:
raise web.notfound()
master = models.Master.get(id=x.master_id)
if not master:
return web.notfound()
if x.task_id:
from celery.result import AsyncResult
from metapolator.config import celery
res = AsyncResult(x.task_id, backend=celery.backend)
if res.ready():
master.task_completed = True
web.ctx.orm.commit()
return ujson.dumps({'done': True})
else:
master.task_updated = datetime.datetime.now()
web.ctx.orm.commit()
return ujson.dumps({'done': False, 'task_id': x.task_id})
master.task_completed = True
web.ctx.orm.commit()
return ujson.dumps({'done': True})
def task_result(request):
post = json.loads(request.body)
task_id = post.get("task_id")
result = AsyncResult(task_id)
if not result.ready():
return json_response(None)
return json_response(result.get(timeout=1))
def instancestate(request):
task_id = request.POST.get("task_id", None)
instance_state = request.POST.get("instance_state", "pending") # Preserve current state
state = {"task_id": None, "instance_state": instance_state} # Reset info to be sent
if task_id:
# If we have a running task, check on instance state
result = AsyncResult(task_id)
if result.ready():
state = result.get()
state["task_id"] = None # Reset but make sure it exists
else:
# If task not ready, send back the task_id
state["task_id"] = task_id
elif "ec2data" in request.session:
# We have no task ID, so start a task to get instance state
form = request.session["ec2data"]
cloud = form["cloud"]
a_key = form["access_key"]
s_key = form["secret_key"]
instance_id = form["instance_id"]
r = tasks.instance_state.delay(cloud, a_key, s_key, instance_id)
state["task_id"] = r.id
else:
state = {"instance_state": "Not available"}
return HttpResponse(simplejson.dumps(state), mimetype="application/json")
def get(self, slug):
print slug
res = AsyncResult(slug)
if res.ready():
self.render("results.html", username=self.current_user.replace('\"',''), jobid=slug)
else:
self.render("results.html", username=self.current_user.replace('\"',''), jobid='00')
def get_async_result(id, backend="celery"):
if backend == 'celery':
res = AsyncResult(id)
if res.ready():
return res.get()
raise ValueError("no result")
def state(self):
result = AsyncResult(self.task_id)
if not result.ready():
return result.state
if result.get() == 'ERROR':
return 'FAILED'
return result.state
def analytics_status(request, graph_slug):
analytics_results = dict()
analytics_request = request.GET.get("analytics_request")
analytics_executing = json.loads(analytics_request)
if request.is_ajax() and analytics_executing is not None:
for task_id in analytics_executing:
task = AsyncResult(task_id)
# We need to take into account if the task has been revoked
if task.ready() and task.status != REVOKED:
# Sometimes, the execution is faster than the revoked flag. So
# we need to control that (ValueError exception)
try:
analytic = Analytic.objects.filter(dump__graph__slug=graph_slug, task_id=task_id).latest()
analytics_results[task_id] = [
STATUS_OK,
analytic.results.url,
analytic.id,
analytic.task_start,
analytic.algorithm,
analytic.values.url,
]
except ValueError:
analytics_results[task_id] = [REVOKED]
elif task.status == REVOKED:
analytics_results[task_id] = [REVOKED]
data = analytics_results
json_data = json.dumps(data)
return HttpResponse(json_data, content_type="application/json")
def getTaskResult(self, task_id):
async_result = AsyncResult(task_id)
can_forget = async_result.ready()
try:
try:
result = async_result.result
if isinstance(result, Exception):
result_cls_name = result.__class__.__name__
try:
errno = ERRNO_NS[result_cls_name]
except KeyError:
LOGGER.error('Undefined errno: %s', result_cls_name)
raise spec.ServerError()
value = [errno, result.message]
else:
value = result
except Exception as exc:
LOGGER.exception(exc)
raise spec.ServerError()
status = getattr(spec.ResultStatus, async_result.status)
return spec.AsyncResult(status=status, value=json.dumps(value))
finally:
if can_forget:
async_result.forget()
LOGGER.info('Forgot the result of task %s', task_id)
def task_list(request):
from celery.result import AsyncResult
runs = CeleryRun.objects.filter(hide=False)
if runs:
header = get_header(runs)
header.append('started')
header.append('created')
header.remove('hide')
else:
header = []
#assert False, header
druns = runs.values()
data = []
for drun in druns:
name = drun['name']
result = AsyncResult(name)
drun['started'] = result.ready()
#assert False,
data.append([drun[h] for h in header])
return render_to_response('celery_task_list.html', {
'data': data,
'header': header,
'title': 'tasks',
})
def check_status(task_id):
task = AsyncResult(task_id, app=tasks.celery)
if task.ready():
if task.successful(): # Task finished successfully
status, result_value = task.result
if status: # Valid result
app.logger.info("result: %s", pprint.pformat(result_value))
return flask.jsonify(result_value)
else: # Handled exception
response = flask.make_response(result_value, 400)
response.mimetype = 'text/plain'
return response
else: # Unhandled exception
exc = task.result
response = flask.make_response(
traceback.format_exception_only(type(exc), exc),
500)
response.mimetype = 'text/plain'
return response
else: # if task.ready()
status = {
'result_id': task.id,
'state': 'PENDING',
'progress': 0,
}
if task.state == 'PROGRESS':
status.update(task.result)
response = flask.jsonify(status)
response.status_code = 202
return response
def taskReady(jobObj, redirect="error"):
"""Checks if celery task is ready.
Args:
celeryID: the id of a celery task.
redirect: page to redirect to on error.
Returns:
True,None: celery task finished successfully.
False, HttpResponseRedirect: celerytask failed.
False,False: celery task is still processing.
"""
task = AsyncResult(jobObj.celeryUID)
if task.ready():
if task.successful():
return True, None
else:
return False, HttpResponseRedirect(reverse(redirect))
# In the case the task isn't ready, we don't really have a clear idea why.
# It is possible the task has left the celery task table, and there for is marked as Pending,
# but it is really done. We store an alternate flag in our own tables to manage this.
if jobObj.state == symTyperTask.DONE:
return True, None
elif jobObj.state == symTyperTask.ERROR :
return False, HttpResponseRedirect(reverse(redirect))
else:
return False, None
def instancestate(request):
"""
Give a POST request with ``task_id`` and ``instance_state`` fields, return
JSON with updated value (given the task has completed or the same as provided)
value for the ``instance_state`` field and the same value for the ``task_id``.
``task_id`` is to correspond to the ID of the background task.
If instance state is not available, return ``Not available`` as the value
for ``instance_state``.
"""
task_id = request.POST.get('task_id', None)
instance_state = request.POST.get('instance_state', 'pending') # Preserve current state
state = {'task_id': None, 'instance_state': instance_state, 'error': ''} # Reset info to be sent
if task_id:
# If we have a running task, check on instance state
result = AsyncResult(task_id)
if result.ready():
state = result.get()
state['task_id'] = None # Reset but make sure it exists
else:
# If task not ready, send back the task_id
state['task_id'] = task_id
elif 'ec2data' in request.session:
# We have no task ID, so start a task to get instance state
form = request.session["ec2data"]
cloud = form['cloud']
a_key = form["access_key"]
s_key = form["secret_key"]
instance_id = form["instance_id"]
r = tasks.instance_state.delay(cloud, a_key, s_key, instance_id)
state['task_id'] = r.id
else:
state = {'instance_state': 'Not available'}
return HttpResponse(simplejson.dumps(state), mimetype="application/json")
def __call__(self):
result = AsyncResult(self.request.form['task_id'])
self.request.response.setHeader('Content-Type', 'text/plain')
if result.ready():
return 'Done! Result is %s' % result.result
else:
return 'Still in progress, retry later!'
def get(self, request, uuid):
res = AsyncResult(uuid) # Reconstruct a AsyncResult object with task id.
if res.ready(): # Judge whether the task is finished,if result is ready,get it with get() method.
task_res = res.get() # Get task result.
else:
task_res = "Task not finished!"
return Response(task_res, status=status.HTTP_200_OK)
def retrieve_mbs_result(target_task_id):
logger = Task.get_logger()
r = AsyncResult(target_task_id)
sr = SimulationResult.objects.get(task_id__exact=target_task_id)
# sr = SimulationResult.objects.get(sim_id__exact=r['sim_id'])
logger.info(r)
while not r.ready():
time.sleep(0.1)
result = json.loads(r.result)
if result['exit_code'] == 0:
## success
sr.sim_id = result['sim_id']
## these are rewrite if you add log collections
sr.collections = json.dumps([
"%s_nwk" % sr.sim_id,
"%s_node" % sr.sim_id,
"%s_msg" % sr.sim_id,
"%s_usr" % sr.sim_id,
"%s_map" % sr.sim_id,
])
sr.task_progress = 100
sr.task_status = "SUCCESS"
sr.save()
else:
sr.sim_id = "NO SIM_ID (FAILED)"
sr.task_status = "FAILED"
sr.task_progress = 0
sr.save()
def instancestate(request):
"""
Given a POST request with ``task_id`` and ``instance_state`` fields, check if
the task has completed. If so, return JSON with updated value for the
``instance_state`` field and start a new task, appropriately setting the
value of ``task_id``. If the initial ``task_id`` has not completed, return
the same value for the ``task_id`` field.
"""
task_id = request.POST.get('task_id', None)
instance_state = request.POST.get('instance_state', 'pending') # Preserve current state
state = {'task_id': None, 'instance_state': instance_state, 'error': ''} # Reset info to be sent
if task_id:
# If we have a running task, check on instance state
result = AsyncResult(task_id)
if result.ready():
state = result.get()
state['task_id'] = None # Reset but make sure it exists
else:
# If task not ready, send back the task_id
state['task_id'] = task_id
elif 'ec2data' in request.session:
# We have no task ID, so start a task to get instance state
form = request.session["ec2data"]
cloud = form.get('cloud', None)
a_key = form.get("access_key", None)
s_key = form.get("secret_key", None)
instance_id = form.get("instance_id", None)
if not instance_id:
state['error'] = "Missing instance ID, cannot check the state."
r = tasks.instance_state.delay(cloud, a_key, s_key, instance_id)
state['task_id'] = r.id
else:
state = {'instance_state': 'Not available'}
return HttpResponse(simplejson.dumps(state), mimetype="application/json")
def get(self,request,uuid):
tk = AsyncResult(uuid)
if tk.ready():
res = 'Task result is:%s' % str(tk.get())
else:
res = 'Task not finished!'
return Response(res, status=status.HTTP_200_OK)
def async_results(result_id):
async_result = AsyncResult(result_id)
if not async_result.ready():
raise DSProcessUnfinished("Result with id {} is not ready.".format(result_id))
if async_result.status != TaskStates.SUCCESS:
raise DSProcessError("An error occurred during background processing.")
return async_result.result
from agam_test_celery.celery import app
from celery.result import AsyncResult
res = AsyncResult("b39b5a64-7eea-410e-b498-47ad21529265")
print(res.ready())
def job_ready_byid(id):
res = AsyncResult(id, app=cel_app)
return res.ready()
def check_status(request):
task_id = request.GET.get('task_id')
res = AsyncResult(task_id)
res.ready()
print('res=', res, res.status)
return render(request, 'status_badge.html', {'status': res.status})
def get_task(task_id):
result = AsyncResult(task_id, app=celery)
task = {'id': result.id, 'state': result.state}
if result.ready():
task.update({'info': result.info})
return {'task': task}
def concurrent(url, param_value, req_id, schema):
# time.sleep(10)
with schema_context(schema_name=schema):
skema = connection.schema_name
print(connection.schema_name)
result = current_task.request.id
tgl = datetime.datetime.now()
print('sysdate %s' % tgl)
get_data = ConcurrentList.objects.get(id=req_id)
get_data.task_id = result
get_data.actual_start_date = tgl
get_data.phase = 'S'
get_data.save()
json_acceptable_string = param_value.replace("'", "\"")
x = json.loads(json_acceptable_string)
print("http://" + schema + ".harpa.com:8000/login/")
headers1 = {'Content-type': 'application/json', 'Accept': 'text/plain'}
r1 = requests.post(
"http://" + schema + ".harpa.com:8000/login/", {
"user_name": "sysadmin@" + schema + ".harpa.com",
"password": "******"
})
# print("return API %s" % r1 )
response_data = r1.json()
print(response_data)
# # print("=================================================")
a = 'JWT ' + response_data['token']
# # print("=================================================")
# # print("=================================================")
# # print("=================================================")
print('JWT %s' % a)
# # print("=================================================")
# # print("=================================================")
# # print("=================================================")
headers = {
"Content-type": 'application/json',
'Accept': 'application/json',
'Authorization': a
}
# # print("=================================================")
# # print("=================================================")
# # print("=================================================")
#
# # print('ini headers %s'%headers)
# # print('data=json.dumps(x) %s'% json.dumps(x))
r = requests.post(url, data=json.dumps(x), headers=headers)
# # print(r.text)
response_data2 = r.json()
print('response_data2 = %s' % r)
#
# # print(result)
# print('schema name %s' % (connection.schema_name))
res = AsyncResult(result)
res.ready()
# print ('status nya %s'%res.ready())
# print ('status nya %s'% AsyncResult(current_task.request.id).state)
# print ('sysdate %s' % tgl)
return 'berhasil'
class QueryTask(object):
MAX_RETRIES = 5
# TODO: this is mapping to the old Job class statuses. Need to update the client side and remove this
STATUSES = {
'PENDING': 1,
'STARTED': 2,
'SUCCESS': 3,
'FAILURE': 4,
'REVOKED': 4
}
def __init__(self, job_id=None, async_result=None):
if async_result:
self._async_result = async_result
else:
self._async_result = AsyncResult(job_id, app=celery)
@property
def id(self):
return self._async_result.id
@classmethod
def add_task(cls, query, data_source, scheduled=False, metadata={}):
query_hash = gen_query_hash(query)
logging.info("[Manager][%s] Inserting job", query_hash)
logging.info("[Manager] Metadata: [%s]", metadata)
try_count = 0
job = None
while try_count < cls.MAX_RETRIES:
try_count += 1
pipe = redis_connection.pipeline()
try:
pipe.watch(cls._job_lock_id(query_hash, data_source.id))
job_id = pipe.get(cls._job_lock_id(query_hash, data_source.id))
if job_id:
logging.info("[Manager][%s] Found existing job: %s", query_hash, job_id)
job = cls(job_id=job_id)
if job.ready():
logging.info("[%s] job found is ready (%s), removing lock", query_hash, job.celery_status)
redis_connection.delete(QueryTask._job_lock_id(query_hash, data_source.id))
job = None
if not job:
pipe.multi()
if scheduled:
queue_name = data_source.scheduled_queue_name
else:
queue_name = data_source.queue_name
result = execute_query.apply_async(args=(query, data_source.id, metadata), queue=queue_name)
job = cls(async_result=result)
logging.info("[Manager][%s] Created new job: %s", query_hash, job.id)
pipe.set(cls._job_lock_id(query_hash, data_source.id), job.id, settings.JOB_EXPIRY_TIME)
pipe.execute()
break
except redis.WatchError:
continue
if not job:
logging.error("[Manager][%s] Failed adding job for query.", query_hash)
return job
def to_dict(self):
if self._async_result.status == 'STARTED':
updated_at = self._async_result.result.get('start_time', 0)
else:
updated_at = 0
if self._async_result.failed() and isinstance(self._async_result.result, Exception):
error = self._async_result.result.message
elif self._async_result.status == 'REVOKED':
error = 'Query execution cancelled.'
else:
error = ''
if self._async_result.successful():
query_result_id = self._async_result.result
else:
query_result_id = None
return {
'id': self._async_result.id,
'updated_at': updated_at,
'status': self.STATUSES[self._async_result.status],
'error': error,
'query_result_id': query_result_id,
}
@property
def is_cancelled(self):
return self._async_result.status == 'REVOKED'
@property
def celery_status(self):
return self._async_result.status
def ready(self):
return self._async_result.ready()
def cancel(self):
return self._async_result.revoke(terminate=True, signal='SIGINT')
@staticmethod
def _job_lock_id(query_hash, data_source_id):
return "query_hash_job:%s:%s" % (data_source_id, query_hash)
def get_job_status(job_id):
res = AsyncResult(job_id, backend=celery_object.backend)
return jsonify({"status": res.ready()})
def checkStatus(req):
task_id = req.GET.get('task_id')
if task_id:
async_result = AsyncResult(task_id)
return JsonResponse({'status': async_result.ready()})
return JsonResponse({'status': False})
class QueryTask(object):
# TODO: this is mapping to the old Job class statuses. Need to update the client side and remove this
STATUSES = {
"PENDING": 1,
"STARTED": 2,
"SUCCESS": 3,
"FAILURE": 4,
"REVOKED": 4
}
def __init__(self, job_id=None, async_result=None):
if async_result:
self._async_result = async_result
else:
self._async_result = AsyncResult(job_id, app=celery)
@property
def id(self):
return self._async_result.id
def to_dict(self):
task_info = self._async_result._get_task_meta()
result, task_status = task_info["result"], task_info["status"]
if task_status == "STARTED":
updated_at = result.get("start_time", 0)
else:
updated_at = 0
status = self.STATUSES[task_status]
if isinstance(result, (TimeLimitExceeded, SoftTimeLimitExceeded)):
error = TIMEOUT_MESSAGE
status = 4
elif isinstance(result, Exception):
error = str(result)
status = 4
elif task_status == "REVOKED":
error = "Query execution cancelled."
else:
error = ""
if task_status == "SUCCESS" and not error:
query_result_id = result
else:
query_result_id = None
return {
"id": self._async_result.id,
"updated_at": updated_at,
"status": status,
"error": error,
"query_result_id": query_result_id,
}
@property
def is_cancelled(self):
return self._async_result.status == "REVOKED"
@property
def celery_status(self):
return self._async_result.status
def ready(self):
return self._async_result.ready()
def cancel(self):
return self._async_result.revoke(terminate=True, signal="SIGINT")
def task_finished(task_id):
"""Returns whether specified task has already finished"""
result = AsyncResult(task_id)
return result.ready()
def task_result(task_id):
"""Returns result of the task. Returns None if no result yet"""
result = AsyncResult(task_id)
if not result.ready():
return None
return result.get(timeout=1)
def task_result(request, task_id):
result = AsyncResult(task_id)
if result.ready():
return HttpResponse('Result is: %s' % (result.result, ))
else:
return HttpResponse('Result is not ready yet!')
def submission_results(request, user, slug):
""" Returns existing submissions results for IO test cases. """
challenge = get_object_or_404(Challenge, slug=slug)
submission = get_object_or_404(
Submission,
challenge=challenge,
author=user,
)
if not submission.has_view_results_permission(request, for_user=user):
return HttpResponseForbidden()
trace = ""
result = None
for tr in TestResult.objects.filter(submission=submission, status="PD"):
async_result = AsyncResult(tr.task_id)
if async_result.ready():
result = async_result.result
trace = str(result)
# regular run task result is either Exception instance or dict
#if isinstance(result, Exception): # for regular run task
# run_popen check if in result string there is Exception traceback
if "Traceback" in trace:
tr.status = 'EX'
else:
# run_popen task result is json string so convert it to dict
result = json.loads(trace)
tr.status = result['status']
tr.memory = result['memory']
tr.cputime = result['cputime']
tr.result = check_output(tr.test_case.output.path,
result['outpath'])
# remove celery task meta from the database
try:
tm = TaskMeta.objects.get(task_id=tr.task_id)
except TaskMeta.DoesNotExist:
pass
else:
tm.delete()
os.remove(result['outpath'])
os.remove(result['errpath'])
tr.save()
#if there are no pending test results delete program
if len(TestResult.objects.filter(submission=submission, status="PD")) == 0:
if result is not None:
os.remove(result['program'])
test_results = TestResult.objects.filter(submission=submission)
trs = [{
'status': dict(TestResult.STATUSES)[tr.status],
'status_code': tr.status.lower(),
'result': dict(TestResult.RESULTS)[tr.result],
'result_code': tr.result.lower(),
'memory': tr.memory,
'cputime': tr.cputime,
'test_case': tr.test_case,
} for tr in test_results]
if (len(test_results) > 0 and len(test_results.filter(status="PD")) == 0
and submission.status != "TS"):
submission.status = "TS" # Tested
submission.save()
return render_to_response(
"onlinejudge/partials/submission_results.html",
{
"submission": submission,
"submission_result_code": submission.result.lower(),
"submission_result": dict(Submission.RESULTS)[submission.result],
"test_results": trs,
"trace": trace,
},
RequestContext(request),
)
import sys
from celery.result import AsyncResult
from tasks import add, pdf_site
if __name__ == "__main__":
if len(sys.argv) != 3:
raise Exception('Call with 3 arguments!')
if sys.argv[1].lower() == 'download':
site = sys.argv[2].lower()
result = pdf_site.delay(site, 'downloads/{}.pdf'.format(site.replace('.','_')))
print('PDF for {} will be generated. Check with this UUID:{}'.format(site, result.task_id))
elif sys.argv[1].lower() == 'check':
task_id = sys.argv[2].lower()
res = AsyncResult(task_id)
if res.ready():
print('PDF for {} is located in download/{}.pdf'.format(res.result, res.result.replace('.','_')))
else:
print('Still not ready')
def read_experiment_search_results(channel_name, view, request, user):
channel = Channel(channel_name)
start = 0
end = None
compendium = CompendiumDatabase.objects.get(id=request['compendium_id'])
task_running = False
operation = 'search_experiment_public_db'
if request['page_size']:
start = (request['page'] - 1) * request['page_size']
end = start + request['page_size']
try:
task_id = ViewTask.objects.using(compendium.compendium_nick_name).get(view=request['view'],
operation=operation)
task = AsyncResult(task_id.task_id)
task_running = not task.ready()
except Exception as e:
pass
order = ''
if request['ordering'] == 'DESC':
order = '-'
query_response = ExperimentSearchResult.objects.using(compendium.compendium_nick_name). \
filter(Q(organism__icontains=request['filter']) |
Q(experiment_access_id__icontains=request['filter']) |
Q(experiment_alternative_access_id__icontains=request['filter']) |
Q(platform__icontains=request['filter']) |
Q(scientific_paper_ref__icontains=request['filter']) |
Q(type__icontains=request['filter']) |
Q(description__icontains=request['filter']) |
Q(experiment_name__icontains=request['filter'])).order_by(
order + request['ordering_value'])
total = query_response.count()
query_response = query_response[start:end]
experiments = []
for exp in query_response:
e = exp.to_dict()
module_name, class_name = '.'.join(exp.data_source.python_class.split('.')[:-1]), \
exp.data_source.python_class.split('.')[-1]
python_class = getattr(importlib.import_module(module_name), class_name)()
e['experiment_accession_base_link'] = python_class.experiment_accession_base_link
e['platform_accession_base_link'] = python_class.platform_accession_base_link
e['scientific_paper_accession_base_link'] = python_class.scientific_paper_accession_base_link
e['tag'] = ''
if exp.platform:
for plt_acc_id in exp.platform.split(';'):
try:
plt = Platform.objects.using(compendium.compendium_nick_name).\
get(platform_access_id=plt_acc_id)
if plt.biofeaturereporter_set.count() > 0:
e['tag'] = 'platform'
except Exception as exc:
pass
try:
already_present_exp = Experiment.objects.using(compendium.compendium_nick_name).\
get(experiment_access_id=exp.experiment_access_id)
if already_present_exp.status.name == 'experiment_excluded':
e['tag'] = 'excluded'
else:
e['tag'] = 'present'
if e['status']['name'] != 'scheduled' and e['status']['name'] != 'experiment_downloading':
e['status'] = already_present_exp.status.to_dict()
except Exception as exc:
pass
experiments.append(e)
channel.send({
'text': json.dumps({
'stream': view,
'payload': {
'request': request,
'data': {
'experiments': experiments,
'task_running': task_running,
'total': total
}
}
})
})
def check(request, task_id):
task = AsyncResult(task_id)
return JsonResponse(
{"questions_count": task.get() if task.ready() else None})
def get(self, uuid):
result = AsyncResult(id=uuid, app=celery_app)
return {
"finished": result.ready(),
"status": result.state,
}
'Flo ne spune buna dimineata',
'Mitza flexeaza muschiul',
'Geo tureaza Bavarian Motor Works',
]
print("---- SENDER ----")
try:
while True:
message = random.sample(MESSAGES, 1)[0]
print(f"Sending: \"{message}\"")
response = app.send_task('bla.some', (message, ))
# what is the id
print(f"Task ID: {response.id}")
response = AsyncResult(id=response.id, app=app)
for i in range(5):
if response.ready():
result = response.get(timeout=1)
print(f"Result is {result}")
break
time.sleep(0.5)
time.sleep(1)
except KeyboardInterrupt:
print("Warm interrupt of the sender")
def testdestination(self, destinationid):
"Test mail destination server"
server = self._get_server(destinationid)
if not server:
abort(404)
taskid = request.GET.get('taskid', None)
if not taskid:
to_addr = 'postmaster@%s' % server.domains.name
task = test_smtp_server.apply_async(args=[
server.address, server.port, '<>', to_addr, server.id, 3
])
taskid = task.task_id
if 'taskids' not in session:
session['taskids'] = []
session['taskids'].append(taskid)
session['testdest-count'] = 1
session.save()
redirect(url.current(taskid=taskid))
else:
result = AsyncResult(taskid)
if result is None or taskid not in session['taskids']:
flash(_('The connection test failed try again later'))
redirect(url('domain-detail', domainid=server.domain_id))
if result.ready():
if ('smtp' in result.result and 'ping' in result.result
and result.result['smtp'] and result.result['ping']):
msg = _('The server: %s is up and accepting mail from us' %
server.address)
flash(msg)
log.info(msg)
else:
if 'ping' in result.result['errors']:
errors = result.result['errors']['ping']
else:
errors = result.result['errors']['smtp']
msg = _('The server: %s is not accepting mail from us: %s') \
% (server.address, errors)
flash(msg)
log.info(msg)
redirect(url('domain-detail', domainid=server.domain_id))
else:
try:
session['testdest-count'] += 1
except KeyError:
session['testdest-count'] = 1
session.save()
if (session['testdest-count'] >= 10
and result.state in ['PENDING', 'RETRY', 'FAILURE']):
result.revoke()
del session['testdest-count']
session.save()
msg = 'Failed to initialize backend, try again later'
flash_alert(msg)
log.info(msg)
redirect(url('domain-detail', domainid=server.domain_id))
c.server = server
c.domainid = server.domain_id
c.taskid = taskid
c.finished = False
return self.render('/domains/testdestination.html')
def download_gradebook(req, teacher, results=None):
"""
Download the wanted gradebook.
Parameters
----------
req : HttpRequest
Request with:
parameters:
task_id: str
Id of the celery task responsible for the gradebook
generation sent with the first request for a gradebook
teacher : Teacher
Teacher instance returned by `teacher_required` (not used)
results : Optional[Dict[str, Any]]
Either
If group gradebook
{
group: str
Title of the group
assignments: List[str]
Assignment identifier
school_id_needed: bool
If a school id is needed
results: [{
school_id: Optional[str]
School id if needed
email: str
Student email
assignments: [{
n_completed: Optional[int]
Number of completed questions
n_correct: Optional[int]
Number of correct questions
}]
}]
}
If assignment gradebook
{
group: str
Title of the group
assignment: str
Title of the assignment
questions: List[str]
Question title
school_id_needed: bool
If a school id is needed
results: [{
school_id: Optional[str]
School id if needed
email: str
Student email
questions: List[Optional[float]]
Grade for each question
}]
}
Returns
-------
StreamingHttpResponse
csv file with the gradebook results
"""
if results is None:
args = get_json_params(req, args=["task_id"])
if isinstance(args, HttpResponse):
return args
(task_id, ), _ = args
result = AsyncResult(task_id)
try:
if not result.ready():
return response_400(
req,
msg="The gradebook isn't ready.",
logger_msg="Not completed gradebook {}".format(task_id) +
" accessed by teacher {}".format(teacher.user.username),
)
except AttributeError:
return response_500(
req,
msg="There is no gradebook corresponding to this url. "
"Please ask for a new one.",
logger_msg="Celery error getting gradebook"
" for teacher {}".format(teacher.user.username) +
" and task {}.".format(task_id),
log=logger.warning,
use_template=False,
)
results = result.result
if RunningTask.objects.filter(id=task_id):
RunningTask.objects.get(id=task_id).delete()
if "assignment" in results:
filename = "myDALITE_gradebook_{}_{}.csv".format(
results["group"], results["assignment"])
else:
filename = "myDALITE_gradebook_{}.csv".format(results["group"])
gradebook_gen = convert_gradebook_to_csv(results)
data = chain(iter((filename + "\n", )), gradebook_gen)
resp = StreamingHttpResponse(data, content_type="text/csv")
return resp
def configure_course(request, course_key=None, course=None):
context = {
"hierarchy":
((settings.APP_NAME, reverse("index")),
(course.name, reverse("course",
kwargs={"course_key":
course.key})), ("Configure", None)),
"course":
course,
"provider_data": [
{
"description":
"{:s}, all submission data are retrieved from here".format(
course.provider_name),
"path":
settings.PROVIDERS[course.provider].get("host", "UNKNOWN"),
},
{
"description":
"Data providers should POST the IDs of new submissions to this path in order to have them automatically downloaded by Radar",
"path":
request.build_absolute_uri(
reverse("hook_submission",
kwargs={"course_key": course.key})),
},
{
"description":
"Login requests using the LTI-protocol should be made to this path",
"path": request.build_absolute_uri(reverse("lti_login")),
},
],
"errors": [],
}
# The state of the API read task is contained in this dict
pending_api_read = {
"task_id": None,
"poll_URL": reverse("configure_course",
kwargs={"course_key": course.key}),
"ready": False,
"poll_interval_seconds": 5,
"config_type": "automatic"
}
if request.method == "GET":
if "true" in request.GET.get("success", ''):
# All done, show success message
context["change_success"] = True
pending_api_read["json"] = json.dumps(pending_api_read)
context["pending_api_read"] = pending_api_read
return render(request, "review/configure.html", context)
if request.method != "POST":
return HttpResponseBadRequest()
p_config = provider_config(course.provider)
if "create-exercises" in request.POST or "overwrite-exercises" in request.POST:
# API data has been fetched in a previous step, now the user wants to add exercises that were shown in the table
if "create-exercises" in request.POST:
# Pre-configured, read-only table
exercises = json.loads(request.POST["exercises-json"])
for exercise_data in exercises:
key_str = str(exercise_data["exercise_key"])
exercise = course.get_exercise(key_str)
exercise.set_from_config(exercise_data)
exercise.save()
# Queue fetch and match for all submissions for this exercise
full_reload = configured_function(p_config, "full_reload")
full_reload(exercise, p_config)
elif "overwrite-exercises" in request.POST:
# Manual configuration, editable table, overwrite existing
checked_rows = (key.split("-", 1)[0] for key in request.POST
if key.endswith("enabled"))
exercises = ({
"exercise_key":
exercise_key,
"name":
request.POST[exercise_key + "-name"],
"template_source":
request.POST.get(exercise_key + "-template-source", ''),
"tokenizer":
request.POST[exercise_key + "-tokenizer"],
"minimum_match_tokens":
request.POST[exercise_key + "-min-match-tokens"]
} for exercise_key in checked_rows)
for exercise_data in exercises:
key = str(exercise_data["exercise_key"])
course.exercises.filter(key=key).delete()
exercise = course.get_exercise(key)
exercise.set_from_config(exercise_data)
exercise.save()
full_reload = configured_function(p_config, "full_reload")
full_reload(exercise, p_config)
return redirect(
reverse("configure_course", kwargs={"course_key": course.key}) +
"?success=true")
if not request.is_ajax():
return HttpResponseBadRequest("Unknown POST request")
pending_api_read = json.loads(request.body.decode("utf-8"))
if pending_api_read["task_id"]:
# Task is pending, check state and return result if ready
async_result = AsyncResult(pending_api_read["task_id"])
if async_result.ready():
pending_api_read["ready"] = True
pending_api_read["task_id"] = None
if async_result.state == "SUCCESS":
exercise_data = async_result.get()
async_result.forget()
config_table = template_loader.get_template(
"review/configure_table.html")
exercise_data["config_type"] = pending_api_read["config_type"]
pending_api_read["resultHTML"] = config_table.render(
exercise_data, request)
else:
pending_api_read["resultHTML"] = ''
return JsonResponse(pending_api_read)
if pending_api_read["ready"]:
# The client might be polling a few times even after it has received the results
return JsonResponse(pending_api_read)
# Put full read of provider API on task queue and store the task id for tracking
has_radar_config = pending_api_read["config_type"] == "automatic"
async_api_read = configured_function(p_config, "async_api_read")
pending_api_read["task_id"] = async_api_read(request, course,
has_radar_config)
return JsonResponse(pending_api_read)
def __poll_tasks(self, project):
status = {}
task_ongoing = False
if not project in self.messages:
return status, task_ongoing
for key in self.messages[project].keys():
job = self.messages[project][key]
msg = self.celery_app.backend.get_task_meta(key)
if not len(msg):
continue
# check for worker failures
if msg['status'] == celery.states.FAILURE:
# append failure message
if 'meta' in msg: #TODO: and isinstance(msg['meta'], BaseException):
info = {'message': html.escape(str(msg['meta']))}
else:
info = {'message': 'an unknown error occurred'}
else:
info = msg['result']
status[key] = {
'type': job['type'],
'submitted':
job['submitted'], #TODO: not broadcast across AIController threads...
'status': msg['status'],
'meta': info
}
if 'subjobs' in job:
subjobEntries = []
for subjob in job['subjobs']:
if isinstance(subjob, GroupResult):
entry = {'id': subjob.id}
subEntries = []
for res in subjob.results:
subEntry = {
'id':
res.id,
'status':
res.status,
'meta': ('complete' if res.status == 'SUCCESS'
else str(res.result)) #TODO
}
subEntries.append(subEntry)
entry['subjobs'] = subEntries
else:
entry = {
'id':
subjob.id,
'status':
subjob.status,
'meta': ('complete' if subjob.status == 'SUCCESS'
else str(subjob.result)) #TODO
}
subjobEntries.append(entry)
status[key]['subjobs'] = subjobEntries
# check if ongoing
result = AsyncResult(key)
if result.ready(): #TODO: chains somehow get stuck in 'PENDING'...
# done; remove from queue
result.forget()
status[key]['status'] = 'SUCCESS'
elif result.failed():
# failed
result.forget()
status[key]['status'] = 'FAILURE'
else:
task_ongoing = True
return status, task_ongoing
class QueryTask(object):
# TODO: this is mapping to the old Job class statuses. Need to update the client side and remove this
STATUSES = {
'PENDING': 1,
'STARTED': 2,
'SUCCESS': 3,
'FAILURE': 4,
'REVOKED': 4
}
def __init__(self, job_id=None, async_result=None):
if async_result:
self._async_result = async_result
else:
self._async_result = AsyncResult(job_id, app=celery)
@property
def id(self):
return self._async_result.id
def to_dict(self):
task_info = self._async_result._get_task_meta()
result, task_status = task_info['result'], task_info['status']
if task_status == 'STARTED':
updated_at = result.get('start_time', 0)
else:
updated_at = 0
status = self.STATUSES[task_status]
if isinstance(result, (TimeLimitExceeded, SoftTimeLimitExceeded)):
error = "Query exceeded Redash query execution time limit."
status = 4
elif isinstance(result, Exception):
error = result.message
status = 4
elif task_status == 'REVOKED':
error = 'Query execution cancelled.'
else:
error = ''
if task_status == 'SUCCESS' and not error:
query_result_id = result
else:
query_result_id = None
return {
'id': self._async_result.id,
'updated_at': updated_at,
'status': status,
'error': error,
'query_result_id': query_result_id,
}
@property
def is_cancelled(self):
return self._async_result.status == 'REVOKED'
@property
def celery_status(self):
return self._async_result.status
def ready(self):
return self._async_result.ready()
def cancel(self):
return self._async_result.revoke(terminate=True, signal='SIGINT')
def requeue_subsequent_tasks(self, task_id):
result = AsyncResult(task_id)
if not result.ready():
self.retry()
result.maybe_reraise()
return result.get()
class CombineBackgroundTask(models.Model):
'''
Model for long running, background tasks
'''
name = models.CharField(max_length=255, null=True, default=None)
task_type = models.CharField(
max_length=255,
choices=[('job_delete', 'Job Deletion'),
('record_group_delete', 'Record Group Deletion'),
('org_delete', 'Organization Deletion'),
('validation_report', 'Validation Report Generation'),
('export_mapped_fields', 'Export Mapped Fields'),
('export_documents', 'Export Documents'),
('job_reindex', 'Job Reindex Records'),
('job_new_validations', 'Job New Validations'),
('job_remove_validation', 'Job Remove Validation')],
default=None,
null=True)
celery_task_id = models.CharField(max_length=128, null=True, default=None)
celery_task_output = models.TextField(null=True, default=None)
task_params_json = models.TextField(null=True, default=None)
task_output_json = models.TextField(null=True, default=None)
start_timestamp = models.DateTimeField(null=True, auto_now_add=True)
finish_timestamp = models.DateTimeField(null=True,
default=None,
auto_now_add=False)
completed = models.BooleanField(default=False)
def __str__(self):
return 'CombineBackgroundTask: %s, ID #%s, Celery Task ID #%s' % (
self.name, self.id, self.celery_task_id)
def update(self):
'''
Method to update completed status, and affix task to instance
'''
# get async task from Redis
try:
self.celery_task = AsyncResult(self.celery_task_id)
self.celery_status = self.celery_task.status
if not self.completed:
# if ready (finished)
if self.celery_task.ready():
# set completed
self.completed = True
# update timestamp
self.finish_timestamp = datetime.datetime.now()
# handle result type
if isinstance(self.celery_task.result, Exception):
result = str(self.celery_task.result)
else:
result = self.celery_task.result
# save json of async task output
task_output = {
'result': result,
'status': self.celery_task.status,
'task_id': self.celery_task.task_id,
'traceback': self.celery_task.traceback
}
self.celery_task_output = json.dumps(task_output)
# save
self.save()
except Exception as err:
self.celery_task = None
self.celery_status = 'STOPPED'
self.completed = True
self.save()
LOGGER.debug(str(err))
def calc_elapsed_as_string(self):
# determine time elapsed in seconds
# completed, with timestamp
if self.completed and self.finish_timestamp != None:
# use finish timestamp
seconds_elapsed = (
self.finish_timestamp.replace(tzinfo=None) -
self.start_timestamp.replace(tzinfo=None)).seconds
# marked as completed, but not timestamp, set to zero
elif self.completed:
seconds_elapsed = 0
# else, calc until now
else:
seconds_elapsed = (
datetime.datetime.now() -
self.start_timestamp.replace(tzinfo=None)).seconds
# return as string
minutes, seconds = divmod(seconds_elapsed, 60)
hours, minutes = divmod(minutes, 60)
return "%d:%02d:%02d" % (hours, minutes, seconds)
@property
def task_params(self):
'''
Property to return JSON params as dict
'''
if self.task_params_json:
return json.loads(self.task_params_json)
return {}
def update_task_params(self, update_d, save=True):
'''
Method to update tasks params
Args:
update_d (dict): Dictionary to update self.task_params with
Returns:
None
'''
# update json
task_params = self.task_params
task_params.update(update_d)
self.task_params_json = json.dumps(task_params)
# save
if save:
self.save()
@property
def task_output(self):
'''
Property to return JSON output as dict
'''
if self.task_output_json:
return json.loads(self.task_output_json)
return {}
def cancel(self):
'''
Method to cancel background task
'''
# attempt to stop any Spark jobs
if 'job_id' in self.task_params.keys():
job_id = self.task_params['job_id']
LOGGER.debug('attempt to kill spark jobs related to Job: %s',
job_id)
job = mod_job.Job.objects.get(pk=int(job_id))
job.stop_job(cancel_livy_statement=False, kill_spark_jobs=True)
# revoke celery task
if self.celery_task_id:
revoke(self.celery_task_id, terminate=True)
# update status
self.refresh_from_db()
self.completed = True
self.save()
@classmethod
def to_rerun_jobs(cls, job_ids):
task = cls(name="Rerun Jobs Prep",
task_type='rerun_jobs_prep',
task_params_json=json.dumps(
{'ordered_job_rerun_set': job_ids}))
task.save()
return task
def getTasksInfo(tasks, forgetIfFinished=True):
if tasks is None:
return None, False, None
if isinstance(tasks, str):
tasks = json.loads(tasks)
errors = []
for t in range(len(tasks)):
result = AsyncResult(tasks[t]['id'])
if result.ready():
tasks[t]['successful'] = result.successful()
if tasks[t]['successful']:
tasks[t]['info'] = None
else:
try:
error = str(result.get())
errors.append(error)
except Exception as e:
error = str(e)
errors.append(error)
tasks[t]['info'] = {}
tasks[t]['info']['message'] = error
if forgetIfFinished:
result.forget()
elif result.info is not None:
tasks[t]['info'] = result.info
if result.status is not None:
tasks[t]['status'] = result.status
if 'children' in tasks[t]:
numDone = 0
numChildren = len(tasks[t]['children'])
for key in range(numChildren):
cResult = AsyncResult(tasks[t]['children'][key]['id'])
if cResult.ready():
numDone += 1
tasks[t]['children'][key][
'successful'] = cResult.successful()
if tasks[t]['children'][key]['successful']:
tasks[t]['children'][key]['info'] = None
else:
try:
error = str(cResult.get())
errors.append(error)
except Exception as e:
error = str(e)
errors.append(error)
tasks[t]['children'][key]['info'] = {}
tasks[t]['children'][key]['info'][
'message'] = error
if forgetIfFinished:
cResult.forget()
elif cResult.info is not None:
tasks[t]['children'][key]['info'] = cResult.info
if cResult.status is not None:
tasks[t]['children'][key]['status'] = cResult.status
tasks[t]['num_done'] = numDone
if numDone == numChildren:
tasks[t]['status'] = 'SUCCESSFUL'
lastResult = AsyncResult(tasks[-1]['id'])
hasFinished = lastResult.ready()
return tasks, hasFinished, errors
def get_task_status(task_id):
result = AsyncResult(task_id)
return result.ready()
def launch_status(request):
"""
Given a task ID of a launch process/task, check if the task has completed.
Return a JSON object with the following keys: ``task_id``, ``ready``,
``error``, and ``starting_text``.
"""
task_id = request.session.get('ec2data', {}).get('task_id')
r = {
'task_id': '',
'ready': '',
'error': '',
'starting_text': '',
'instance_id': '',
'sg_name': '',
'kp_name': ''
}
if task_id:
r['task_id'] = task_id
result = AsyncResult(task_id)
r['ready'] = result.ready()
if r['ready']: # The task completed; let's get the outcome
# Set session data based on the task result
# TODO: this should always return JSON and not mess with the session
# Then, need to redo how monitor page is displayed...
response = result.get()
if response.get("error", ""):
# If a key got created, allow the user to download it even if
# an error was encountered.
if response.get('kp_material'):
request.session['ec2data']['kp_name'] = response['kp_name']
request.session['ec2data']['kp_material'] = response[
'kp_material']
r['kp_name'] = response['kp_name']
r['kp_material'] = response['kp_material']
r['error'] = response['error']
else:
request.session['ec2data']['cluster_name'] = response[
'cluster_name']
request.session['ec2data']['instance_id'] = response[
'instance_id']
request.session['ec2data']['public_ip'] = response[
'instance_ip']
request.session['ec2data']['image_id'] = response['image_id']
request.session['ec2data']['kp_name'] = response['kp_name']
request.session['ec2data']['kp_material'] = response[
'kp_material']
request.session['ec2data']['sg_name'] = response['sg_names'][0]
request.session['ec2data']['password'] = response['password']
# Pass data needed for the additional instance information table
# on the monitor page
r['instance_id'] = response['instance_id']
r['sg_name'] = response['sg_names'][0]
r['kp_name'] = response['kp_name']
r['kp_material'] = response.get('kp_material', '')
r['image_id'] = response['image_id']
# Add an entry to the Usage table now
try:
cluster_type = response['cluster_type']
storage_type = response['storage_type']
storage_size = response['storage_size']
# Depending on the cluster type, set null values vs.
# form defaults
if cluster_type == 'None':
cluster_type = None
storage_type = None
storage_size = None
elif response['storage_type'] == 'transient':
storage_size = None
u = models.Usage(cloud_name=response["cloud_name"],
cloud_type=response["cloud_type"],
image_id=response['image_id'],
instance_type=response['instance_type'],
cluster_type=cluster_type,
storage_type=storage_type,
storage_size=storage_size,
user_id=response["access_key"],
email=response.get(
'institutional_email', ''))
u.save()
except Exception, e:
log.debug("Trouble saving Usage data: {0}".format(e))
else:
starting_text_list = [
'Starting an instance... please wait', 'Really starting!',
'Still starting.', 'Hopefully done soon!'
]
st = starting_text_list[randint(0, len(starting_text_list) - 1)]
r['starting_text'] = st
