def ret_results(task_id):
"""
Given a task id, return the dictionary result of all urls one level down
The AsyncResult object returned looks like the following:
Format: Async.groupresults[].Async.result.nextresult.groupresults[].Async.result
Args:
task_id: task_id of rabbitmq task
Returns:
List of Result objects of top level urls and all sub urls combined
"""
# initiate the results dictionary
ret_list = []
# list of result tuples to be processed
group_list = []
res = AsyncResult(task_id, app=app_celery)
if not res:
raise TaskNotFoundException("Task id %s not found or has expired." % task_id)
# remove the async wrapper
if res.ready():
group_res = res.get()
else:
raise TaskNotStartedException("Task id %s has not started. Please try again later." % task_id)
# process top urls and place them in results list
for group in group_res:
comb_res = process_top_urls(group)
ret_list.append(comb_res)
if comb_res.next_results:
group_list.append((comb_res, comb_res.next_results))
# if there are results to process
while group_list:
# res_tup is a (Result, AsyncResult)
res_tup = group_list.pop()
# comb_res represents the top level url result to be returned
comb_res = res_tup[0]
# res is AsyncResult
res = res_tup[1]
res = AsyncResult(res.id, app=app_celery)
# remove async wrapper
if res.ready():
res = res.get()
else:
comb_res.ready = False
# if there is one group that is not ready, set the result to not ready
# disabled for now since you can't check on the group in the docker vm
#if not res.ready():
# comb_res.ready = False
# if the top level url is ready to be processed
if comb_res.ready:
# group_list is an GroupResult
for group in res:
next_results = process_inner_urls(comb_res, group)
if next_results:
group_list.append((comb_res, next_results))
# make sure all urls collected in each group is unique
for comb_res in ret_list:
comb_res.results = list(set(comb_res.results))
return ret_list
def wait_for_task(celery_task):
# Normally celery would die with "Never call result.get() within a task!"
# See http://docs.celeryq.org/en/latest/userguide/tasks.html#task-synchronous-subtasks
with celery.result.allow_join_result():
#logging.info("*** Wait_for_task: %s", celery_task)
result = AsyncResult(celery_task)
result.get(timeout=MINUTE_SECS * 5)
def main(rowid):
id = models.ServiceDeployStatus.objects.get(id=rowid).service_id
configinfo = models.ServiceDeployConfig.objects.get(id=id)
hostinfo = models.ServiceManager.objects.filter(
servicename=configinfo.service.servicename).values_list(
'inhost_id__hostname', 'number')
tmplist = [
configinfo.service.servicename,
configinfo.image.imagename,
configinfo.project.projectname,
]
data = []
for item in hostinfo:
tmplist1 = tmplist[:]
tmplist1.append(item[0])
tmplist1.append(item[1])
data.append(tmplist1)
res = tasks.deploy.delay(
json.dumps(data)) # 同一个服务中的多个docker容器需要串行执行，灰度发布模式
taskid = res.id
while True:
res = AsyncResult(id=taskid) # 在任务完成之前会阻塞
print(res.get())
if res.get() == 0:
models.ServiceDeployStatus.objects.filter(id=rowid).update(
status=False)
print('false')
return 0
else:
return 1
def task_queue_handler(id_task, from_queue):
if from_queue == "celery":
res = AsyncResult(id_task)
status = {
"PENDING": 0,
"STARTED": 1,
"RETRY": 1,
"SUCCESS": 2,
"FAILURE": 3
}
result = {"status": int(status.get(res.status)), "ml_units": 0}
elif from_queue == "ai-platform":
res = get_train_job_status(id_task)
status = {
"QUEUED": 0,
"PREPARING": 0,
"RUNNING": 1,
"SUCCEEDED": 2,
"FAILED": 3,
"CANCELLING": 3,
"CANCELLED": 3,
"STATE_UNSPECIFIED": 3,
}
result = {
"status": int(status.get(res.get("state"))),
"ml_units": res.get("trainingOutput",
{}).get("consumedMLUnits", 0),
}
else:
raise ValidationError("Invalid queue")
return result
def get_task(id_):
"""
Get the result or status of a single task.
:param id_: id of the task
"""
# lazy load the task app to avoid circular imports
from ibutsu_server.tasks.queues import app
async_result = AsyncResult(id_, app=app)
response = {"state": async_result.state}
if async_result.state == "SUCCESS":
response["message"] = "Task has succeeded"
result = async_result.get()
if result:
response.update(async_result.get())
elif async_result.state == "PENDING":
response["message"] = "Task not yet started or invalid, check back later"
elif async_result.state == "STARTED":
response["message"] = "Task has started but is still running, check back later"
elif async_result.state == "RETRY":
response["message"] = "Task has been retried, possibly due to failure"
else:
response["message"] = "Task has failed!"
response["error"] = async_result.traceback.split("\n")
return response, _STATE_TO_CODE.get(async_result.state)
def get(self, request, task_id, format=None):
if task_id:
result = AsyncResult(task_id)
if result.state == 'SUCCESS':
print(result.get('file', ''))
return Response({
"status": "Success",
"result": {
"state": result.state,
"done": result.info,
"result": result.get(),
"file": "{}{}".format(settings.MEDIA_URL, result.get())
}
})
return Response({
"status": "Success",
"result": {
"state": result.state,
"done": result.info
}
})
return Response({
"status": "Error",
"message": "Task id is not provided."
})
def getTasksInfo(tasks, forgetIfFinished=True):
if tasks is None:
return None, False, None
if isinstance(tasks, str):
tasks = json.loads(tasks)
errors = []
for t in range(len(tasks)):
result = AsyncResult(tasks[t]['id'])
if result.ready():
tasks[t]['successful'] = result.successful()
if tasks[t]['successful']:
tasks[t]['info'] = None
else:
try:
error = str(result.get())
errors.append(error)
except Exception as e:
error = str(e)
errors.append(error)
tasks[t]['info'] = {}
tasks[t]['info']['message'] = error
if forgetIfFinished:
result.forget()
elif result.info is not None:
tasks[t]['info'] = result.info
if result.status is not None:
tasks[t]['status'] = result.status
if 'children' in tasks[t]:
numDone = 0
for key in tasks[t]['children']:
cResult = AsyncResult(tasks[t]['children'][key]['id'])
if cResult.ready():
numDone += 1
tasks[t]['children'][key][
'successful'] = cResult.successful()
if tasks[t]['children'][key]['successful']:
tasks[t]['children'][key]['info'] = None
else:
try:
error = str(cResult.get())
errors.append(error)
except Exception as e:
error = str(e)
errors.append(error)
tasks[t]['children'][key]['info'] = {}
tasks[t]['children'][key]['info'][
'message'] = error
if forgetIfFinished:
cResult.forget()
elif cResult.info is not None:
tasks[t]['children'][key]['info'] = cResult.info
if cResult.status is not None:
tasks[t]['children'][key]['status'] = cResult.status
tasks[t]['num_done'] = numDone
lastResult = AsyncResult(tasks[-1]['id'])
hasFinished = lastResult.ready()
return tasks, hasFinished, errors
def test_get_sync_subtask_option(self, task_join_will_block):
task_join_will_block.return_value = True
tid = uuid()
backend = _MockBackend()
res_subtask_async = AsyncResult(tid, backend=backend)
with pytest.raises(RuntimeError):
res_subtask_async.get()
res_subtask_async.get(disable_sync_subtasks=False)
def test_get(self):
ok_res = AsyncResult(self.task1["id"])
ok2_res = AsyncResult(self.task2["id"])
nok_res = AsyncResult(self.task3["id"])
self.assertEquals(ok_res.get(), "the")
self.assertEquals(ok2_res.get(), "quick")
self.assertRaises(KeyError, nok_res.get)
def test_get_sync_subtask_option(self, task_join_will_block):
task_join_will_block.return_value = True
tid = uuid()
backend = _MockBackend()
res_subtask_async = AsyncResult(tid, backend=backend)
with pytest.raises(RuntimeError):
res_subtask_async.get()
res_subtask_async.get(disable_sync_subtasks=False)
def test_get_timeout(self):
res = AsyncResult(self.task4['id']) # has RETRY state
with self.assertRaises(TimeoutError):
res.get(timeout=0.1)
pending_res = AsyncResult(uuid())
with self.assertRaises(TimeoutError):
pending_res.get(timeout=0.1)
def test_get_timeout(self):
res = AsyncResult(self.task4['id']) # has RETRY state
with self.assertRaises(TimeoutError):
res.get(timeout=0.1)
pending_res = AsyncResult(uuid())
with self.assertRaises(TimeoutError):
pending_res.get(timeout=0.1)
def test_get(self):
ok_res = AsyncResult(self.task1["id"])
ok2_res = AsyncResult(self.task2["id"])
nok_res = AsyncResult(self.task3["id"])
nok2_res = AsyncResult(self.task4["id"])
self.assertEqual(ok_res.get(), "the")
self.assertEqual(ok2_res.get(), "quick")
self.assertRaises(KeyError, nok_res.get)
self.assertIsInstance(nok2_res.result, KeyError)
def test_drain_events_decodes_exceptions_in_meta(self):
tid = uuid()
b = self.create_backend(serializer="json")
b.store_result(tid, RuntimeError("aap"), states.FAILURE)
result = AsyncResult(tid, backend=b)
with self.assertRaises(Exception) as cm:
result.get()
self.assertEqual(cm.exception.__class__.__name__, "RuntimeError")
self.assertEqual(str(cm.exception), "aap")
def test_drain_events_decodes_exceptions_in_meta(self):
tid = uuid()
b = self.create_backend(serializer='json')
b.store_result(tid, RuntimeError('aap'), states.FAILURE)
result = AsyncResult(tid, backend=b)
with pytest.raises(Exception) as excinfo:
result.get()
assert excinfo.value.__class__.__name__ == 'RuntimeError'
assert str(excinfo.value) == 'aap'
def test_drain_events_decodes_exceptions_in_meta(self):
tid = uuid()
b = self.create_backend(serializer="json")
b.store_result(tid, RuntimeError("aap"), states.FAILURE)
result = AsyncResult(tid, backend=b)
with self.assertRaises(Exception) as cm:
result.get()
self.assertEqual(cm.exception.__class__.__name__, "RuntimeError")
self.assertEqual(str(cm.exception), "aap")
def test_get(self):
ok_res = AsyncResult(self.task1["id"])
ok2_res = AsyncResult(self.task2["id"])
nok_res = AsyncResult(self.task3["id"])
nok2_res = AsyncResult(self.task4["id"])
self.assertEqual(ok_res.get(), "the")
self.assertEqual(ok2_res.get(), "quick")
self.assertRaises(KeyError, nok_res.get)
self.assertIsInstance(nok2_res.result, KeyError)
self.assertEqual(ok_res.info, "the")
def test_drain_events_decodes_exceptions_in_meta(self):
tid = uuid()
b = self.create_backend(serializer='json')
b.store_result(tid, RuntimeError('aap'), states.FAILURE)
result = AsyncResult(tid, backend=b)
with pytest.raises(Exception) as excinfo:
result.get()
assert excinfo.value.__class__.__name__ == 'RuntimeError'
assert str(excinfo.value) == 'aap'
def test_get(self):
ok_res = AsyncResult(self.task1["id"])
ok2_res = AsyncResult(self.task2["id"])
nok_res = AsyncResult(self.task3["id"])
nok2_res = AsyncResult(self.task4["id"])
self.assertEqual(ok_res.get(), "the")
self.assertEqual(ok2_res.get(), "quick")
with self.assertRaises(KeyError):
nok_res.get()
self.assertTrue(nok_res.get(propagate=False))
self.assertIsInstance(nok2_res.result, KeyError)
self.assertEqual(ok_res.info, "the")
def test_get(self):
ok_res = AsyncResult(self.task1["id"])
ok2_res = AsyncResult(self.task2["id"])
nok_res = AsyncResult(self.task3["id"])
nok2_res = AsyncResult(self.task4["id"])
self.assertEqual(ok_res.get(), "the")
self.assertEqual(ok2_res.get(), "quick")
with self.assertRaises(KeyError):
nok_res.get()
self.assertTrue(nok_res.get(propagate=False))
self.assertIsInstance(nok2_res.result, KeyError)
self.assertEqual(ok_res.info, "the")
def consolidate_metadata(self, wait=True):
"""
Tries to find an abstract for the paper, if none is available yet,
possibly by fetching it from Zotero via doi-cache.
"""
if self.task is None:
task = send_task('consolidate_paper', [], {'pk':self.id})
self.task = task.id
self.save(update_fields=['task'])
else:
task = AsyncResult(self.task)
if wait:
task.get()
def consolidate_metadata(self, wait=True):
"""
Tries to find an abstract for the paper, if none is available yet,
possibly by fetching it from Zotero via doi-cache.
"""
if self.task is None:
task = send_task('consolidate_paper', [], {'pk': self.id})
self.task = task.id
self.save(update_fields=['task'])
else:
task = AsyncResult(self.task)
if wait:
task.get()
def test_get(self):
ok_res = AsyncResult(self.task1['id'])
ok2_res = AsyncResult(self.task2['id'])
nok_res = AsyncResult(self.task3['id'])
nok2_res = AsyncResult(self.task4['id'])
self.assertEqual(ok_res.get(), 'the')
self.assertEqual(ok2_res.get(), 'quick')
with self.assertRaises(KeyError):
nok_res.get()
self.assertTrue(nok_res.get(propagate=False))
self.assertIsInstance(nok2_res.result, KeyError)
self.assertEqual(ok_res.info, 'the')
def test_get(self):
ok_res = AsyncResult(self.task1['id'])
ok2_res = AsyncResult(self.task2['id'])
nok_res = AsyncResult(self.task3['id'])
nok2_res = AsyncResult(self.task4['id'])
self.assertEqual(ok_res.get(), 'the')
self.assertEqual(ok2_res.get(), 'quick')
with self.assertRaises(KeyError):
nok_res.get()
self.assertTrue(nok_res.get(propagate=False))
self.assertIsInstance(nok2_res.result, KeyError)
self.assertEqual(ok_res.info, 'the')
def head(self, request, *args, **kwargs):
self.query, self.last_item, self.first_item = self.get_query(request)
result = cache.get(self.get_cache_key())
if result is not None and "task" in result:
search_task = AsyncResult(result["task"])
try:
search_task.get(self.timeout)
except exceptions.TimeoutError:
return http.HttpResponse(status=202) # 202: still waiting for task
return http.HttpResponse(status=201) # 201: search results ready
elif result is not None:
return http.HttpResponse(status=201) # 201: search results ready
else:
return http.HttpResponseBadRequest() # 400: no search is being performed
def get(self, request, *args, **kw):
# these are not really necessary, the important is the id
name = kw['service_name']
tname = kw['task_name']
id = kw['job_id']
#f= open(os.path.dirname(__file__)+PROCESSING_SETTINGS_URL)
#parsed_json = json.load(f)
#query result backend for details
try:
result = AsyncResult(id=id)
state = result.state
if state.lower() in ["pending", "started"]:
s = {"jobId": id, "jobStatus": state.lower(), "content": []}
elif state.lower() == "revoked":
s = {"jobId": id, "jobStatus": "revoked", "content": []}
elif state.lower() == "retry":
s = {
"jobId": id,
"jobStatus": "retry",
"content": result.get()
}
elif state.lower() == "failure":
s = {
"jobId": id,
"jobStatus": "failure",
"content": result.get()
}
elif state.lower() == "success":
s = {
"jobId": id,
"jobStatus": "success",
"content": result.get()
}
else:
s = {
"jobId": id,
"jobStatus": "unknown",
"content": result.get()
}
return Response(s)
except Exception as e: #this will intercept the error raised by pysdss in the form ["error","error info here"]
s = {"jobId": id, "jobStatus": 'failure', "content": eval(str(e))}
return Response(s)
def retrieve(self, request, pk=None):
res = AsyncResult(pk)
if res.state == 'SUCCESS':
content = {'state': res.state, 'result': res.get()}
return Response(content, status=status.HTTP_200_OK)
elif res.state == 'FAILURE':
try:
res.get()
except Exception as e:
content = {'state': res.state, 'cause': str(e)}
return Response(content, status=status.HTTP_400_BAD_REQUEST)
else:
content = {'state': res.state}
return Response(content, status=status.HTTP_409_CONFLICT)
def head(self, request, *args, **kwargs):
self.query, self.last_item, self.first_item = self.get_query(request)
result = cache.get(self.get_cache_key())
if result is not None and "task" in result:
search_task = AsyncResult(result["task"])
try:
search_task.get(self.timeout)
except exceptions.TimeoutError:
return http.HttpResponse(status=202) # 202: still waiting for task
return http.HttpResponse(status=201) # 201: search results ready
elif result is not None:
return http.HttpResponse(status=201) # 201: search results ready
else:
return http.HttpResponseBadRequest() # 400: no search is being performed
def get(self, taskid):
"""
Get a task result
**Example request**:
.. sourcecode:: http
GET /api/task/result/c60be250-fe52-48df-befb-ac66174076e6 HTTP/1.1
Host: localhost:5555
**Example response**:
.. sourcecode:: http
HTTP/1.1 200 OK
Content-Length: 84
Content-Type: application/json; charset=UTF-8
{
"result": 3,
"state": "SUCCESS",
"task-id": "c60be250-fe52-48df-befb-ac66174076e6"
}
:query timeout: how long to wait, in seconds, before the operation times out
:reqheader Authorization: optional OAuth token to authenticate
:statuscode 200: no error
:statuscode 401: unauthorized request
:statuscode 503: result backend is not configured
"""
timeout = self.get_argument('timeout', None)
timeout = float(timeout) if timeout is not None else None
result = AsyncResult(taskid)
if not self.backend_configured(result):
raise HTTPError(503)
response = {
'task-id': taskid,
'state': result.state,
'result': result.result
}
if timeout:
result.get(timeout=timeout, propagate=False)
self.update_response_result(response, result)
elif result.ready():
self.update_response_result(response, result)
self.write(response)
def test_task_exception(swh_scheduler_celery_app, swh_scheduler_celery_worker,
swh_scheduler):
task_type = swh_scheduler.get_task_type("swh-test-error")
assert task_type
assert task_type["backend_name"] == TASK_ERROR
swh_scheduler.create_tasks([create_task_dict("swh-test-error", "oneshot")])
backend_tasks = run_ready_tasks(swh_scheduler, swh_scheduler_celery_app)
assert len(backend_tasks) == 1
task = backend_tasks[0]
result = AsyncResult(id=task["backend_id"])
with pytest.raises(NotImplementedError):
result.get()
def instancestate(request):
"""
Give a POST request with ``task_id`` and ``instance_state`` fields, return
JSON with updated value (given the task has completed or the same as provided)
value for the ``instance_state`` field and the same value for the ``task_id``.
``task_id`` is to correspond to the ID of the background task.
If instance state is not available, return ``Not available`` as the value
for ``instance_state``.
"""
task_id = request.POST.get('task_id', None)
instance_state = request.POST.get('instance_state', 'pending') # Preserve current state
state = {'task_id': None, 'instance_state': instance_state, 'error': ''} # Reset info to be sent
if task_id:
# If we have a running task, check on instance state
result = AsyncResult(task_id)
if result.ready():
state = result.get()
state['task_id'] = None # Reset but make sure it exists
else:
# If task not ready, send back the task_id
state['task_id'] = task_id
elif 'ec2data' in request.session:
# We have no task ID, so start a task to get instance state
form = request.session["ec2data"]
cloud = form['cloud']
a_key = form["access_key"]
s_key = form["secret_key"]
instance_id = form["instance_id"]
r = tasks.instance_state.delay(cloud, a_key, s_key, instance_id)
state['task_id'] = r.id
else:
state = {'instance_state': 'Not available'}
return HttpResponse(simplejson.dumps(state), mimetype="application/json")
def getCaseRunningResult(id):
print "called getCaseRunningResult"
# do any things here
print "thread start successfully and sleep for 5 seconds"
running = SuiteRunningModel.objects.get(pk=id)
#running = SuiteRunningModel.objects.all().order_by('-id')[0]
print running.pk
case_account = running.case_runnings.count()
print "case_account:", case_account
case_finished = 0
case_failure = 0
for item in running.case_runnings.all():
print item
print "task-id:", item.task_id
print "get-result-timeout:", item.case.timeout
res = AsyncResult(item.task_id, app=app)
tc_running = res.get(timeout=item.case.timeout, propagate=False)
print "tc_running", tc_running
item.ret_code = tc_running.ret_code
if item.ret_code <> 0:
case_failure += 1
print "hahahahahhahahahhahah", tc_running.error_msg
item.error_msg = tc_running.error_msg
item.save()
case_finished += 1
if case_failure == 0 and case_finished > 0:
running.ret = 0
print "set running result:", running.ret
else:
print "set running result:", running.ret
running.ret = 1
running.progress = case_finished * 100 / case_account
running.status = "Completed"
running.save()
def get(self,request,uuid):
tk = AsyncResult(uuid)
if tk.ready():
res = 'Task result is:%s' % str(tk.get())
else:
res = 'Task not finished!'
return Response(res, status=status.HTTP_200_OK)
def state(self):
result = AsyncResult(self.task_id)
if not result.ready():
return result.state
if result.get() == 'ERROR':
return 'FAILED'
return result.state
def status(request, task_id):
"""
Returns status of a queued task.
"""
res={}
task_log=TaskAuthentication.objects.get(task_id=task_id)
if task_log.user_id==get_user_id(request):
result=AsyncResult(task_id)
res['status']=TASK_NEW #no such tasks in queue
if result:
if(result.ready()):
if(result.successful()):
result=result.get()
if isinstance(result,dict):
res['status']=TASK_FAILED
res['error_message']=result['error_message']
else:
res['status']=TASK_SUCCESSFUL #succcessfull
else:
res['status']=TASK_RUNNING #waiting in queue
else:
logging.debug("Unauthorized User")
res['status']=TASK_UNKNOWN
response = json.dumps(res)
return HttpResponse(response, mimetype="application/json")
def update_clusters(request):
"""
Given a task ID as part of the ``request`` (as ``request_id``), check on the
status of a job retrieving clusters' persistent data. Return a JSON with the
following fields:
``task_id``: return the job request ID
``ready``: ``True`` if the job has completed; ``False`` otherwise
``clusters_list``: a list of clusters' persistent data (if the job
has completed) or an empty list otherwise
"""
task_id = request.POST.get("task_id", "")
result = AsyncResult(task_id)
fetching_data_text_list = [
"Fetching data... please wait",
"Fetching data...",
"Still fetching data...",
"Hopefully done soon!",
]
fdt = fetching_data_text_list[randint(0, len(fetching_data_text_list) - 1)]
r = {"task_id": task_id, "ready": result.ready(), "clusters_list": [], "wait_text": fdt, "error": None}
if result.ready():
clusters_pd = result.get()
r["clusters_list"] = clusters_pd.get("clusters", [])
if clusters_pd.get("error", None):
r["error"] = clusters_pd["error"]
return HttpResponse(simplejson.dumps(r), mimetype="application/json")
def error_handler(uuid):
"""Error handler function."""
result = AsyncResult(uuid)
exc = result.get(propagate=False)
logger.info("Task %s raised exception: %s\n%s" %
(uuid, exc, result.traceback))
def _get_asynchronous_payload(self):
new_task = False
# fetch the task from celery
task_id = self.async_restore_task_id_cache.get_value()
task = AsyncResult(task_id)
task_exists = task.status == ASYNC_RESTORE_SENT
if not task_exists:
# start a new task
# NOTE this starts a nested timer (wait_for_task_to_start),
# which will be stopped by self.generate_payload(async_task)
# in the asynchronous task. It is expected that
# get_async_restore_payload.delay(self) will serialize this
# RestoreConfig and it's associated TimingContext before it
# returns, and thereby fork the timing context. The timing
# context associated with this side of the fork will not be
# recorded since it is async (see self.get_response).
with self.timing_context("wait_for_task_to_start"):
task = get_async_restore_payload.delay(self, self.domain, self.restore_user.username)
new_task = True
# store the task id in cache
self.async_restore_task_id_cache.set_value(task.id)
try:
response = task.get(timeout=self._get_task_timeout(new_task))
except TimeoutError:
# return a 202 with progress
response = AsyncRestoreResponse(task, self.restore_user.username)
return response
def instancestate(request):
task_id = request.POST.get("task_id", None)
instance_state = request.POST.get("instance_state", "pending") # Preserve current state
state = {"task_id": None, "instance_state": instance_state} # Reset info to be sent
if task_id:
# If we have a running task, check on instance state
result = AsyncResult(task_id)
if result.ready():
state = result.get()
state["task_id"] = None # Reset but make sure it exists
else:
# If task not ready, send back the task_id
state["task_id"] = task_id
elif "ec2data" in request.session:
# We have no task ID, so start a task to get instance state
form = request.session["ec2data"]
cloud = form["cloud"]
a_key = form["access_key"]
s_key = form["secret_key"]
instance_id = form["instance_id"]
r = tasks.instance_state.delay(cloud, a_key, s_key, instance_id)
state["task_id"] = r.id
else:
state = {"instance_state": "Not available"}
return HttpResponse(simplejson.dumps(state), mimetype="application/json")
def get(self, request, uuid):
res = AsyncResult(uuid) # Reconstruct a AsyncResult object with task id.
if res.ready(): # Judge whether the task is finished,if result is ready,get it with get() method.
task_res = res.get() # Get task result.
else:
task_res = "Task not finished!"
return Response(task_res, status=status.HTTP_200_OK)
def update_clusters(request):
"""
Given a task ID as part of the ``request`` (as ``request_id``), check on the
status of a job retrieving clusters' persistent data. Return a JSON with the
following fields:
``task_id``: return the job request ID
``ready``: ``True`` if the job has completed; ``False`` otherwise
``clusters_list``: a list of clusters' persistent data (if the job
has completed) or an empty list otherwise
"""
task_id = request.POST.get('task_id', '')
result = AsyncResult(task_id)
fetching_data_text_list = ['Fetching data... please wait', 'Fetching data...',
'Still fetching data...', 'Hopefully done soon!']
fdt = fetching_data_text_list[randint(0, len(fetching_data_text_list) - 1)]
r = {'task_id': task_id,
'ready': result.ready(),
'clusters_list': [],
'wait_text': fdt,
'error': None}
if task_id == 'missing_form_data': # See `fetch_clusters` method
r['error'] = "Missing form data. Please supply the data and try again."
elif result.ready():
clusters_pd = result.get()
r['clusters_list'] = clusters_pd.get('clusters', [])
if clusters_pd.get('error', None):
r['error'] = clusters_pd['error']
return HttpResponse(simplejson.dumps(r), mimetype="application/json")
def get(self, request, *args, **kwargs):
task_id = kwargs.pop('task_id')
async_result = AsyncResult(task_id)
try:
# Poll the task to see if the result is available
result = async_result.get(timeout=20)
except TimeoutError:
result = None
task_status = TaskStatus.objects.get(task_id=task_id)
# Strip parentheses and get the name of the function to call
task_name = re.sub(r'\([^)]*\)', '', task_status.signature)
if task_name in request.session['tasks']:
del(request.session['tasks'][task_name])
request.session.modified = True
status = task_status.status
# Setup the response
response = {'task_id': task_id, 'task_status': status, 'task_name': task_name, 'task_result': result}
return HttpResponse(json.dumps(response), content_type='application/json')
def sentiment_results_page(request):
result_id= request.session['result']# define result
result = AsyncResult(result_id, app=app)# get the result of the task
data_str = result.get()
json_data = json.loads(data_str)
print(json_data)
if request.method == 'POST':
return render(request, 'document_sentiment/sentiment_results_page.html', {'json_data':json_data})
else:
docs = Sentiment_Documents.objects.filter(author=request.user.id, document__contains=".csv")
return render(request, 'document_sentiment/sentiment_preview_data_file.html', {'docs':docs})
#########################################################################
'''
def get_async_result(id, backend="celery"):
if backend == 'celery':
res = AsyncResult(id)
if res.ready():
return res.get()
raise ValueError("no result")
def results(id):
result = AsyncResult(id, app=celery)
""" Format the final results page and return template."""
png = re.compile("png$")
txt = re.compile("triPOD_Results.txt$")
log = re.compile("triPOD_log.txt$")
bed = re.compile("bed$")
error = re.compile("FAILED.+")
command, exitstatus, stdout, stderr, timer = result.get()
stdout.replace('\n','')
outdir = command['out'].split('=')[-1]
if (exitstatus == 3) or (exitstatus == 5):
errmesg = re.search(error,stdout).group(0)
flash(u"Please check your input file: {0}".format(errmesg), 'error')
return redirect(url_for('upload'))
elif exitstatus == 4:
flash(u"No regions of abnormal parental contribution were detected at this alpha level.")
if not any([re.search('.resize.png', file) for file in os.listdir(outdir)]):
bulkResize(outdir, width=640, height=480)
images = []
for path, dirs, files in os.walk(outdir):
for file in files:
if re.search(png, file):
images.append(file)
elif re.search(txt, file):
f = open(os.path.join(outdir, file), 'r')
textresults = f.readlines()
f.close()
f = open(os.path.join(outdir, file), 'w')
for line in textresults:
f.write(line.replace(installpath, ''))
f.close()
with open(os.path.join(outdir, file), 'r') as f:
table = extract_table(f)
txtfile = file
elif re.search(bed, file):
bedfile = file
else:
continue
r = re.compile('resize.png$')
thumbnails = filter(r.search, images)
buildfile = command['build'].split('=')[-1]
build = os.path.basename(buildfile).split('_')[0]
return render_template('results.html',
id=os.path.basename(outdir),
name=os.path.basename(command['filepath']),
build=build,
txtfile=txtfile,
bedfile=bedfile,
images=reversed(thumbnails),
table=table,
tablerange=range(0,len(table['Sample']) + 1),
timer=timer)
def get_async_csv_submission_status(job_uuid):
""" Gets CSV Submision progress or result
Can be used to pol long running submissions
:param str job_uuid: The submission job uuid returned by _submit_csv.delay
:return: Dict with import progress info (insertions & total)
:rtype: Dict
"""
if not job_uuid:
return async_status(FAILED, u'Empty job uuid')
job = AsyncResult(job_uuid)
try:
# result = (job.result or job.state)
if job.state not in ['SUCCESS', 'FAILURE']:
response = async_status(celery_state_to_status(job.state))
response.update(job.info)
return response
if job.state == 'FAILURE':
return async_status(celery_state_to_status(job.state),
text(job.result))
except BacklogLimitExceeded:
return async_status(celery_state_to_status('PENDING'))
return job.get()
def get_context_data(self, **kwargs):
context = super(TaskResultView, self).get_context_data(**kwargs)
task_id = kwargs['task_id']
async_res = AsyncResult(task_id)
context['result'] = async_res.get()
context['task_id'] = task_id
return context
def task_result(request):
post = json.loads(request.body)
task_id = post.get("task_id")
result = AsyncResult(task_id)
if not result.ready():
return json_response(None)
return json_response(result.get(timeout=1))
def onedl(dlid):
"""Get details on one download"""
res = AsyncResult(dlid, app=tasks.app)
finres = {'id': dlid,
'state': res.state,
'celery_state': res.state,
'url': downloads.get(dlid),
'result': None,
'error': None,
'links': {'self': url_for('onedl', dlid=dlid)},
'ignoreerrors': None}
if res.ready():
#try:
fullres = res.get()
if fullres.get('error'):
finres['error'] = fullres['error']
finres['result'] = False
if finres['state'] == 'SUCCESS':
finres['state'] = 'FAILURE'
else:
finres['result'] = fullres.get('data')
if not finres.get('url') and fullres.get('url'):
finres['url'] = fullres['url']
finres['ignoreerrors'] = fullres.get('ignoreerrors')
# TODO parse and return other relevant sections...
#except YoutubeDLError as e:
# finres['result'] = False
# finres['error'] = str(e)
#if request.args.get('fmt') == 'html':
# return render_template("check.html", finres=finres)
return jsonify(finres)
def instancestate(request):
"""
Given a POST request with ``task_id`` and ``instance_state`` fields, check if
the task has completed. If so, return JSON with updated value for the
``instance_state`` field and start a new task, appropriately setting the
value of ``task_id``. If the initial ``task_id`` has not completed, return
the same value for the ``task_id`` field.
"""
task_id = request.POST.get('task_id', None)
instance_state = request.POST.get('instance_state', 'pending') # Preserve current state
state = {'task_id': None, 'instance_state': instance_state, 'error': ''} # Reset info to be sent
if task_id:
# If we have a running task, check on instance state
result = AsyncResult(task_id)
if result.ready():
state = result.get()
state['task_id'] = None # Reset but make sure it exists
else:
# If task not ready, send back the task_id
state['task_id'] = task_id
elif 'ec2data' in request.session:
# We have no task ID, so start a task to get instance state
form = request.session["ec2data"]
cloud = form.get('cloud', None)
a_key = form.get("access_key", None)
s_key = form.get("secret_key", None)
instance_id = form.get("instance_id", None)
if not instance_id:
state['error'] = "Missing instance ID, cannot check the state."
r = tasks.instance_state.delay(cloud, a_key, s_key, instance_id)
state['task_id'] = r.id
else:
state = {'instance_state': 'Not available'}
return HttpResponse(simplejson.dumps(state), mimetype="application/json")
def task_result(request):
post = json.loads(request.body)
task_id = post.get("task_id")
result = AsyncResult(task_id)
if not result.ready():
return json_response(None)
return json_response(result.get(timeout=1))
def check(self, request, *args, **kwargs):
try:
task_id = request.data['id']
uuid.UUID(task_id)
except KeyError:
raise ValidationError("You should specify id")
except ValueError:
raise ValidationError("Not valid id")
async_result = AsyncResult(task_id)
try:
result = async_result.get(timeout=0.5, propagate=False)
except TimeoutError:
result = None
status = async_result.status
if isinstance(result, Exception):
return Response({
'status': status,
'error': str(result),
},
status=HTTP_200_OK)
elif result is None:
return Response({'status': status}, status=HTTP_204_NO_CONTENT)
else:
return Response({
'status': status,
'result': result,
},
status=HTTP_200_OK)
class Progress(object):
def __init__(self, task_id):
self.task_id = task_id
self.result = AsyncResult(task_id)
def get_info(self):
if self.result.ready():
success = self.result.successful()
with allow_join_result():
return {
'complete': True,
'success': success,
'progress': _get_completed_progress(),
'result': self.result.get(self.task_id) if success else str(self.result.info),
}
elif self.result.state == PROGRESS_STATE:
return {
'complete': False,
'success': None,
'progress': self.result.info,
}
elif self.result.state in ['PENDING', 'STARTED']:
return {
'complete': False,
'success': None,
'progress': _get_unknown_progress(),
}
return self.result.info
def check(self, request, *args, **kwargs):
try:
task_id = request.data['id']
uuid.UUID(task_id)
except KeyError:
raise ValidationError("You should specify id")
except ValueError:
raise ValidationError("Not valid id")
async_result = AsyncResult(task_id)
try:
result = async_result.get(timeout=0.5, propagate=False)
except TimeoutError:
result = None
status = async_result.status
if isinstance(result, Exception):
return Response({
'status': status,
'error': str(result),
}, status=HTTP_200_OK)
elif result is None:
return Response({
'status': status
}, status=HTTP_204_NO_CONTENT)
else:
return Response({
'status': status,
'result': result,
}, status=HTTP_200_OK)
def get(self, request, *args, **kwargs):
task_id = kwargs.pop('task_id')
async_result = AsyncResult(task_id)
try:
# Poll the task to see if the result is available
result = async_result.get(timeout=20)
except TimeoutError:
result = None
task_status = TaskStatus.objects.get(task_id=task_id)
# Strip parentheses and get the name of the function to call
task_name = re.sub(r'\([^)]*\)', '', task_status.signature)
if task_name in request.session['tasks']:
del (request.session['tasks'][task_name])
request.session.modified = True
status = task_status.status
# Setup the response
response = {
'task_id': task_id,
'task_status': status,
'task_name': task_name,
'task_result': result
}
return HttpResponse(json.dumps(response),
content_type='application/json')
def launch_status(request):
"""
Given a task ID of a launch process/task, check if the task has completed.
Return a JSON object with the following keys: ``task_id``, ``ready``,
``error``, and ``starting_text``.
"""
# task_id = request.POST.get('task_id', '')
task_id = request.session["ec2data"]["task_id"]
r = {"task_id": "", "ready": "", "error": "", "starting_text": "", "instance_id": "", "sg_name": "", "kp_name": ""}
if task_id:
r["task_id"] = task_id
result = AsyncResult(task_id)
r["ready"] = result.ready()
if r["ready"]: # The task completed; let's get the outcome
# Set session data based on the task result
# TODO: this should always return JSON and not mess with the session
# Then, need to redo how monitor page is displayed...
response = result.get()
if response.get("error", ""):
r["error"] = response["error"]
else:
request.session["ec2data"]["cluster_name"] = response["cluster_name"]
request.session["ec2data"]["instance_id"] = response["instance_id"]
request.session["ec2data"]["public_ip"] = response["instance_ip"]
request.session["ec2data"]["image_id"] = response["image_id"]
request.session["ec2data"]["kp_name"] = response["kp_name"]
request.session["ec2data"]["kp_material"] = response["kp_material"]
request.session["ec2data"]["sg_name"] = response["sg_names"][0]
request.session["ec2data"]["password"] = response["password"]
# Pass data needed for the additional instance information table
# on the monitor page
r["instance_id"] = response["instance_id"]
r["sg_name"] = response["sg_names"][0]
r["kp_name"] = response["kp_name"]
r["image_id"] = response["image_id"]
# Add an entry to the Usage table now
try:
u = models.Usage(
cloud_name=response["cloud_name"],
cloud_type=response["cloud_type"],
image_id=response["image_id"],
instance_type=response["instance_type"],
user_id=response["access_key"],
email=response.get("institutional_email", ""),
)
u.save()
except Exception, e:
log.debug("Trouble saving Usage data: {0}".format(e))
else:
starting_text_list = [
"Starting an instance... please wait",
"Really starting!",
"Still starting.",
"Hopefully done soon!",
]
st = starting_text_list[randint(0, len(starting_text_list) - 1)]
r["starting_text"] = st
def error_handler(uuid, task_id):
print('Tasks::error_handler', uuid, task_id)
from celery.result import AsyncResult
result = AsyncResult(uuid)
exc = result.get(propagate=False)
print('Task {0} raised exception: {1!r}\n{2!r}'.format(
uuid, exc, result.traceback))
app.backend.mark_as_failure(task_id, exc, traceback=result.traceback)
def task_result(request):
task_id = request.GET.get('task')
result = AsyncResult(task_id)
if result.ready():
data = result.get()
return HttpResponse("<htmL><body><h1>Result: %s</h1></body></html>" % data)
else:
return HttpResponse("<htmL><body><h1>Not Ready</h1></body></html>")
