def update_clusters(request):
"""
Given a task ID as part of the ``request`` (as ``request_id``), check on the
status of a job retrieving clusters' persistent data. Return a JSON with the
following fields:
``task_id``: return the job request ID
``ready``: ``True`` if the job has completed; ``False`` otherwise
``clusters_list``: a list of clusters' persistent data (if the job
has completed) or an empty list otherwise
"""
task_id = request.POST.get('task_id', '')
result = AsyncResult(task_id)
fetching_data_text_list = ['Fetching data... please wait', 'Fetching data...',
'Still fetching data...', 'Hopefully done soon!']
fdt = fetching_data_text_list[randint(0, len(fetching_data_text_list) - 1)]
r = {'task_id': task_id,
'ready': result.ready(),
'clusters_list': [],
'wait_text': fdt,
'error': None}
if task_id == 'missing_form_data': # See `fetch_clusters` method
r['error'] = "Missing form data. Please supply the data and try again."
elif result.ready():
clusters_pd = result.get()
r['clusters_list'] = clusters_pd.get('clusters', [])
if clusters_pd.get('error', None):
r['error'] = clusters_pd['error']
return HttpResponse(simplejson.dumps(r), mimetype="application/json")
def get(self,request,uuid):
tk = AsyncResult(uuid)
if tk.ready():
res = 'Task result is:%s' % str(tk.get())
else:
res = 'Task not finished!'
return Response(res, status=status.HTTP_200_OK)
def cleanup_tasks():
in_progress = QueryTaskTracker.all(QueryTaskTracker.IN_PROGRESS_LIST)
for tracker in in_progress:
result = AsyncResult(tracker.task_id)
if result.ready():
logging.info("in progress tracker %s finished", tracker.query_hash)
_unlock(tracker.query_hash, tracker.data_source_id)
tracker.update(state='finished')
# Maintain constant size of the finished tasks list:
removed = 1000
while removed > 0:
removed = QueryTaskTracker.prune(QueryTaskTracker.DONE_LIST, 1000)
waiting = QueryTaskTracker.all(QueryTaskTracker.WAITING_LIST)
for tracker in waiting:
if tracker is None:
continue
result = AsyncResult(tracker.task_id)
if result.ready():
logging.info("waiting tracker %s finished", tracker.query_hash)
_unlock(tracker.query_hash, tracker.data_source_id)
tracker.update(state='finished')
def get_context_data(self, **kwargs):
context = super(TaskResultView, self).get_context_data(**kwargs)
task_id = kwargs['task_id']
async_res = AsyncResult(task_id)
context['result'] = async_res.get()
context['task_id'] = task_id
return context
def getTaskResult(self, task_id):
async_result = AsyncResult(task_id)
can_forget = async_result.ready()
try:
try:
result = async_result.result
if isinstance(result, Exception):
result_cls_name = result.__class__.__name__
try:
errno = ERRNO_NS[result_cls_name]
except KeyError:
LOGGER.error('Undefined errno: %s', result_cls_name)
raise spec.ServerError()
value = [errno, result.message]
else:
value = result
except Exception as exc:
LOGGER.exception(exc)
raise spec.ServerError()
status = getattr(spec.ResultStatus, async_result.status)
return spec.AsyncResult(status=status, value=json.dumps(value))
finally:
if can_forget:
async_result.forget()
LOGGER.info('Forgot the result of task %s', task_id)
def analytics_status(request, graph_slug):
analytics_results = dict()
analytics_request = request.GET.get("analytics_request")
analytics_executing = json.loads(analytics_request)
if request.is_ajax() and analytics_executing is not None:
for task_id in analytics_executing:
task = AsyncResult(task_id)
# We need to take into account if the task has been revoked
if task.ready() and task.status != REVOKED:
# Sometimes, the execution is faster than the revoked flag. So
# we need to control that (ValueError exception)
try:
analytic = Analytic.objects.filter(dump__graph__slug=graph_slug, task_id=task_id).latest()
analytics_results[task_id] = [
STATUS_OK,
analytic.results.url,
analytic.id,
analytic.task_start,
analytic.algorithm,
analytic.values.url,
]
except ValueError:
analytics_results[task_id] = [REVOKED]
elif task.status == REVOKED:
analytics_results[task_id] = [REVOKED]
data = analytics_results
json_data = json.dumps(data)
return HttpResponse(json_data, content_type="application/json")
def instancestate(request):
"""
Given a POST request with ``task_id`` and ``instance_state`` fields, check if
the task has completed. If so, return JSON with updated value for the
``instance_state`` field and start a new task, appropriately setting the
value of ``task_id``. If the initial ``task_id`` has not completed, return
the same value for the ``task_id`` field.
"""
task_id = request.POST.get('task_id', None)
instance_state = request.POST.get('instance_state', 'pending') # Preserve current state
state = {'task_id': None, 'instance_state': instance_state, 'error': ''} # Reset info to be sent
if task_id:
# If we have a running task, check on instance state
result = AsyncResult(task_id)
if result.ready():
state = result.get()
state['task_id'] = None # Reset but make sure it exists
else:
# If task not ready, send back the task_id
state['task_id'] = task_id
elif 'ec2data' in request.session:
# We have no task ID, so start a task to get instance state
form = request.session["ec2data"]
cloud = form.get('cloud', None)
a_key = form.get("access_key", None)
s_key = form.get("secret_key", None)
instance_id = form.get("instance_id", None)
if not instance_id:
state['error'] = "Missing instance ID, cannot check the state."
r = tasks.instance_state.delay(cloud, a_key, s_key, instance_id)
state['task_id'] = r.id
else:
state = {'instance_state': 'Not available'}
return HttpResponse(simplejson.dumps(state), mimetype="application/json")
def test_forget(self):
tb = DatabaseBackend(backend="memory://")
tid = uuid()
tb.mark_as_done(tid, {"foo": "bar"})
x = AsyncResult(tid)
x.forget()
self.assertIsNone(x.result)
def task_result(request):
post = json.loads(request.body)
task_id = post.get("task_id")
result = AsyncResult(task_id)
if not result.ready():
return json_response(None)
return json_response(result.get(timeout=1))
def cancel(event):
"""
This is bound to the "jobs.cancel" event, and will be triggered any time
a job is canceled. This handler will process any job that has the
handler field set to "worker_handler".
"""
job = event.info
if job['handler'] in ['worker_handler', 'celery_handler']:
# Stop event propagation and prevent default, we are using a custom state
event.stopPropagation().preventDefault()
celeryTaskId = job.get('celeryTaskId')
if celeryTaskId is None:
msg = ("Unable to cancel Celery task. Job '%s' doesn't have a Celery task id."
% job['_id'])
logger.warn(msg)
return
if job['status'] not in [CustomJobStatus.CANCELING, JobStatus.CANCELED,
JobStatus.SUCCESS, JobStatus.ERROR]:
# Set the job status to canceling
ModelImporter.model('job', 'jobs').updateJob(job, status=CustomJobStatus.CANCELING)
# Send the revoke request.
asyncResult = AsyncResult(celeryTaskId, app=getCeleryApp())
asyncResult.revoke()
def test_forget(self):
tb = CacheBackend(backend="memory://")
tid = gen_unique_id()
tb.mark_as_done(tid, {"foo": "bar"})
x = AsyncResult(tid, backend=tb)
x.forget()
self.assertIsNone(x.result)
def POST(self):
x = web.input(master_id='', project_id='', task_id='')
project = models.Project.get(id=x.project_id)
if not project:
raise web.notfound()
master = models.Master.get(id=x.master_id)
if not master:
return web.notfound()
if x.task_id:
from celery.result import AsyncResult
from metapolator.config import celery
res = AsyncResult(x.task_id, backend=celery.backend)
if res.ready():
master.task_completed = True
web.ctx.orm.commit()
return ujson.dumps({'done': True})
else:
master.task_updated = datetime.datetime.now()
web.ctx.orm.commit()
return ujson.dumps({'done': False, 'task_id': x.task_id})
master.task_completed = True
web.ctx.orm.commit()
return ujson.dumps({'done': True})
def check(self, request, *args, **kwargs):
try:
task_id = request.data['id']
uuid.UUID(task_id)
except KeyError:
raise ValidationError("You should specify id")
except ValueError:
raise ValidationError("Not valid id")
async_result = AsyncResult(task_id)
try:
result = async_result.get(timeout=0.5, propagate=False)
except TimeoutError:
result = None
status = async_result.status
if isinstance(result, Exception):
return Response({
'status': status,
'error': str(result),
}, status=HTTP_200_OK)
elif result is None:
return Response({
'status': status
}, status=HTTP_204_NO_CONTENT)
else:
return Response({
'status': status,
'result': result,
}, status=HTTP_200_OK)
def get(self, slug):
print slug
res = AsyncResult(slug)
if res.ready():
self.render("results.html", username=self.current_user.replace('\"',''), jobid=slug)
else:
self.render("results.html", username=self.current_user.replace('\"',''), jobid='00')
def task_list(request):
from celery.result import AsyncResult
runs = CeleryRun.objects.filter(hide=False)
if runs:
header = get_header(runs)
header.append('started')
header.append('created')
header.remove('hide')
else:
header = []
#assert False, header
druns = runs.values()
data = []
for drun in druns:
name = drun['name']
result = AsyncResult(name)
drun['started'] = result.ready()
#assert False,
data.append([drun[h] for h in header])
return render_to_response('celery_task_list.html', {
'data': data,
'header': header,
'title': 'tasks',
})
def verify_task(task_id):
result = AsyncResult(task_id)
if result.read():
# do_something_with(result.get())
pass
else:
verify_task.retry(countdown=1)
def state(self):
result = AsyncResult(self.task_id)
if not result.ready():
return result.state
if result.get() == 'ERROR':
return 'FAILED'
return result.state
def repo_check(self, repo_name):
c.repo = repo_name
task_id = request.GET.get('task_id')
if task_id and task_id not in ['None']:
from kallithea import CELERY_ON
from celery.result import AsyncResult
if CELERY_ON:
task = AsyncResult(task_id)
if task.failed():
raise HTTPInternalServerError(task.traceback)
repo = Repository.get_by_repo_name(repo_name)
if repo and repo.repo_state == Repository.STATE_CREATED:
if repo.clone_uri:
clone_uri = repo.clone_uri_hidden
h.flash(_('Created repository %s from %s')
% (repo.repo_name, clone_uri), category='success')
else:
repo_url = h.link_to(repo.repo_name,
h.url('summary_home',
repo_name=repo.repo_name))
fork = repo.fork
if fork:
fork_name = fork.repo_name
h.flash(h.literal(_('Forked repository %s as %s')
% (fork_name, repo_url)), category='success')
else:
h.flash(h.literal(_('Created repository %s') % repo_url),
category='success')
return {'result': True}
return {'result': False}
def async_results(result_id):
async_result = AsyncResult(result_id)
if not async_result.ready():
raise DSProcessUnfinished("Result with id {} is not ready.".format(result_id))
if async_result.status != TaskStates.SUCCESS:
raise DSProcessError("An error occurred during background processing.")
return async_result.result
def get_async_result(id, backend="celery"):
if backend == 'celery':
res = AsyncResult(id)
if res.ready():
return res.get()
raise ValueError("no result")
def taskReady(jobObj, redirect="error"):
"""Checks if celery task is ready.
Args:
celeryID: the id of a celery task.
redirect: page to redirect to on error.
Returns:
True,None: celery task finished successfully.
False, HttpResponseRedirect: celerytask failed.
False,False: celery task is still processing.
"""
task = AsyncResult(jobObj.celeryUID)
if task.ready():
if task.successful():
return True, None
else:
return False, HttpResponseRedirect(reverse(redirect))
# In the case the task isn't ready, we don't really have a clear idea why.
# It is possible the task has left the celery task table, and there for is marked as Pending,
# but it is really done. We store an alternate flag in our own tables to manage this.
if jobObj.state == symTyperTask.DONE:
return True, None
elif jobObj.state == symTyperTask.ERROR :
return False, HttpResponseRedirect(reverse(redirect))
else:
return False, None
def check_status(task_id):
task = AsyncResult(task_id, app=tasks.celery)
if task.ready():
if task.successful(): # Task finished successfully
status, result_value = task.result
if status: # Valid result
app.logger.info("result: %s", pprint.pformat(result_value))
return flask.jsonify(result_value)
else: # Handled exception
response = flask.make_response(result_value, 400)
response.mimetype = 'text/plain'
return response
else: # Unhandled exception
exc = task.result
response = flask.make_response(
traceback.format_exception_only(type(exc), exc),
500)
response.mimetype = 'text/plain'
return response
else: # if task.ready()
status = {
'result_id': task.id,
'state': 'PENDING',
'progress': 0,
}
if task.state == 'PROGRESS':
status.update(task.result)
response = flask.jsonify(status)
response.status_code = 202
return response
def get(self, request, uuid):
res = AsyncResult(uuid) # Reconstruct a AsyncResult object with task id.
if res.ready(): # Judge whether the task is finished,if result is ready,get it with get() method.
task_res = res.get() # Get task result.
else:
task_res = "Task not finished!"
return Response(task_res, status=status.HTTP_200_OK)
def instancestate(request):
"""
Give a POST request with ``task_id`` and ``instance_state`` fields, return
JSON with updated value (given the task has completed or the same as provided)
value for the ``instance_state`` field and the same value for the ``task_id``.
``task_id`` is to correspond to the ID of the background task.
If instance state is not available, return ``Not available`` as the value
for ``instance_state``.
"""
task_id = request.POST.get('task_id', None)
instance_state = request.POST.get('instance_state', 'pending') # Preserve current state
state = {'task_id': None, 'instance_state': instance_state, 'error': ''} # Reset info to be sent
if task_id:
# If we have a running task, check on instance state
result = AsyncResult(task_id)
if result.ready():
state = result.get()
state['task_id'] = None # Reset but make sure it exists
else:
# If task not ready, send back the task_id
state['task_id'] = task_id
elif 'ec2data' in request.session:
# We have no task ID, so start a task to get instance state
form = request.session["ec2data"]
cloud = form['cloud']
a_key = form["access_key"]
s_key = form["secret_key"]
instance_id = form["instance_id"]
r = tasks.instance_state.delay(cloud, a_key, s_key, instance_id)
state['task_id'] = r.id
else:
state = {'instance_state': 'Not available'}
return HttpResponse(simplejson.dumps(state), mimetype="application/json")
def update_clusters(request):
"""
Given a task ID as part of the ``request`` (as ``request_id``), check on the
status of a job retrieving clusters' persistent data. Return a JSON with the
following fields:
``task_id``: return the job request ID
``ready``: ``True`` if the job has completed; ``False`` otherwise
``clusters_list``: a list of clusters' persistent data (if the job
has completed) or an empty list otherwise
"""
task_id = request.POST.get("task_id", "")
result = AsyncResult(task_id)
fetching_data_text_list = [
"Fetching data... please wait",
"Fetching data...",
"Still fetching data...",
"Hopefully done soon!",
]
fdt = fetching_data_text_list[randint(0, len(fetching_data_text_list) - 1)]
r = {"task_id": task_id, "ready": result.ready(), "clusters_list": [], "wait_text": fdt, "error": None}
if result.ready():
clusters_pd = result.get()
r["clusters_list"] = clusters_pd.get("clusters", [])
if clusters_pd.get("error", None):
r["error"] = clusters_pd["error"]
return HttpResponse(simplejson.dumps(r), mimetype="application/json")
def __call__(self):
result = AsyncResult(self.request.form['task_id'])
self.request.response.setHeader('Content-Type', 'text/plain')
if result.ready():
return 'Done! Result is %s' % result.result
else:
return 'Still in progress, retry later!'
def cleanup_tasks():
in_progress = QueryTaskTracker.all(QueryTaskTracker.IN_PROGRESS_LIST)
for tracker in in_progress:
result = AsyncResult(tracker.task_id)
# If the AsyncResult status is PENDING it means there is no celery task object for this tracker, and we can
# mark it as "dead":
if result.status == 'PENDING':
logging.info("In progress tracker for %s is no longer enqueued, cancelling (task: %s).",
tracker.query_hash, tracker.task_id)
_unlock(tracker.query_hash, tracker.data_source_id)
tracker.update(state='cancelled')
if result.ready():
logging.info("in progress tracker %s finished", tracker.query_hash)
_unlock(tracker.query_hash, tracker.data_source_id)
tracker.update(state='finished')
waiting = QueryTaskTracker.all(QueryTaskTracker.WAITING_LIST)
for tracker in waiting:
result = AsyncResult(tracker.task_id)
if result.ready():
logging.info("waiting tracker %s finished", tracker.query_hash)
_unlock(tracker.query_hash, tracker.data_source_id)
tracker.update(state='finished')
# Maintain constant size of the finished tasks list:
QueryTaskTracker.prune(QueryTaskTracker.DONE_LIST, 1000)
def retrieve_mbs_result(target_task_id):
logger = Task.get_logger()
r = AsyncResult(target_task_id)
sr = SimulationResult.objects.get(task_id__exact=target_task_id)
# sr = SimulationResult.objects.get(sim_id__exact=r['sim_id'])
logger.info(r)
while not r.ready():
time.sleep(0.1)
result = json.loads(r.result)
if result['exit_code'] == 0:
## success
sr.sim_id = result['sim_id']
## these are rewrite if you add log collections
sr.collections = json.dumps([
"%s_nwk" % sr.sim_id,
"%s_node" % sr.sim_id,
"%s_msg" % sr.sim_id,
"%s_usr" % sr.sim_id,
"%s_map" % sr.sim_id,
])
sr.task_progress = 100
sr.task_status = "SUCCESS"
sr.save()
else:
sr.sim_id = "NO SIM_ID (FAILED)"
sr.task_status = "FAILED"
sr.task_progress = 0
sr.save()
def get(self, request, *args, **kwargs):
task_id = kwargs.pop('task_id')
async_result = AsyncResult(task_id)
try:
# Poll the task to see if the result is available
result = async_result.get(timeout=20)
except TimeoutError:
result = None
task_status = TaskStatus.objects.get(task_id=task_id)
# Strip parentheses and get the name of the function to call
task_name = re.sub(r'\([^)]*\)', '', task_status.signature)
if task_name in request.session['tasks']:
del(request.session['tasks'][task_name])
request.session.modified = True
status = task_status.status
# Setup the response
response = {'task_id': task_id, 'task_status': status, 'task_name': task_name, 'task_result': result}
return HttpResponse(json.dumps(response), content_type='application/json')
def instancestate(request):
task_id = request.POST.get("task_id", None)
instance_state = request.POST.get("instance_state", "pending") # Preserve current state
state = {"task_id": None, "instance_state": instance_state} # Reset info to be sent
if task_id:
# If we have a running task, check on instance state
result = AsyncResult(task_id)
if result.ready():
state = result.get()
state["task_id"] = None # Reset but make sure it exists
else:
# If task not ready, send back the task_id
state["task_id"] = task_id
elif "ec2data" in request.session:
# We have no task ID, so start a task to get instance state
form = request.session["ec2data"]
cloud = form["cloud"]
a_key = form["access_key"]
s_key = form["secret_key"]
instance_id = form["instance_id"]
r = tasks.instance_state.delay(cloud, a_key, s_key, instance_id)
state["task_id"] = r.id
else:
state = {"instance_state": "Not available"}
return HttpResponse(simplejson.dumps(state), mimetype="application/json")
def test_reduce_direct(self):
x = AsyncResult('1', app=self.app)
fun, args = x.__reduce__()
self.assertEqual(fun(*args), x)
def get_task_status(strategy_id):
return AsyncResult(find_task(strategy_id)).status
def stop(self, message=None):
if self.is_running:
task = AsyncResult(self.celery_task_id)
task.revoke(terminate=True, signal='SIGKILL')
self.on_stop(message=message)
def test_without_id(self):
with self.assertRaises(ValueError):
AsyncResult(None, app=self.app)
class QueryTask(object):
# TODO: this is mapping to the old Job class statuses. Need to update the client side and remove this
STATUSES = {
'PENDING': 1,
'STARTED': 2,
'SUCCESS': 3,
'FAILURE': 4,
'REVOKED': 4
}
def __init__(self, job_id=None, async_result=None):
if async_result:
self._async_result = async_result
else:
self._async_result = AsyncResult(job_id, app=celery)
@property
def id(self):
return self._async_result.id
def to_dict(self):
if self._async_result.status == 'STARTED':
updated_at = self._async_result.result.get('start_time', 0)
else:
updated_at = 0
status = self.STATUSES[self._async_result.status]
if isinstance(self._async_result.result, Exception):
error = self._async_result.result.message
status = 4
elif self._async_result.status == 'REVOKED':
error = 'Query execution cancelled.'
else:
error = ''
if self._async_result.successful() and not error:
query_result_id = self._async_result.result
else:
query_result_id = None
return {
'id': self._async_result.id,
'updated_at': updated_at,
'status': status,
'error': error,
'query_result_id': query_result_id,
}
@property
def is_cancelled(self):
return self._async_result.status == 'REVOKED'
@property
def celery_status(self):
return self._async_result.status
def ready(self):
return self._async_result.ready()
def cancel(self):
return self._async_result.revoke(terminate=True, signal='SIGINT')
def task_processing(filename):
"""Process the image endpoint."""
task = processing.apply_async((filename, ))
async_result = AsyncResult(id=task.task_id, app=celery)
processing_result = async_result.get()
return render_template('result.html', video_name=processing_result)
def _get_async_result(task_id):
"""
Use this minor indirection to facilitate mocking the AsyncResult in tests.
"""
return AsyncResult(task_id)
def edit_upload(request, upload_id):
qobuz_upload = QobuzUpload.objects.get(id=upload_id)
if request.method == 'POST':
form = EditUploadForm(request.POST, qobuz_upload=qobuz_upload)
if form.is_valid():
qobuz_upload.artists = form.cleaned_data['artists']
qobuz_upload.album_name = form.cleaned_data['album_name']
tracks = []
for i, track in enumerate(qobuz_upload.track_data):
qobuz_track = qobuz_upload.album_data['tracks']['items'][i]
title_field = 'track_{0}_{1}_title'.format(
qobuz_track['media_number'], i + 1)
artists_field = 'track_{0}_{1}_artists'.format(
qobuz_track['media_number'], i + 1)
tracks.append({
'media_number': qobuz_track['media_number'],
'artists': form.cleaned_data[artists_field],
'title': form.cleaned_data[title_field],
})
qobuz_upload.track_data_json = ujson.dumps(tracks)
qobuz_upload.save()
else:
form = EditUploadForm(qobuz_upload=qobuz_upload)
download_error = ''
if qobuz_upload.download_task_id:
async_result = AsyncResult(qobuz_upload.download_task_id)
if async_result.state == states.PENDING:
download_status = 'Waiting to start'
elif async_result.state == states.STARTED:
download_status = 'Started'
elif async_result.state == states.SUCCESS:
download_status = 'Completed'
elif async_result.state == states.FAILURE:
download_status = 'Failed'
download_error = '{0}: {1}'.format(
type(async_result.result).__name__,
async_result.result.message)
else:
download_status = 'Unknown Status'
else:
download_status = 'not_started'
try:
spectral_files = sorted(os.listdir(wm_str(
qobuz_upload.spectrals_path)))
except OSError:
spectral_files = []
try:
cover_files = sorted([
f for f in os.listdir(wm_str(qobuz_upload.temp_media_path))
if f.endswith('.jpg')
],
reverse=True)
except OSError:
cover_files = []
data = {
'upload': qobuz_upload,
'form': form,
'download_status': download_status,
'download_error': download_error,
'spectral_files': spectral_files,
'cover_files': cover_files,
}
return render(request, 'qobuz/edit_upload.html', data)
def __init__(self,task_id):
self.task_id=task_id
self.task_object = AsyncResult(self.task_id)
def get_job_status(job_id):
res = AsyncResult(job_id, backend=celery_object.backend)
return jsonify({"status": res.ready()})
def setUp(self):
self.ts = TaskSetResult(
gen_unique_id(),
[AsyncResult(gen_unique_id()),
AsyncResult(gen_unique_id())])
def setUp(self):
self.task = AsyncResult(gen_unique_id())
def make_mock_taskset(size=10):
tasks = [mock_task("ts%d" % i, states.SUCCESS, i) for i in xrange(size)]
[save_result(task) for task in tasks]
return [AsyncResult(task["id"]) for task in tasks]
def test_successful(self):
ok_res = AsyncResult(self.task1["id"])
nok_res = AsyncResult(self.task3["id"])
nok_res2 = AsyncResult(self.task4["id"])
self.assertTrue(ok_res.successful())
self.assertFalse(nok_res.successful())
self.assertFalse(nok_res2.successful())
pending_res = AsyncResult(gen_unique_id())
self.assertFalse(pending_res.successful())
def test_get_timeout_longer(self):
res = AsyncResult(self.task4["id"]) # has RETRY status
self.assertRaises(TimeoutError, res.get, timeout=1)
def test_reduce_direct(self):
x = AsyncResult('1', app=self.app)
fun, args = x.__reduce__()
assert fun(*args) == x
def test_get_timeout(self):
res = AsyncResult(self.task4["id"]) # has RETRY status
self.assertRaises(TimeoutError, res.get, timeout=0.1)
pending_res = AsyncResult(gen_unique_id())
self.assertRaises(TimeoutError, pending_res.get, timeout=0.1)
def test_join_timeout(self):
ar = MockAsyncResultSuccess(gen_unique_id())
ar2 = MockAsyncResultSuccess(gen_unique_id())
ar3 = AsyncResult(gen_unique_id())
ts = TaskSetResult(gen_unique_id(), [ar, ar2, ar3])
self.assertRaises(TimeoutError, ts.join, timeout=0.0000001)
def wait_task(task_id):
task = AsyncResult(task_id)
return task.get(timeout=10)
class test_RedisBackend_chords_complex(basetest_RedisBackend):
@pytest.fixture(scope="function", autouse=True)
def complex_header_result(self):
with patch("celery.result.GroupResult.restore") as p:
yield p
@pytest.mark.parametrize(
['results', 'assert_save_called'],
[
# No results in the header at all - won't call `save()`
(tuple(), False),
# Simple results in the header - won't call `save()`
((AsyncResult("foo"), ), False),
# Many simple results in the header - won't call `save()`
((AsyncResult("foo"), ) * 42, False),
# A single complex result in the header - will call `save()`
((GroupResult("foo", []), ), True),
# Many complex results in the header - will call `save()`
((GroupResult("foo"), ) * 42, True),
# Mixed simple and complex results in the header - will call `save()`
(itertools.islice(
itertools.cycle((
AsyncResult("foo"),
GroupResult("foo"),
)),
42,
), True),
])
def test_apply_chord_complex_header(self, results, assert_save_called):
mock_group_result = Mock()
mock_group_result.return_value.results = results
self.app.GroupResult = mock_group_result
header_result_args = ("gid11", results)
self.b.apply_chord(header_result_args, None)
if assert_save_called:
mock_group_result.return_value.save.assert_called_once_with(
backend=self.b)
else:
mock_group_result.return_value.save.assert_not_called()
def test_on_chord_part_return_timeout(self, complex_header_result):
tasks = [self.create_task(i) for i in range(10)]
random.shuffle(tasks)
try:
self.app.conf.result_chord_join_timeout += 1.0
for task, result_val in zip(tasks, itertools.cycle((42, ))):
self.b.on_chord_part_return(
task.request,
states.SUCCESS,
result_val,
)
finally:
self.app.conf.result_chord_join_timeout -= 1.0
join_func = complex_header_result.return_value.join_native
join_func.assert_called_once_with(timeout=4.0, propagate=True)
@pytest.mark.parametrize("supports_native_join", (True, False))
def test_on_chord_part_return(
self,
complex_header_result,
supports_native_join,
):
mock_result_obj = complex_header_result.return_value
mock_result_obj.supports_native_join = supports_native_join
tasks = [self.create_task(i) for i in range(10)]
random.shuffle(tasks)
with self.chord_context(10) as (tasks, request, callback):
for task, result_val in zip(tasks, itertools.cycle((42, ))):
self.b.on_chord_part_return(
task.request,
states.SUCCESS,
result_val,
)
# Confirm that `zadd` was called even though we won't end up
# using the data pushed into the sorted set
assert self.b.client.zadd.call_count == 1
self.b.client.zadd.reset_mock()
# Confirm that neither `zrange` not `lrange` were called
self.b.client.zrange.assert_not_called()
self.b.client.lrange.assert_not_called()
# Confirm that the `GroupResult.restore` mock was called
complex_header_result.assert_called_once_with(request.group)
# Confirm the the callback was called with the `join()`ed group result
if supports_native_join:
expected_join = mock_result_obj.join_native
else:
expected_join = mock_result_obj.join
callback.delay.assert_called_once_with(expected_join())
def get_task(task_id):
return AsyncResult(task_id)
def test_without_id(self):
with pytest.raises(ValueError):
AsyncResult(None, app=self.app)
def test_execute_ignore_result(self):
task_id = uuid()
ret = jail(task_id, MyTaskIgnoreResult.name, [4], {})
self.assertEqual(ret, 256)
self.assertFalse(AsyncResult(task_id).ready())
def test_celery_curl():
if request.method == 'GET':
task = example_celery_method.delay()
async_result = AsyncResult(id=task.task_id, app=celery)
processing_result = async_result.get()
return str(processing_result)
def get_payload(ts_id):
'''Goes through the task_ids for an analysis and determines the status
of each task (pre-processing, processing and cleanup)
'''
#print "get_payload \n"
payload = []
ts = AsyncResult( ts_id )
if ts:
if ts.result:
if type(ts.result) ==type(dict()):
if type(ts.result['message']) ==type(dict()):
temp_ret = ts.result['message']
else:
temp_ret = ts.result
temp_ret['state'] = ts.state
temp_ret['task_id'] = ts.task_id
payload.append(temp_ret)
elif (ts.result.__class__.__name__ == 'TaskSetResult'):
n_tasks = len(ts.result.results)
if n_tasks > 0:
for j in range(0,n_tasks):
temp_ret = {};
if ts.result.results[j].result:
if ts.result.results[j].result.__class__.__name__ == 'FileStoreItem':
temp_ret['state'] = ts.result.results[j].state
temp_ret['task_id'] = ts.result.results[j].task_id
payload.append(temp_ret)
else:
# if result returns as dictionary
if type(ts.result.results[j].result) ==type(dict()):
temp_ret = ts.result.results[j].result
# if result is just a string
else:
#print "-------------"
#print "PAYLOAD TEST"
#print "j"
#print j
#print "results all"
#print ts.result.results[j]
#print "result"
#print ts.result.results[j].result
#print "state"
#print ts.result.results[j].state
temp_ret = {}
temp_ret['state'] = str(ts.result.results[j].state)
temp_ret['task_id'] = str(ts.result.results[j].task_id)
payload.append(temp_ret)
else:
temp_ret['state'] = ts.result.results[j].state
temp_ret['task_id'] = ts.result.results[j].task_id
payload.append(temp_ret)
#else:
# print "00000 tasks"
# print ts.result
else:
temp_ret = {'state':ts.state, 'info':str(ts.result), 'task_id':ts.task_id}
payload.append(temp_ret)
else:
temp_ret = {'state':ts.state, 'task_id':ts.task_id}
payload.append(temp_ret)
else:
temp_ret = {'state':"### WAITING ###"}
payload.append(temp_ret)
#print "payload called"
#print "################################"
#print payload
#print len(payload)
#print "################################"
return payload
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# 开发人员 ：Davis Niu
# 开发时间 ：6/30/2020 11:25 PM
# 文件名称 ：result.py
from celery.result import AsyncResult
from celery_task import cel
async_result = AsyncResult(id='8628904d-e02a-473f-966e-9eca4286ba4d', app=cel)
if async_result.successful():
result = async_result.get()
print(result)
elif async_result.failed():
print('failed')
elif async_result.status == 'PENDING':
print('PENDING')
else:
print('other')
def __init__(self, job_id=None, async_result=None):
if async_result:
self._async_result = async_result
else:
self._async_result = AsyncResult(job_id, app=celery)
# ------------------------------
# @Time : 2020/5/5
# @Author : gao
# @File : result.py
# @Project : AmazingQuant
# ------------------------------
import time
from celery.result import AsyncResult
from test.celery.run import app
from test.celery.celery_app_task import add
result = add.delay(1, 1)
print(result.id)
a = 3
async1 = AsyncResult(id=result.id, app=app)
while True:
if async1.successful():
result = async1.get()
print(result)
print('执行完成', time.time())
# async1.forget() # 将结果删除
break
elif async1.failed():
print('执行失败')
elif async1.status == 'PENDING':
print('任务等待中被执行')
elif async1.status == 'RETRY':
print('任务异常后正在重试')
elif async1.status == 'STARTED':
class QueryTask(object):
MAX_RETRIES = 5
# TODO: this is mapping to the old Job class statuses. Need to update the client side and remove this
STATUSES = {
'PENDING': 1,
'STARTED': 2,
'SUCCESS': 3,
'FAILURE': 4,
'REVOKED': 4
}
def __init__(self, job_id=None, async_result=None):
if async_result:
self._async_result = async_result
else:
self._async_result = AsyncResult(job_id, app=celery)
@property
def id(self):
return self._async_result.id
@classmethod
def add_task(cls, query, data_source, scheduled=False, metadata={}):
query_hash = gen_query_hash(query)
logging.info("[Manager][%s] Inserting job", query_hash)
logging.info("[Manager] Metadata: [%s]", metadata)
try_count = 0
job = None
while try_count < cls.MAX_RETRIES:
try_count += 1
pipe = redis_connection.pipeline()
try:
pipe.watch(cls._job_lock_id(query_hash, data_source.id))
job_id = pipe.get(cls._job_lock_id(query_hash, data_source.id))
if job_id:
logging.info("[Manager][%s] Found existing job: %s", query_hash, job_id)
job = cls(job_id=job_id)
if job.ready():
logging.info("[%s] job found is ready (%s), removing lock", query_hash, job.celery_status)
redis_connection.delete(QueryTask._job_lock_id(query_hash, data_source.id))
job = None
if not job:
pipe.multi()
if scheduled:
queue_name = data_source.scheduled_queue_name
else:
queue_name = data_source.queue_name
result = execute_query.apply_async(args=(query, data_source.id, metadata), queue=queue_name)
job = cls(async_result=result)
logging.info("[Manager][%s] Created new job: %s", query_hash, job.id)
pipe.set(cls._job_lock_id(query_hash, data_source.id), job.id, settings.JOB_EXPIRY_TIME)
pipe.execute()
break
except redis.WatchError:
continue
if not job:
logging.error("[Manager][%s] Failed adding job for query.", query_hash)
return job
def to_dict(self):
if self._async_result.status == 'STARTED':
updated_at = self._async_result.result.get('start_time', 0)
else:
updated_at = 0
if self._async_result.failed() and isinstance(self._async_result.result, Exception):
error = self._async_result.result.message
elif self._async_result.status == 'REVOKED':
error = 'Query execution cancelled.'
else:
error = ''
if self._async_result.successful():
query_result_id = self._async_result.result
else:
query_result_id = None
return {
'id': self._async_result.id,
'updated_at': updated_at,
'status': self.STATUSES[self._async_result.status],
'error': error,
'query_result_id': query_result_id,
}
@property
def is_cancelled(self):
return self._async_result.status == 'REVOKED'
@property
def celery_status(self):
return self._async_result.status
def ready(self):
return self._async_result.ready()
def cancel(self):
return self._async_result.revoke(terminate=True, signal='SIGINT')
@staticmethod
def _job_lock_id(query_hash, data_source_id):
return "query_hash_job:%s:%s" % (data_source_id, query_hash)
def update_main_task(self, main_task_id: str):
all_task_info_rows = self.filter(
Q(id=main_task_id) | Q(main_task_id=main_task_id)) \
.exclude(name__in=self.EXCLUDE_FROM_TRACKING) \
.values_list('propagate_exceptions', 'own_status', 'own_progress', 'own_date_done',
'run_after_sub_tasks_finished')
total_status_non_propagating_exceptions = None
total_status_propagating_exceptions = None
found_propagating_exceptions = False
total_progress = 0
total_done = True
total_date_done = None
sub_tasks_finished = True
has_deferred_tasks = False
for propagate_exceptions, status, progress, date_done, run_after_sub_tasks_finished in all_task_info_rows:
if propagate_exceptions:
found_propagating_exceptions = True
total_status_propagating_exceptions = status \
if total_status_propagating_exceptions is None \
else max(total_status_propagating_exceptions,
status,
key=precedence_propagating_exceptions)
else:
total_status_non_propagating_exceptions = status \
if total_status_non_propagating_exceptions is None \
else max(total_status_non_propagating_exceptions,
status,
key=precedence_non_propagating_exceptions)
if not run_after_sub_tasks_finished:
sub_tasks_finished = sub_tasks_finished and status in READY_STATES
else:
has_deferred_tasks = True
if progress is None:
progress = 0
if status in READY_STATES:
progress = 100
total_progress += progress
# if there is at least one date_done = None then we are not done
if date_done is None:
total_done = False
# if we still did not got any date_done = None (meaning we are done)
# then calc total date done as the max date done we got
if total_done:
total_date_done = max(
total_date_done,
date_done) if total_date_done else date_done
else:
total_date_done = None
total_progress = (
total_progress /
len(all_task_info_rows)) if len(all_task_info_rows) else 100
if total_status_propagating_exceptions in PROPAGATE_STATES:
# we have error in an important sub-task
total_status = total_status_propagating_exceptions
elif found_propagating_exceptions:
# we don't have errors in important sub-tasks
# but there were important sub-tasks and we should take their statuses into account
# (this is mostly to take care of None state in total_status_propagating_exceptions
# meaning PENDING)
total_status = max(total_status_propagating_exceptions,
total_status_non_propagating_exceptions,
key=precedence_non_propagating_exceptions)
else:
# we did not see any important sub-tasks and should not take into account
# None in total_status_propagating_exceptions
total_status = total_status_non_propagating_exceptions
if has_deferred_tasks and sub_tasks_finished:
if total_status not in PROPAGATE_STATES:
self.run_after_sub_tasks(main_task_id)
else:
self \
.filter(main_task_id=main_task_id, run_after_sub_tasks_finished=True) \
.update(status=total_status, date_done=total_date_done)
self.filter(id=main_task_id).update(date_done=total_date_done,
status=total_status,
completed=total_progress == 100,
progress=total_progress)
if total_status in READY_STATES:
try:
main_task = self.get(id=main_task_id) # type: Task
if total_status == SUCCESS:
main_task.write_log(
'{0} #{1}: all sub-tasks have been processed successfully'
.format(main_task.name, main_task_id))
else:
main_task.write_log(
'{0} #{1}: some/all of sub-tasks have been crashed'.
format(main_task.name, main_task_id),
level='error')
except:
import logging
logging.error(
'Was unable to log SUCCESS/FAILURE to task log. Task id: {0}'
.format(main_task_id))
if total_status_propagating_exceptions in PROPAGATE_STATES:
revoke_task(AsyncResult(main_task_id))
