def test_restore_current_app_fallback(self):
subs = [MockAsyncResultSuccess(uuid(), app=self.app)]
ts = self.app.GroupResult(uuid(), subs)
ts.save()
with pytest.raises(RuntimeError,
message="Test depends on current_app"):
GroupResult.restore(ts.id)
def test_restore_current_app_fallback(self):
subs = [MockAsyncResultSuccess(uuid(), app=self.app)]
ts = self.app.GroupResult(uuid(), subs)
ts.save()
with pytest.raises(RuntimeError,
message="Test depends on current_app"):
GroupResult.restore(ts.id)
def test_save_restore(self):
subs = [MockAsyncResultSuccess(uuid()), MockAsyncResultSuccess(uuid())]
ts = GroupResult(uuid(), subs)
ts.save()
with self.assertRaises(AttributeError):
ts.save(backend=object())
self.assertEqual(GroupResult.restore(ts.id).subtasks, ts.subtasks)
ts.delete()
self.assertIsNone(GroupResult.restore(ts.id))
def test_save_restore(self):
subs = [MockAsyncResultSuccess(uuid()),
MockAsyncResultSuccess(uuid())]
ts = GroupResult(uuid(), subs)
ts.save()
with self.assertRaises(AttributeError):
ts.save(backend=object())
self.assertEqual(GroupResult.restore(ts.id).subtasks,
ts.subtasks)
ts.delete()
self.assertIsNone(GroupResult.restore(ts.id))
def cancel(request, instance, import_type, import_event_id):
ie = _get_import_event(instance, import_type, import_event_id)
ie.status = GenericImportEvent.CANCELED
ie.save()
# If verifications tasks are still scheduled, we need to revoke them
if ie.task_id:
GroupResult.restore(ie.task_id).revoke()
return list_imports(request, instance)
def cancel(request, instance, import_type, import_event_id):
ie = _get_import_event(instance, import_type, import_event_id)
ie.status = GenericImportEvent.CANCELED
ie.save()
# If verifications tasks are still scheduled, we need to revoke them
if ie.task_id:
GroupResult.restore(ie.task_id).revoke()
return list_imports(request, instance)
def handle(self, *args, **kwargs): # pylint: disable=unused-argument
edx_course_key = kwargs.get('edx_course_key')
try:
run = CourseRun.objects.get(edx_course_key=edx_course_key)
except CourseRun.DoesNotExist:
raise CommandError('Course Run for course_id "{0}" does not exist'.format(edx_course_key))
con = get_redis_connection("redis")
failed_users_count = con.llen(CACHE_KEY_FAILED_USERS_BASE_STR.format(edx_course_key))
if CourseRunGradingStatus.is_complete(run):
self.stdout.write(
self.style.SUCCESS(
'Final grades for course "{0}" are complete'.format(edx_course_key)
)
)
elif CourseRunGradingStatus.is_pending(run):
cache_id = CACHE_ID_BASE_STR.format(edx_course_key)
group_results_id = cache_redis.get(cache_id)
if group_results_id is not None:
results = GroupResult.restore(group_results_id, app=app)
if not results.ready():
self.stdout.write(
self.style.WARNING(
'Final grades for course "{0}" are being processed'.format(edx_course_key)
)
)
else:
self.stdout.write(
self.style.WARNING(
'Async task to freeze grade for course "{0}" '
'are done, but course is not marked as complete.'.format(edx_course_key)
)
)
else:
self.stdout.write(
self.style.ERROR(
'Final grades for course "{0}" are marked as they are being processed'
', but no task found.'.format(edx_course_key)
)
)
else:
self.stdout.write(
self.style.WARNING(
'Final grades for course "{0}" are not being processed yet'.format(edx_course_key)
)
)
message_detail = ', where {0} failed authentication'.format(failed_users_count) if failed_users_count else ''
users_in_cache = set(CachedEnrollment.get_cached_users(run)).intersection(
set(CachedCurrentGrade.get_cached_users(run))
)
self.stdout.write(
self.style.SUCCESS(
'The students with a final grade are {0}/{1}{2}'.format(
FinalGrade.objects.filter(course_run=run).count(),
len(users_in_cache),
message_detail
)
)
)
def poll_state(request):
""" A view to report the progress to the user """
data = 'Fail'
if request.is_ajax():
if 'task_id' in request.POST.keys() and request.POST['task_id']:
task_id = request.POST['task_id']
task_total = request.POST['task_total']
task = GroupResult.restore(task_id, app=app)
progress = task.completed_count()/float(task_total)
if progress <= 0:
progress = 0.01
data = {}
if progress >= 1.0 and request.session.get('saved') == None:
request.session['saved'] = 'saved'
progress = None
result = task.get()
task.forget()
uuid = request.POST['product_uuid']
p = Product.objects.get(uuid=uuid)
historic_id = save_product_indexing(result, p)
data['historic_id'] = urlsafe_base64_encode(force_bytes(historic_id))
data['uuid'] = str(p.uuid)
del request.session['saved']
data['process_percent'] = progress
else:
data = 'No task_id in the request'
else:
data = 'This is not an ajax request'
if isinstance(data, dict):
data = json.dumps(data)
return JsonResponse(data, safe=False)
def on_chord_part_return(self, task, propagate=True):
if not self.implements_incr:
return
from celery import subtask
from celery.result import GroupResult
gid = task.request.group
if not gid:
return
key = self.get_key_for_chord(gid)
deps = GroupResult.restore(gid, backend=task.backend)
val = self.incr(key)
if val >= len(deps):
j = deps.join_native if deps.supports_native_join else deps.join
callback = subtask(task.request.chord)
try:
ret = j(propagate=propagate)
except Exception as exc:
culprit = next(deps._failed_join_report())
self.app._tasks[callback.task].backend.fail_from_current_stack(
callback.id, exc=ChordError('Dependency %s raised %r' % (
culprit.id, exc))
)
else:
callback.delay(ret)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def on_chord_part_return(self, task, state, result, propagate=None):
if not self.implements_incr:
return
app = self.app
if propagate is None:
propagate = app.conf.CELERY_CHORD_PROPAGATES
gid = task.request.group
if not gid:
return
key = self.get_key_for_chord(gid)
try:
deps = GroupResult.restore(gid, backend=task.backend)
except Exception as exc:
callback = maybe_signature(task.request.chord, app=app)
return self.chord_error_from_stack(
callback,
ChordError('Cannot restore group: {0!r}'.format(exc)),
)
if deps is None:
try:
raise ValueError(gid)
except ValueError as exc:
callback = maybe_signature(task.request.chord, app=app)
return self.chord_error_from_stack(
callback,
ChordError('GroupResult {0} no longer exists'.format(gid)),
)
val = self.incr(key)
if val >= len(deps):
callback = maybe_signature(task.request.chord, app=app)
j = deps.join_native if deps.supports_native_join else deps.join
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
self.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
self.chord_error_from_stack(
callback,
ChordError('Callback error: {0!r}'.format(exc)),
)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def on_chord_part_return(self, task, state, result, propagate=None):
if not self.implements_incr:
return
app = self.app
if propagate is None:
propagate = app.conf.CELERY_CHORD_PROPAGATES
gid = task.request.group
if not gid:
return
key = self.get_key_for_chord(gid)
try:
deps = GroupResult.restore(gid, backend=task.backend)
except Exception as exc:
callback = maybe_signature(task.request.chord, app=app)
return self.chord_error_from_stack(
callback,
ChordError('Cannot restore group: {0!r}'.format(exc)),
)
if deps is None:
try:
raise ValueError(gid)
except ValueError as exc:
callback = maybe_signature(task.request.chord, app=app)
return self.chord_error_from_stack(
callback,
ChordError('GroupResult {0} no longer exists'.format(gid)),
)
val = self.incr(key)
if val >= len(deps):
callback = maybe_signature(task.request.chord, app=app)
j = deps.join_native if deps.supports_native_join else deps.join
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit, exc,
)
except StopIteration:
reason = repr(exc)
self.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
self.chord_error_from_stack(
callback,
ChordError('Callback error: {0!r}'.format(exc)),
)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def post(self, request):
task = request.POST['task_id']
res = GroupResult.restore(task)
if res and not res.ready():
return HttpResponse(json.dumps({"status": "loading"}), content_type="application/json")
# Task completion allows for origin information to be pulled
try:
task_origin = TaskTracker.objects.get(group_id=task)
record_type = task_origin.type
indicator = task_origin.keyword
except MultipleObjectsReturned:
task_origin = TaskTracker.objects.filter(group_id=task).latest('date')
record_type = task_origin.type
indicator = task_origin.keyword
except ObjectDoesNotExist:
record_type = None
indicator = None
# Pull data according to the record type
if record_type == "Recent":
self.template_name = "pivoteer/RecentRecords.html"
# Current hosting records
host_record = IndicatorRecord.objects.recent_hosts(indicator)
self.template_vars["current_hosts"] = host_record
# Current WHOIS record
whois_record = IndicatorRecord.objects.recent_whois(indicator)
self.template_vars["current_whois"] = whois_record
elif record_type == "Historical":
self.template_name = "pivoteer/HistoricalRecords.html"
# Historical hosting records
host_records = IndicatorRecord.objects.historical_hosts(indicator, request)
self.template_vars["hosting_records"] = host_records
# Historical WHOIS records
whois_record = IndicatorRecord.objects.historical_whois(indicator)
self.template_vars["historical_whois"] = whois_record
elif record_type == "Malware":
self.template_name = "pivoteer/MalwareRecords.html"
malware_records = IndicatorRecord.objects.malware_records(indicator)
self.template_vars["malware_records"] = malware_records
self.template_vars["origin"] = indicator
return render(request, self.template_name, self.template_vars)
def group_status(request, group_id):
"""Return group id and its async results status & result in JSON format."""
result = GroupResult.restore(group_id)
retval = [{
"result": async_result.result,
"status": async_result.status
} for async_result in result.results]
response_data = {'id': group_id, 'results': retval}
return JsonResponse({'group': response_data})
def test_empty_group_result(self, manager):
if not manager.app.conf.result_backend.startswith('redis'):
raise pytest.skip('Requires redis result backend.')
task = group([])
result = task.apply_async()
GroupResult.save(result)
task = GroupResult.restore(result.id)
assert task.results == []
def test_empty_group_result(self, manager):
if not manager.app.conf.result_backend.startswith('redis'):
raise pytest.skip('Requires redis result backend.')
task = group([])
result = task.apply_async()
GroupResult.save(result)
task = GroupResult.restore(result.id)
assert task.results == []
def handle(self, *args, **kwargs): # pylint: disable=unused-argument
edx_course_key = kwargs.get('edx_course_key')
try:
run = CourseRun.objects.get(edx_course_key=edx_course_key)
except CourseRun.DoesNotExist:
raise CommandError(
'Course Run for course_id "{0}" does not exist'.format(
edx_course_key))
con = get_redis_connection("redis")
failed_users_count = con.llen(
CACHE_KEY_FAILED_USERS_BASE_STR.format(edx_course_key))
if CourseRunGradingStatus.is_complete(run):
self.stdout.write(
self.style.SUCCESS(
'Final grades for course "{0}" are complete'.format(
edx_course_key)))
elif CourseRunGradingStatus.is_pending(run):
cache_id = CACHE_ID_BASE_STR.format(edx_course_key)
group_results_id = cache_redis.get(cache_id)
if group_results_id is not None:
results = GroupResult.restore(group_results_id, app=app)
if not results.ready():
self.stdout.write(
self.style.WARNING(
'Final grades for course "{0}" are being processed'
.format(edx_course_key)))
else:
self.stdout.write(
self.style.WARNING(
'Async task to freeze grade for course "{0}" '
'are done, but course is not marked as complete.'.
format(edx_course_key)))
else:
self.stdout.write(
self.style.ERROR(
'Final grades for course "{0}" are marked as they are being processed'
', but no task found.'.format(edx_course_key)))
else:
self.stdout.write(
self.style.WARNING(
'Final grades for course "{0}" are not being processed yet'
.format(edx_course_key)))
message_detail = ', where {0} failed authentication'.format(
failed_users_count) if failed_users_count else ''
users_in_cache = set(
CachedEnrollment.get_cached_users(run)).intersection(
set(CachedCurrentGrade.get_cached_users(run)))
self.stdout.write(
self.style.SUCCESS(
'The students with a final grade are {0}/{1}{2}'.format(
FinalGrade.objects.filter(course_run=run).count(),
len(users_in_cache), message_detail)))
def collect_stats_transcoding(group_id):
"""
Collect statistics about the given transcoding group job.
"""
group = GroupResult.restore(group_id)
data = {}
data['state'] = 'TRANSCODING'
data['current'] = group.completed_count()
data['total'] = len(group)
return data, group.ready()
def post(self, request):
desired_time = time_jump(hours=-24)
task = request.POST['task_id']
res = GroupResult.restore(task)
if res and not res.ready():
return HttpResponse(json.dumps({"status": "loading"}), content_type="application/json")
# Task completion allows for origin information to be pulled
try:
task_origin = TaskTracker.objects.get(group_id=task)
record_type = task_origin.type
indicator = task_origin.keyword
except MultipleObjectsReturned:
task_origin = TaskTracker.objects.filter(group_id=task).latest('date')
record_type = task_origin.type
indicator = task_origin.keyword
except ObjectDoesNotExist:
record_type = None
indicator = None
# Pull data according to the record type
if record_type == "current":
# Collect whois record for current records
whois_record = WhoisRecord.objects.recent_record(indicator)
self.template_vars["whois_record"] = whois_record
# Collect host records for current records
host_record = HostRecord.objects.current_hosts(indicator, desired_time)
self.template_vars["host_record"] = host_record
self.template_name = "pivoteer/CurrentRecords.html"
elif record_type == "passive":
host_records = HostRecord.objects.passive_records(indicator, request)
self.template_vars["passive_records"] = host_records
self.template_name = "pivoteer/PassiveRecords.html"
elif record_type == "malware":
malware_records = MalwareRecord.objects.malware_records(indicator)
self.template_vars["malware_records"] = malware_records
self.template_name = "pivoteer/MalwareRecords.html"
elif record_type == "other":
google_search = SearchEngineHits.objects.recent_record(indicator)
self.template_vars["google_search"] = google_search
self.template_name = "pivoteer/OtherRecords.html"
return render(request, self.template_name, self.template_vars)
def on_chord_part_return(self, request, state, result, **kwargs):
if not self.implements_incr:
return
app = self.app
gid = request.group
if not gid:
return
key = self.get_key_for_chord(gid)
try:
deps = GroupResult.restore(gid, backend=self)
except Exception as exc:
callback = maybe_signature(request.chord, app=app)
logger.error("Chord %r raised: %r", gid, exc, exc_info=1)
return self.chord_error_from_stack(callback, ChordError("Cannot restore group: {0!r}".format(exc)))
if deps is None:
try:
raise ValueError(gid)
except ValueError as exc:
callback = maybe_signature(request.chord, app=app)
logger.error("Chord callback %r raised: %r", gid, exc, exc_info=1)
return self.chord_error_from_stack(callback, ChordError("GroupResult {0} no longer exists".format(gid)))
val = self.incr(key)
size = len(deps)
if val > size: # pragma: no cover
logger.warning("Chord counter incremented too many times for %r", gid)
elif val == size:
callback = maybe_signature(request.chord, app=app)
j = deps.join_native if deps.supports_native_join else deps.join
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=True)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = "Dependency {0.id} raised {1!r}".format(culprit, exc)
except StopIteration:
reason = repr(exc)
logger.error("Chord %r raised: %r", gid, reason, exc_info=1)
self.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
logger.error("Chord %r raised: %r", gid, exc, exc_info=1)
self.chord_error_from_stack(callback, ChordError("Callback error: {0!r}".format(exc)))
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def on_chord_part_return(self, task, propagate=None):
if not self.implements_incr:
return
from celery import subtask
from celery.result import GroupResult
app = self.app
if propagate is None:
propagate = self.app.conf.CELERY_CHORD_PROPAGATES
gid = task.request.group
if not gid:
return
key = self.get_key_for_chord(gid)
deps = GroupResult.restore(gid, backend=task.backend)
if deps is None:
callback = subtask(task.request.chord)
return app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('GroupResult {0} no longer exists'.format(gid))
)
val = self.incr(key)
if val >= len(deps):
callback = subtask(task.request.chord)
j = deps.join_native if deps.supports_native_join else deps.join
try:
ret = j(propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit, exc,
)
except StopIteration:
reason = repr(exc)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id, exc=ChordError(reason),
)
else:
try:
callback.delay(ret)
except Exception as exc:
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def get(self, request, *args, **kwargs):
task_id = kwargs.get('pk')
if not task_id:
return Response(status=status.HTTP_400_BAD_REQUEST)
group_result = GroupResult.restore(task_id)
if not group_result:
return Response(status=status.HTTP_404_NOT_FOUND)
if not group_result.ready():
return Response({
"status": settings.PROGRESS,
"progress": {
"count": group_result.completed_count(),
"total": len(group_result)
}
})
else:
commit_if_not_yet(group_result)
succeeded = {}
failed = {}
failed_count = 0
succeeded_count = 0
if group_result.supports_native_join:
results = group_result.join_native()
else:
results = group_result.join()
for res in results:
task_stat, man_id, rem_url, errors, warnings = res
if task_stat == settings.SUCCESS:
succeeded_count += 1
succeeded[rem_url] = {
'url': '/manifests/' + man_id,
'warnings': warnings
}
else:
failed_count += 1
failed[rem_url] = {'errors': errors}
d = {
'succeeded': succeeded,
'succeeded_count': succeeded_count,
'failed': failed,
'failed_count': failed_count,
'total_count': len(group_result),
'status': settings.SUCCESS
}
return Response(d)
def is_group_successful(request, group_id):
"""Return if group was successfull as boolean."""
results = GroupResult.restore(group_id)
return JsonResponse({
'group': {
'id':
group_id,
'results': [{
'id': task.id,
'executed': task.successful()
} for task in results] if results else []
}
})
def process(self, taskid, format=None):
"process a taskset"
result = GroupResult.restore(taskid, backend=RBACKEND)
if (result is None or 'taskids' not in session
or taskid not in session['taskids']):
if format == 'json':
return ajax_code(
404,
_('The task status requested '
'has expired or does not exist'))
flash(_('The task status requested has expired or does not exist'))
redirect(url(controller='messages', action='quarantine'))
percent = "0.0"
status = 'PROGRESS'
results = []
if result.ready():
finished = True
results = result.join_native()
else:
session['bulkprocess-count'] += 1
if (session['bulkprocess-count'] >= 10
and result.completed_count() == 0):
result.revoke()
del session['bulkprocess-count']
session.save()
if format == 'json':
return ajax_code(
503,
_('An error occured in processing, try again later'))
flash_alert(
_('An error occured in processing, try again later'))
redirect(url(controller='messages', action='quarantine'))
finished = False
percent = "%.1f" % (
(1.0 * int(result.completed_count()) / len(result)) * 100)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = dict(finished=finished,
results=results,
status=status,
completed=percent)
return json.dumps(data)
c.finished = finished
c.results = results
c.status = status
c.completed = percent
return self.render('/messages/taskstatus.html')
def cancel(request, instance, import_type, import_event_id):
ie = _get_import_event(instance, import_type, import_event_id)
ie.status = GenericImportEvent.CANCELED
ie.mark_finished_and_save()
# If verifications tasks are still scheduled, we need to revoke them
if ie.task_id:
result = GroupResult.restore(ie.task_id)
if result:
result.revoke()
# If we couldn't get the task, it is already effectively cancelled
return list_imports(request, instance)
def cancel(request, instance, import_type, import_event_id):
ie = _get_import_event(instance, import_type, import_event_id)
ie.status = GenericImportEvent.CANCELED
ie.mark_finished_and_save()
# If verifications tasks are still scheduled, we need to revoke them
if ie.task_id:
result = GroupResult.restore(ie.task_id)
if result:
result.revoke()
# If we couldn't get the task, it is already effectively cancelled
return list_imports(request, instance)
def process(self, taskid, format=None):
"process a taskset"
result = GroupResult.restore(taskid, backend=RBACKEND)
if (result is None or
'taskids' not in session or
taskid not in session['taskids']):
if format == 'json':
return ajax_code(404,
_('The task status requested '
'has expired or does not exist'))
flash(_('The task status requested has expired or does not exist'))
redirect(url(controller='messages', action='quarantine'))
percent = "0.0"
status = 'PROGRESS'
results = []
if result.ready():
finished = True
results = result.join_native()
else:
session['bulkprocess-count'] += 1
if (session['bulkprocess-count'] >= 10 and
result.completed_count() == 0):
result.revoke()
del session['bulkprocess-count']
session.save()
if format == 'json':
return ajax_code(503,
_('An error occured in processing, try again later'))
flash_alert(
_('An error occured in processing, try again later'))
redirect(url(controller='messages', action='quarantine'))
finished = False
percent = "%.1f" % ((1.0 * int(result.completed_count()) /
len(result)) * 100)
if format == 'json':
response.headers['Content-Type'] = 'application/json'
data = dict(finished=finished,
results=results,
status=status,
completed=percent)
return json.dumps(data)
c.finished = finished
c.results = results
c.status = status
c.completed = percent
return self.render('/messages/taskstatus.html')
def test_relister_instance(lister, swh_app, celery_session_worker):
# setup the mocked GitlabLister
lister.return_value = lister
lister.run.return_value = None
lister.get_pages_information.return_value = (None, 85, None)
lister.db_partition_indices.return_value = [
(i, i + 9) for i in range(0, 80, 10)
] + [(80, 85)]
res = swh_app.send_task(
'swh.lister.gitlab.tasks.FullGitLabRelister',
kwargs=dict(api_baseurl='https://0xacab.org/api/v4'))
assert res
res.wait()
assert res.successful()
# retrieve the GroupResult for this task and wait for all the subtasks
# to complete
promise_id = res.result
assert promise_id
promise = GroupResult.restore(promise_id, app=swh_app)
for i in range(5):
if promise.ready():
break
sleep(1)
lister.assert_called_with(api_baseurl='https://0xacab.org/api/v4',
instance=None,
sort='asc',
per_page=20)
# one by the FullGitlabRelister task
# + 9 for the RangeGitlabLister subtasks
assert lister.call_count == 10
lister.db_last_index.assert_not_called()
lister.db_partition_indices.assert_not_called()
lister.get_pages_information.assert_called_once_with()
# lister.run should have been called once per partition interval
for i in range(8):
# XXX inconsistent behavior: max_bound is EXCLUDED here
assert (dict(min_bound=10*i, max_bound=10*i + 10),) \
in lister.run.call_args_list
assert (dict(min_bound=80, max_bound=85),) \
in lister.run.call_args_list
def on_chord_part_return(self, task, propagate=False):
if not self.implements_incr:
return
from celery import subtask
from celery.result import GroupResult
gid = task.request.group
if not gid:
return
key = self.get_key_for_chord(gid)
deps = GroupResult.restore(gid, backend=task.backend)
val = self.incr(key)
if val >= len(deps):
subtask(task.request.chord).delay(deps.join(propagate=propagate))
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def on_chord_part_return(self, task, propagate=None):
if not self.implements_incr:
return
from celery import subtask
from celery.result import GroupResult
app = self.app
if propagate is None:
propagate = self.app.conf.CELERY_CHORD_PROPAGATES
gid = task.request.group
if not gid:
return
key = self.get_key_for_chord(gid)
deps = GroupResult.restore(gid, backend=task.backend)
val = self.incr(key)
if val >= len(deps):
j = deps.join_native if deps.supports_native_join else deps.join
callback = subtask(task.request.chord)
try:
ret = j(propagate=propagate)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError(reason),
)
else:
try:
callback.delay(ret)
except Exception as exc:
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Callback error: {0!r}'.format(exc)),
)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def on_chord_part_return(self, task, propagate=False):
if not self.implements_incr:
return
from celery import subtask
from celery.result import GroupResult
gid = task.request.group
if not gid:
return
key = self.get_key_for_chord(gid)
deps = GroupResult.restore(gid, backend=task.backend)
val = self.incr(key)
if val >= len(deps):
subtask(task.request.chord).delay(deps.join(propagate=propagate))
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def group_task_progress_bar(request, group_id, should_update_version):
g_res = GroupResult.restore(group_id)
if g_res is None:
return JsonResponse({"detail": "Notfound"}, status=404)
if g_res.waiting():
done_count = len(g_res) - [res.state for res in g_res].count("PENDING")
percent_str = "%.2f" % (done_count / len(g_res))
data = {"percent": percent_str}
return JsonResponse(data)
# Summarize the results and send
key = str(g_res.id) + "_timestart"
try:
cache = redis.Redis()
delta = (
(
datetime.now(timezone.utc)
- datetime.strptime(cache.get(key).decode(), "%Y-%m-%d %H:%M:%S.%f %z")
).seconds
if cache.exists(key)
else "Not calculated"
)
results = g_res.join_native(propagate=False)
states = [res.state for res in g_res]
errors = [str(res) for res in results if isinstance(res, Exception)]
results = [str(res) for res in results if not isinstance(res, Exception)]
data = {
"percent": 1,
"successful": states.count("SUCCESS"),
"failed": states.count("FAILURE"),
"exec_time": delta,
"error_messages": [
{"type": error_type, "num": errors.count(error_type)}
for error_type in set(errors)
],
"results": [
{"type": result_type, "num": results.count(result_type)}
for result_type in set(results)
],
}
finally:
if should_update_version:
Bucket().update_version()
cache.delete(key)
g_res.forget()
return JsonResponse(data, safe=False)
def cancel_task(taskid):
"""
Cancel a task depending on taskid parameter.
The taskid can be a list of subtasks id's or
the gruoptask id.
:param taskid: Task id.
"""
if isinstance(taskid, list):
# taskid is a subtasks id's list
for stask in taskid:
logger.info("Cancelling subtask: {0}".format(stask))
AsyncResult(stask).revoke(terminate=True, signal="SIGKILL")
else:
# taskid is a GroupResult
logger.info("Canceling group task: {0}".format(taskid))
gtask = GroupResult.restore(taskid)
gtask.revoke(terminate=True, signal="SIGKILL")
def get_task_status(task_id):
status = ''
task_status = (GroupResult.restore(task_id, backend=app_celery.backend)
or AsyncResult(task_id, backend=app_celery.backend))
if ((isinstance(task_status, GroupResult) and task_status.waiting())
or (isinstance(task_status, AsyncResult)
and task_status.status == 'PENDING')):
status = 'WAITING/PENDING'
elif ((isinstance(task_status, GroupResult) and task_status.successful())
or (isinstance(task_status, AsyncResult)
and task_status.status == 'SUCCESS')):
status = 'SUCCESS'
elif ((isinstance(task_status, GroupResult) and task_status.failed())
or (isinstance(task_status, AsyncResult)
and task_status.status == 'FAILURE')):
status = 'FAILURE'
return status
def handle(self, *args, **kwargs): # pylint: disable=unused-argument
edx_course_key = kwargs.get('edx_course_key')
try:
run = CourseRun.objects.get(edx_course_key=edx_course_key)
except CourseRun.DoesNotExist:
raise CommandError('Course Run for course_id "{0}" does not exist'.format(edx_course_key))
if not run.can_freeze_grades:
self.stdout.write(
self.style.ERROR(
'Course Run "{0}" cannot be marked as frozen yet'.format(edx_course_key)
)
)
return
if CourseRunGradingStatus.is_complete(run):
self.stdout.write(
self.style.SUCCESS(
'Course Run "{0}" is already marked as complete'.format(edx_course_key)
)
)
return
# check if there are tasks running
cache_id = CACHE_ID_BASE_STR.format(edx_course_key)
group_results_id = cache_redis.get(cache_id)
if group_results_id is not None:
results = GroupResult.restore(group_results_id, app=app)
if results and not results.ready():
self.stdout.write(
self.style.WARNING(
'Tasks for Course Run "{0}" are still running. '
'Impossible to set the global "complete" status'.format(edx_course_key)
)
)
return
# if the tasks are done remove the entry in the cache
cache_redis.delete(group_results_id)
CourseRunGradingStatus.set_to_complete(run)
self.stdout.write(
self.style.SUCCESS(
'Course Run "{0}" has been marked as complete'.format(edx_course_key)
)
)
def status(group_id):
result = GroupResult.restore(group_id)
if not result:
return json.dumps({
'error': 'Does not exist'
})
return json.dumps({
'tasks': [
{
'id': task.id,
'status': task.status,
'ready': task.ready(),
'result': task.result
} for task in result.children
],
'ready': result.ready(),
'completed': result.completed_count()
})
def stop_jobs(jobs: List[Job], job_options: Dict[str, Any]):
raise NotImplementedError
# TODO: not sure whether we should .revoke(terminate=True)
# TODO: one of the options below should work, but I don't have time to test it right now...
# http://docs.celeryproject.org/en/latest/userguide/workers.html#worker-persistent-revokes
from celery.result import AsyncResult
AsyncResult(job_options['command_id']).revoke()
# https://docs.celeryproject.org/en/stable/reference/celery.result.html
from celery.result import GroupResult
g = GroupResult(id=job_options['command_id'])
# https://stackoverflow.com/questions/13685344/retrieving-groupresult-from-taskset-id-in-celery
# We may need to call result.save() in the task above for it to work...
from celery.result import GroupResult
result = GroupResult.restore(job_options['command_id'])
result.revoke()
from celery.task.control import revoke
revoke(job_options['command_id'])
def get(self, bulk_task_id):
if not bulk_task_id:
raise BadRequest('The bulk_task_id is not supplied.')
group_result = GroupResult.restore(bulk_task_id)
task_url = app.api.url_for(TaskSubscriptionAPI,
bulk_task_id=str(group_result.id),
_external=True)
successful = all_finished(group_result.results)
task_results = {}
if successful:
task_results = {
'task_results': get_all_task_results(group_result.results)
}
return {
'status': 'SUCCESS' if successful else 'PENDING',
'bulk_task_id': str(group_result.id),
'bulk_task_url': task_url,
'completed_count': group_result.completed_count(),
'total_count': len(group_result.results),
**task_results
}
def test_multiping(swh_scheduler_celery_app, swh_scheduler_celery_worker):
"Test that a task that spawns subtasks (group) works"
res = swh_scheduler_celery_app.send_task(TASK_MULTIPING, kwargs={"n": 5})
assert res
res.wait()
assert res.successful()
# retrieve the GroupResult for this task and wait for all the subtasks
# to complete
promise_id = res.result
assert promise_id
promise = GroupResult.restore(promise_id, app=swh_scheduler_celery_app)
for i in range(5):
if promise.ready():
break
sleep(1)
results = [x.get() for x in promise.results]
assert len(results) == 5
for i in range(5):
assert ("OK (kw={'i': %s})" % i) in results
def get_status(request, group_task_id):
"""
Fetches the GroupResult object for the celery tasks and checks whether or not they were ALL successful.
** Path Parameters **
The unique identifier of the celery group task
** Return Value **
{
"group_task_id": unique identifier of the celery group task,
"group_task_success": truthly value of whether or not the tasks have completed
}
"""
group_task_result = GroupResult.restore(id=group_task_id, app=app)
all_successful = str((group_task_result.successful()))
result = {
"group_task_id": group_task_id,
"group_task_success": all_successful
}
return JsonResponse(result, status=200)
def test_lister_full(lister, swh_scheduler_celery_app,
swh_scheduler_celery_worker):
last_index = 1000000
expected_bounds = list(range(0, last_index + 1, 100000))
if expected_bounds[-1] != last_index:
expected_bounds.append(last_index)
# setup the mocked GitHubLister
lister.state = GitHubListerState(last_seen_id=last_index)
lister.from_configfile.return_value = lister
lister.run.return_value = ListerStats(pages=10, origins=10000)
res = swh_scheduler_celery_app.send_task(
"swh.lister.github.tasks.FullGitHubRelister")
assert res
res.wait()
assert res.successful()
# retrieve the GroupResult for this task and wait for all the subtasks
# to complete
promise_id = res.result
assert promise_id
promise = GroupResult.restore(promise_id, app=swh_scheduler_celery_app)
for i in range(5):
if promise.ready():
break
sleep(1)
# pulling the state out of the database
assert lister.from_configfile.call_args_list[0] == call()
# Calls for each of the ranges
range_calls = lister.from_configfile.call_args_list[1:]
# Check exhaustivity of the range calls
assert sorted(range_calls, key=lambda c: c[1]["first_id"]) == [
call(first_id=f, last_id=l)
for f, l in zip(expected_bounds[:-1], expected_bounds[1:])
]
def get(self, request, *args, **kwargs):
task_id = kwargs.get('pk')
if not task_id:
return Response(status=status.HTTP_400_BAD_REQUEST)
group_result = GroupResult.restore(task_id)
if not group_result:
return Response(status=status.HTTP_404_NOT_FOUND)
if not group_result.ready():
return Response({"status": settings.PROGRESS,
"progress": {"count": group_result.completed_count(),
"total": len(group_result)}})
else:
succeeded = {}
failed = {}
failed_count = 0
succeeded_count = 0
if group_result.supports_native_join:
results = group_result.join_native()
else:
results = group_result.join()
for res in results:
task_stat, man_id, rem_url, errors, warnings = res
if task_stat == settings.SUCCESS:
succeeded_count += 1
succeeded[rem_url] = {'url': '/manifests/'+man_id,
'warnings': warnings}
else:
failed_count += 1
failed[rem_url] = {'errors': errors}
d = {'succeeded': succeeded, 'succeeded_count': succeeded_count,
'failed': failed, 'failed_count': failed_count,
'total_count': len(group_result), 'status': settings.SUCCESS}
return Response(d)
def get_task_result(task_id):
g_result = GroupResult.restore(task_id, app=celery_)
if g_result and g_result.successful():
jvec = []
jmod = []
jfoc = []
#通过get()取值只能够取一次，多次会阻塞(可能是设置问题)，没时间查找其它的方式，目前先把取出的值放入到redis中，然后从redis中再进行取值
nlist = None
if r_redis.get("result_" + task_id):
nlist = json.loads(r_redis.get("result_" + task_id))
else:
nlist = g_result.get()
r_redis.set("result_" + task_id, json.dumps(nlist), ex=10800)
for avec, amod, afoc in nlist:
jvec = jvec + avec
jmod = jmod + amod
jfoc = jfoc + afoc
dvec = {}
dmod = {}
dfoc = {}
dvec = tfner.stat_element_num(jvec)
dmod = tfner.stat_element_num(jmod)
dfoc = tfner.stat_element_num(jfoc)
matedata = {}
matedata["VEC"] = dvec
matedata["MOD"] = dmod
matedata["FOC"] = dfoc
r_vel = json.loads(r_redis.get(task_id))
r_vel["matedate"] = matedata
ret = result_json(task_id, "complted")
ret["result"] = r_vel
return json.dumps(ret, ensure_ascii=False)
elif not g_result:
return json.dumps(result_json(task_id, "NO_TASK"), ensure_ascii=False)
else:
return json.dumps(result_json(task_id, "Waitting"), ensure_ascii=False)
def test_relister(lister, swh_app, celery_session_worker):
# setup the mocked GitHubLister
lister.return_value = lister
lister.run.return_value = None
lister.db_partition_indices.return_value = [(i, i + 9)
for i in range(0, 50, 10)]
res = swh_app.send_task(
'swh.lister.launchpad.tasks.FullLaunchpadGitRelister')
assert res
res.wait()
assert res.successful()
# retrieve the GroupResult for this task and wait for all the subtasks
# to complete
promise_id = res.result
assert promise_id
promise = GroupResult.restore(promise_id, app=swh_app)
for i in range(5):
if promise.ready():
break
sleep(1)
lister.assert_called_with(api_baseurl='https://api.launchpad.net/devel')
# one by the FullLaunchpadGitRelister task
# + 5 for the RangeLaunchpadGitLister subtasks
assert lister.call_count == 6
lister.db_last_index.assert_not_called()
lister.db_partition_indices.assert_called_once_with(1000)
# lister.run should have been called once per partition interval
for i in range(5):
assert (dict(min_bound=10*i, max_bound=10*i + 9),) \
in lister.run.call_args_list
def on_chord_part_return(self, task, propagate=None):
if not self.implements_incr:
return
from celery import subtask
from celery.result import GroupResult
app = self.app
if propagate is None:
propagate = self.app.conf.CELERY_CHORD_PROPAGATES
gid = task.request.group
if not gid:
return
key = self.get_key_for_chord(gid)
deps = GroupResult.restore(gid, backend=task.backend)
val = self.incr(key)
if val >= len(deps):
j = deps.join_native if deps.supports_native_join else deps.join
callback = subtask(task.request.chord)
try:
ret = j(propagate=propagate)
except Exception, exc:
try:
culprit = deps._failed_join_report().next()
reason = 'Dependency %s raised %r' % (culprit.id, exc)
except StopIteration:
reason = repr(exc)
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id, exc=ChordError(reason),
)
else:
try:
callback.delay(ret)
except Exception, exc:
app._tasks[callback.task].backend.fail_from_current_stack(
callback.id,
exc=ChordError('Callback error: %r' % (exc, )),
)
def on_chord_part_return(self, request, state, result, **kwargs):
if not self.implements_incr:
return
app = self.app
gid = request.group
if not gid:
return
key = self.get_key_for_chord(gid)
try:
deps = GroupResult.restore(gid, backend=self)
except Exception as exc:
callback = maybe_signature(request.chord, app=app)
logger.error('Chord %r raised: %r', gid, exc, exc_info=1)
return self.chord_error_from_stack(
callback,
ChordError('Cannot restore group: {0!r}'.format(exc)),
)
if deps is None:
try:
raise ValueError(gid)
except ValueError as exc:
callback = maybe_signature(request.chord, app=app)
logger.error('Chord callback %r raised: %r',
gid,
exc,
exc_info=1)
return self.chord_error_from_stack(
callback,
ChordError('GroupResult {0} no longer exists'.format(gid)),
)
val = self.incr(key)
size = len(deps)
if val > size: # pragma: no cover
logger.warning('Chord counter incremented too many times for %r',
gid)
elif val == size:
callback = maybe_signature(request.chord, app=app)
j = deps.join_native if deps.supports_native_join else deps.join
try:
with allow_join_result():
ret = j(timeout=3.0, propagate=True)
except Exception as exc:
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
logger.error('Chord %r raised: %r', gid, reason, exc_info=1)
self.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc:
logger.error('Chord %r raised: %r', gid, exc, exc_info=1)
self.chord_error_from_stack(
callback,
ChordError('Callback error: {0!r}'.format(exc)),
)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, 86400)
def task(self):
from celery.result import GroupResult
result = GroupResult.restore(self.task_id)
return result
def group_status(request, group_id):
"""Return task status and result in JSON format."""
result = GroupResult.restore(group_id)
retval = result.result
response_data = {'id': group_id, 'result': retval}
return JsonResponse({'task': response_data})
def task(self):
from celery.result import GroupResult
result = GroupResult.restore(self.task_id)
return result
def on_chord_part_return(self,
request,
state,
result,
propagate=None,
**kwargs):
app = self.app
tid, gid, group_index = request.id, request.group, request.group_index
if not gid or not tid:
return
if group_index is None:
group_index = '+inf'
client = self.client
jkey = self.get_key_for_group(gid, '.j')
tkey = self.get_key_for_group(gid, '.t')
skey = self.get_key_for_group(gid, '.s')
result = self.encode_result(result, state)
encoded = self.encode([1, tid, state, result])
with client.pipeline() as pipe:
pipeline = (pipe.zadd(jkey, {
encoded: group_index
}).zcount(jkey, "-inf",
"+inf") if self._chord_zset else pipe.rpush(
jkey, encoded).llen(jkey)).get(tkey).get(skey)
if self.expires:
pipeline = pipeline \
.expire(jkey, self.expires) \
.expire(tkey, self.expires) \
.expire(skey, self.expires)
_, readycount, totaldiff, chord_size_bytes = pipeline.execute()[:4]
totaldiff = int(totaldiff or 0)
if chord_size_bytes:
try:
callback = maybe_signature(request.chord, app=app)
total = int(chord_size_bytes) + totaldiff
if readycount == total:
header_result = GroupResult.restore(gid)
if header_result is not None:
# If we manage to restore a `GroupResult`, then it must
# have been complex and saved by `apply_chord()` earlier.
#
# Before we can join the `GroupResult`, it needs to be
# manually marked as ready to avoid blocking
header_result.on_ready()
# We'll `join()` it to get the results and ensure they are
# structured as intended rather than the flattened version
# we'd construct without any other information.
join_func = (header_result.join_native
if header_result.supports_native_join else
header_result.join)
with allow_join_result():
resl = join_func(
timeout=app.conf.result_chord_join_timeout,
propagate=True)
else:
# Otherwise simply extract and decode the results we
# stashed along the way, which should be faster for large
# numbers of simple results in the chord header.
decode, unpack = self.decode, self._unpack_chord_result
with client.pipeline() as pipe:
if self._chord_zset:
pipeline = pipe.zrange(jkey, 0, -1)
else:
pipeline = pipe.lrange(jkey, 0, total)
resl, = pipeline.execute()
resl = [unpack(tup, decode) for tup in resl]
try:
callback.delay(resl)
except Exception as exc: # pylint: disable=broad-except
logger.exception('Chord callback for %r raised: %r',
request.group, exc)
return self.chord_error_from_stack(
callback,
ChordError(f'Callback error: {exc!r}'),
)
finally:
with client.pipeline() as pipe:
pipe \
.delete(jkey) \
.delete(tkey) \
.delete(skey) \
.execute()
except ChordError as exc:
logger.exception('Chord %r raised: %r', request.group, exc)
return self.chord_error_from_stack(callback, exc)
except Exception as exc: # pylint: disable=broad-except
logger.exception('Chord %r raised: %r', request.group, exc)
return self.chord_error_from_stack(
callback,
ChordError(f'Join error: {exc!r}'),
)
def test_restore_app(self):
subs = [MockAsyncResultSuccess(uuid(), app=self.app)]
ts = self.app.GroupResult(uuid(), subs)
ts.save()
restored = GroupResult.restore(ts.id, app=self.app)
assert restored.id == ts.id
def index(request):
"""
Main view.
:param request: Http request
:return: json content.
"""
if request.is_ajax():
if "name" in request.GET:
name = request.GET.get("name")
wnodes = Node.objects.all() if name == "all" else Node.objects.filter(site__sitename__exact=name)
json_ = serializers.serialize("json", wnodes, fields=("hostname", "ip"))
return HttpResponse(json_, content_type="application/json")
elif "selectedhosts" in request.GET:
hosts = request.GET.getlist("selectedhosts")
logger.debug("Hosts: {0}".format(hosts))
rescmd = request.GET.getlist("cmd").pop()
logger.info("Command: {0}".format(rescmd))
grouptask = group(execute_ipmi_command.s(host, rescmd) for host in hosts)()
logger.info("Group task id: {0}".format(grouptask.id))
logger.info("Executing ipmi command")
time.sleep(1)
if grouptask.successful():
result = grouptask.get()
logger.info("Task executed successfully. Getting result.")
return HttpResponse(json.dumps(result), content_type="application/json")
else:
# GroupTask id
request.session["taskid"] = grouptask.id
# Subtasks id's of the GroupTask
request.session[grouptask.id] = [taid.id for taid in grouptask.subtasks]
grouptask.save()
return HttpResponse(json.dumps({}), content_type="application/json")
elif "status" in request.GET:
result = []
gtaskid = request.session.get("taskid")
stids = request.session.get(gtaskid)
for stask in stids:
res = check_subtask_status(stask)
if isinstance(res, dict):
host = res.keys().pop()
if "N/A" in res.get(host).values():
# Task has reached max retries.
logger.error("Max retries exceeded for host: {0}".format(host))
else:
logger.info("Getting result for task: {0}".format(stask))
result.append(res)
stids.remove(stask)
elif res == FAILURE:
logger.info("Removing subtask from list: {0}".format(stask))
stids.remove(stask)
if not result and len(stids) > 0:
logger.info("No subtasks have finished yet")
elif not result and len(stids) == 0:
logger.warning("No more tasks in queue")
else:
logger.info("Subtasks finished: {0} ---- {1}".format(result, len(result)))
logger.info("Subtasks remaining: {0}".format(len(stids)))
if not stids:
result.insert(0, {"status": "complete"})
logger.info("Task executed successfully.")
return HttpResponse(json.dumps(result), content_type="application/json")
request.session[gtaskid] = stids
return HttpResponse(json.dumps(result), content_type="application/json")
elif "cancel" in request.GET:
gtaskid = request.session.get("taskid")
grtask = GroupResult.restore(gtaskid)
subtasks = request.session.get(gtaskid)
if len(grtask.subtasks) == len(subtasks):
cancel_task(gtaskid)
else:
cancel_task(subtasks)
return HttpResponse(json.dumps({}), content_type="application/json")
else:
sites = Site.objects.all()
return render(request, "nodes/index.html", {"listsites": sites})
def post(self, request):
task = request.POST['task_id']
res = GroupResult.restore(task)
if res and not res.ready():
return HttpResponse(json.dumps({"status": "loading"}), content_type="application/json")
# Task completion allows for origin information to be pulled
try:
task_origin = TaskTracker.objects.get(group_id=task)
record_type = task_origin.type
indicator = task_origin.keyword
except MultipleObjectsReturned:
task_origin = TaskTracker.objects.filter(group_id=task).latest('date')
record_type = task_origin.type
indicator = task_origin.keyword
except ObjectDoesNotExist:
record_type = None
indicator = None
# Pull data according to the record type
if record_type == "Recent":
self.template_name = "pivoteer/RecentRecords.html"
# Current hosting records
host_record = IndicatorRecord.objects.recent_hosts(indicator)
# We must lookup the country for each IP address for use in the template.
# We do this outside the task because we don't know the IP addresses until the task completes.
host_records_complete = []
for record in host_record:
info = getattr(record, 'info')
record.location = geolocate_ip(info['ip'])
host_records_complete.append(record)
self.template_vars["current_hosts"] = host_records_complete
# Current WHOIS record
whois_record = IndicatorRecord.objects.recent_whois(indicator)
self.template_vars["current_whois"] = whois_record
# Current ThreatCrowd record
tc_info = IndicatorRecord.objects.recent_tc(indicator)
self.template_vars["tc_info"] = tc_info
cert_info = IndicatorRecord.objects.recent_cert(indicator)
self.template_vars["cert_info"] = cert_info
elif record_type == "Historical":
self.template_name = "pivoteer/HistoricalRecords.html"
# Historical hosting records
host_records = IndicatorRecord.objects.historical_hosts(indicator, request)
# We must lookup the country for each IP address for use in the template.
# We do this outside the task because we don't know the IP addresses until the task completes.
host_records_complete = []
for record in host_records:
info = getattr(record, 'info')
record.location = geolocate_ip(info['ip'])
host_records_complete.append(record)
self.template_vars["hosting_records"] = host_records_complete
# Historical WHOIS records
whois_record = IndicatorRecord.objects.historical_whois(indicator)
self.template_vars["historical_whois"] = whois_record
elif record_type == "Malware":
self.template_name = "pivoteer/MalwareRecords.html"
malware_records = IndicatorRecord.objects.malware_records(indicator)
self.template_vars["malware_records"] = malware_records
self.template_vars["origin"] = indicator
elif record_type == "SafeBrowsing":
safebrowsing_records = IndicatorRecord.objects.safebrowsing_record(indicator)
self.template_name = "pivoteer/Google.html"
self.template_vars["records"] = safebrowsing_records
self.template_vars["google_url"] = settings.GOOGLE_SAFEBROWSING_URL + indicator
self.template_vars["origin"] = indicator
elif record_type == "Search":
self.template_name = "pivoteer/SearchRecords.html"
search_records = IndicatorRecord.objects.get_search_records(indicator)
self.template_vars["search_records"] = search_records
elif record_type == "External":
self.template_name = "pivoteer/ExternalRecords.html"
self.template_vars['indicator'] = indicator
self.template_vars['type'] = discover_type(indicator)
return render(request, self.template_name, self.template_vars)
def on_chord_part_return(self, request, state, result, **kwargs):
if not self.implements_incr:
return
app = self.app
gid = request.group
if not gid:
return
key = self.get_key_for_chord(gid)
try:
deps = GroupResult.restore(gid, backend=self)
except Exception as exc: # pylint: disable=broad-except
callback = maybe_signature(request.chord, app=app)
logger.exception('Chord %r raised: %r', gid, exc)
return self.chord_error_from_stack(
callback,
ChordError(f'Cannot restore group: {exc!r}'),
)
if deps is None:
try:
raise ValueError(gid)
except ValueError as exc:
callback = maybe_signature(request.chord, app=app)
logger.exception('Chord callback %r raised: %r', gid, exc)
return self.chord_error_from_stack(
callback,
ChordError(f'GroupResult {gid} no longer exists'),
)
val = self.incr(key)
# Set the chord size to the value defined in the request, or fall back
# to the number of dependencies we can see from the restored result
size = request.chord.get("chord_size")
if size is None:
size = len(deps)
if val > size: # pragma: no cover
logger.warning('Chord counter incremented too many times for %r',
gid)
elif val == size:
callback = maybe_signature(request.chord, app=app)
j = deps.join_native if deps.supports_native_join else deps.join
try:
with allow_join_result():
ret = j(timeout=app.conf.result_chord_join_timeout,
propagate=True)
except Exception as exc: # pylint: disable=broad-except
try:
culprit = next(deps._failed_join_report())
reason = 'Dependency {0.id} raised {1!r}'.format(
culprit,
exc,
)
except StopIteration:
reason = repr(exc)
logger.exception('Chord %r raised: %r', gid, reason)
self.chord_error_from_stack(callback, ChordError(reason))
else:
try:
callback.delay(ret)
except Exception as exc: # pylint: disable=broad-except
logger.exception('Chord %r raised: %r', gid, exc)
self.chord_error_from_stack(
callback,
ChordError(f'Callback error: {exc!r}'),
)
finally:
deps.delete()
self.client.delete(key)
else:
self.expire(key, self.expires)
def freeze_course_run_final_grades(course_run_id):
"""
Async task manager to freeze all the users' final grade in a course run
Args:
course_run_id (int): a course run id
Returns:
None
"""
course_run = CourseRun.objects.get(id=course_run_id)
# no need to do anything if the course run is not ready
if not course_run.can_freeze_grades:
log.info('the grades course "%s" cannot be frozen yet', course_run.edx_course_key)
return
# if it has already completed, do not do anything
if CourseRunGradingStatus.is_complete(course_run):
log.info('Final Grades freezing for course run "%s" has already been completed', course_run.edx_course_key)
return
# cache id string for this task
cache_id = CACHE_ID_BASE_STR.format(course_run.edx_course_key)
# try to get the result id from a previous iteration of this task for this course run
group_results_id = cache_redis.get(cache_id)
# if the id is not none, it means that this task already run before for this course run
# so we need to check if its subtasks have finished
if group_results_id is not None:
# delete the entry from the cache (if needed it will be added again later)
cache_redis.delete(cache_id)
# extract the results from the id
results = GroupResult.restore(group_results_id, app=app)
# if the subtasks are not done, revoke them
results.revoke()
# delete the results anyway
results.delete()
# extract the users to be frozen for this course
user_ids_qset = api.get_users_without_frozen_final_grade(course_run).values_list('id', flat=True)
# find number of users for which cache could not be updated
con = get_redis_connection("redis")
failed_users_cache_key = api.CACHE_KEY_FAILED_USERS_BASE_STR.format(course_run.edx_course_key)
failed_users_count = con.llen(failed_users_cache_key)
# get the list of users that failed authentication last run of the task
failed_users_list = list(map(int, con.lrange(failed_users_cache_key, 0, failed_users_count)))
users_need_freeze = list(user_ids_qset)
users_left = list(set(users_need_freeze) - set(failed_users_list))
# if there are no more users to be frozen, just complete the task
if not users_left:
log.info('Completing grading with %d users getting refresh cache errors', len(failed_users_list))
CourseRunGradingStatus.set_to_complete(course_run)
con.delete(failed_users_cache_key)
return
# if the task reaches this point, it means there are users still to be processed
# clear the list for users for whom cache update failed
con.delete(failed_users_cache_key)
# create an entry in with pending status ('pending' is the default status)
CourseRunGradingStatus.create_pending(course_run=course_run)
# create a group of subtasks to be run in parallel
job = group(
freeze_users_final_grade_async.s(list_user_ids, course_run.id) for list_user_ids in chunks(user_ids_qset)
)
results = job.apply_async()
# save the result ID in the celery backend
results.save()
# put the results id in the cache to be retrieved and finalized later
cache_redis.set(cache_id, results.id, None)
