def bulk_update_status(self, request, *args, **kwargs):
task_status = request.data.get('status', -1)
all_task_workers = TaskWorker.objects.filter(
id__in=tuple(request.data.get('workers', [])))
relevant_task_workers = all_task_workers.exclude(status=task_status)
workers = relevant_task_workers.values_list('worker_id', flat=True)
task_worker_ids = relevant_task_workers.values_list('id', flat=True)
if task_status == TaskWorker.STATUS_RETURNED:
update_worker_cache.delay(list(workers), constants.TASK_RETURNED)
elif task_status == TaskWorker.STATUS_REJECTED:
update_worker_cache.delay(list(workers), constants.TASK_REJECTED)
mturk_reject(list(task_worker_ids))
elif task_status == TaskWorker.STATUS_ACCEPTED:
mturk_approve.delay(list(task_worker_ids))
all_task_workers.update(status=task_status, updated_at=timezone.now())
return Response(
TaskWorkerSerializer(instance=all_task_workers,
many=True,
fields=('id', 'task', 'status',
'worker_alias',
'updated_delta')).data,
status.HTTP_200_OK)
def create(self, request, *args, **kwargs):
serializer = TaskWorkerSerializer()
instance, http_status = serializer.create(worker=request.user,
project=request.data.get('project', None))
serialized_data = {}
if http_status == 200:
serialized_data = TaskWorkerSerializer(instance=instance, fields=('id', 'task')).data
update_worker_cache.delay([instance.worker_id], constants.TASK_ACCEPTED)
mturk_hit_update.delay({'id': instance.task.id})
return Response(serialized_data, http_status)
def create(self, request, *args, **kwargs):
serializer = TaskWorkerSerializer()
instance, http_status = serializer.create(worker=request.user,
project=request.data.get(
'project', None))
serialized_data = {}
if http_status == 200:
serialized_data = TaskWorkerSerializer(instance=instance,
fields=('id', 'task')).data
update_worker_cache.delay([instance.worker_id],
constants.TASK_ACCEPTED)
mturk_hit_update.delay({'id': instance.task.id})
return Response(serialized_data, http_status)
def accept_all(self, request, *args, **kwargs):
task_id = request.query_params.get('task_id', -1)
from itertools import chain
task_workers = TaskWorker.objects.filter(status=TaskWorker.STATUS_SUBMITTED, task_id=task_id)
list_workers = list(chain.from_iterable(task_workers.values_list('id')))
update_worker_cache.delay(list(task_workers.values_list('worker_id', flat=True)), constants.TASK_APPROVED)
task_workers.update(status=TaskWorker.STATUS_ACCEPTED, updated_at=timezone.now())
post_approve.delay(task_id, len(list_workers))
mturk_approve.delay(list_workers)
return Response(data=list_workers, status=status.HTTP_200_OK)
def post(self, request, *args, **kwargs):
identifier = request.query_params.get('daemo_id', False)
if not identifier:
return Response("Missing identifier", status=status.HTTP_400_BAD_REQUEST)
try:
from django.conf import settings
from hashids import Hashids
identifier_hash = Hashids(salt=settings.SECRET_KEY, min_length=settings.ID_HASH_MIN_LENGTH)
if len(identifier_hash.decode(identifier)) == 0:
return Response("Invalid identifier", status=status.HTTP_400_BAD_REQUEST)
task_worker_id, task_id, template_item_id = identifier_hash.decode(identifier)
template_item = models.TemplateItem.objects.get(id=template_item_id)
task = models.Task.objects.get(id=task_id)
source_url = None
if template_item.aux_attributes['src']:
source_url = urlsplit(template_item.aux_attributes['src'])
else:
source_url = urlsplit(task.data[template_item.aux_attributes['data_source']])
if 'HTTP_REFERER' not in request.META.keys():
return Response(data={"message": "Missing referer"}, status=status.HTTP_403_FORBIDDEN)
referer_url = urlsplit(request.META['HTTP_REFERER'])
if referer_url.netloc != source_url.netloc or referer_url.scheme != source_url.scheme:
return Response(data={"message": "Referer does not match source"}, status=status.HTTP_403_FORBIDDEN)
redis_publisher = RedisPublisher(facility='external', broadcast=True)
task_hash = Hashids(salt=settings.SECRET_KEY, min_length=settings.ID_HASH_MIN_LENGTH)
message = RedisMessage(json.dumps({"task_id": task_hash.encode(task_id),
"daemo_id": identifier,
"template_item": template_item_id
}))
redis_publisher.publish_message(message)
with transaction.atomic():
task_worker = TaskWorker.objects.get(id=task_worker_id, task_id=task_id)
task_worker_result, created = TaskWorkerResult.objects.get_or_create(task_worker_id=task_worker.id,
template_item_id=template_item_id)
# only accept in progress, submitted, or returned tasks
if task_worker.status in [1, 2, 5]:
task_worker_result.result = request.data
task_worker_result.save()
update_worker_cache.delay([task_worker.worker_id], constants.TASK_SUBMITTED)
return Response(request.data, status=status.HTTP_200_OK)
else:
return Response("Task cannot be modified now", status=status.HTTP_400_BAD_REQUEST)
except ValueError:
return Response("Invalid identifier", status=status.HTTP_400_BAD_REQUEST)
except Exception:
return Response("Fail", status=status.HTTP_400_BAD_REQUEST)
def accept_all(self, request, *args, **kwargs):
task_id = request.query_params.get('task_id', -1)
from itertools import chain
task_workers = TaskWorker.objects.filter(
status=TaskWorker.STATUS_SUBMITTED, task_id=task_id)
list_workers = list(chain.from_iterable(
task_workers.values_list('id')))
update_worker_cache.delay(
list(task_workers.values_list('worker_id', flat=True)),
constants.TASK_APPROVED)
task_workers.update(status=TaskWorker.STATUS_ACCEPTED,
updated_at=timezone.now())
post_approve.delay(task_id, len(list_workers))
mturk_approve.delay(list_workers)
return Response(data=list_workers, status=status.HTTP_200_OK)
def destroy(self, request, *args, **kwargs):
serializer = TaskWorkerSerializer()
obj = self.queryset.get(id=kwargs['pk'], worker=request.user)
auto_accept = False
user_prefs = get_model_or_none(UserPreferences, user=request.user)
instance, http_status = None, status.HTTP_204_NO_CONTENT
if user_prefs is not None:
auto_accept = user_prefs.auto_accept
if auto_accept:
instance, http_status = serializer.create(worker=request.user, project=obj.task.project_id)
obj.status = TaskWorker.STATUS_SKIPPED
obj.save()
refund_task.delay([{'id': obj.id}])
update_worker_cache.delay([obj.worker_id], constants.TASK_SKIPPED)
mturk_hit_update.delay({'id': obj.task.id})
serialized_data = {}
if http_status == status.HTTP_200_OK:
serialized_data = TaskWorkerSerializer(instance=instance).data
return Response(serialized_data, http_status)
def destroy(self, request, *args, **kwargs):
serializer = TaskWorkerSerializer()
obj = self.queryset.get(id=kwargs['pk'], worker=request.user)
auto_accept = False
user_prefs = get_model_or_none(UserPreferences, user=request.user)
instance, http_status = None, status.HTTP_204_NO_CONTENT
if user_prefs is not None:
auto_accept = user_prefs.auto_accept
if auto_accept:
instance, http_status = serializer.create(
worker=request.user, project=obj.task.project_id)
obj.status = TaskWorker.STATUS_SKIPPED
obj.save()
refund_task.delay([{'id': obj.id}])
update_worker_cache.delay([obj.worker_id], constants.TASK_SKIPPED)
mturk_hit_update.delay({'id': obj.task.id})
serialized_data = {}
if http_status == status.HTTP_200_OK:
serialized_data = TaskWorkerSerializer(instance=instance).data
return Response(serialized_data, http_status)
def bulk_update_status(self, request, *args, **kwargs):
task_status = request.data.get('status', -1)
all_task_workers = TaskWorker.objects.filter(id__in=tuple(request.data.get('workers', [])))
relevant_task_workers = all_task_workers.exclude(status=task_status)
workers = relevant_task_workers.values_list('worker_id', flat=True)
task_worker_ids = relevant_task_workers.values_list('id', flat=True)
if task_status == TaskWorker.STATUS_RETURNED:
update_worker_cache.delay(list(workers), constants.TASK_RETURNED)
elif task_status == TaskWorker.STATUS_REJECTED:
update_worker_cache.delay(list(workers), constants.TASK_REJECTED)
mturk_reject(list(task_worker_ids))
elif task_status == TaskWorker.STATUS_ACCEPTED:
mturk_approve.delay(list(task_worker_ids))
all_task_workers.update(status=task_status, updated_at=timezone.now())
return Response(TaskWorkerSerializer(instance=all_task_workers, many=True,
fields=('id', 'task', 'status',
'worker_alias', 'updated_delta')).data, status.HTTP_200_OK)
def post(self, request, *args, **kwargs):
identifier = request.query_params.get('daemo_id', False)
if not identifier:
return Response("Missing identifier",
status=status.HTTP_400_BAD_REQUEST)
try:
from django.conf import settings
from hashids import Hashids
identifier_hash = Hashids(salt=settings.SECRET_KEY,
min_length=settings.ID_HASH_MIN_LENGTH)
if len(identifier_hash.decode(identifier)) == 0:
return Response("Invalid identifier",
status=status.HTTP_400_BAD_REQUEST)
task_worker_id, task_id, template_item_id = identifier_hash.decode(
identifier)
template_item = models.TemplateItem.objects.get(
id=template_item_id)
task = models.Task.objects.get(id=task_id)
source_url = None
if template_item.aux_attributes['src']:
source_url = urlsplit(template_item.aux_attributes['src'])
else:
source_url = urlsplit(
task.data[template_item.aux_attributes['data_source']])
if 'HTTP_REFERER' not in request.META.keys():
return Response(data={"message": "Missing referer"},
status=status.HTTP_403_FORBIDDEN)
referer_url = urlsplit(request.META['HTTP_REFERER'])
if referer_url.netloc != source_url.netloc or referer_url.scheme != source_url.scheme:
return Response(
data={"message": "Referer does not match source"},
status=status.HTTP_403_FORBIDDEN)
redis_publisher = RedisPublisher(facility='external',
broadcast=True)
task_hash = Hashids(salt=settings.SECRET_KEY,
min_length=settings.ID_HASH_MIN_LENGTH)
message = RedisMessage(
json.dumps({
"task_id": task_hash.encode(task_id),
"daemo_id": identifier,
"template_item": template_item_id
}))
redis_publisher.publish_message(message)
with transaction.atomic():
task_worker = TaskWorker.objects.get(id=task_worker_id,
task_id=task_id)
task_worker_result, created = TaskWorkerResult.objects.get_or_create(
task_worker_id=task_worker.id,
template_item_id=template_item_id)
# only accept in progress, submitted, or returned tasks
if task_worker.status in [1, 2, 5]:
task_worker_result.result = request.data
task_worker_result.save()
update_worker_cache.delay([task_worker.worker_id],
constants.TASK_SUBMITTED)
return Response(request.data, status=status.HTTP_200_OK)
else:
return Response("Task cannot be modified now",
status=status.HTTP_400_BAD_REQUEST)
except ValueError:
return Response("Invalid identifier",
status=status.HTTP_400_BAD_REQUEST)
except Exception:
return Response("Fail", status=status.HTTP_400_BAD_REQUEST)
def submit_results(self, request, mock=False, *args, **kwargs):
task = request.data.get('task', None)
auto_accept = request.data.get('auto_accept', False)
template_items = request.data.get('items', [])
task_status = request.data.get('status', None)
saved = request.data.get('saved')
task_worker = None
if mock:
task_status = TaskWorker.STATUS_SUBMITTED
template_items = kwargs['items']
task_worker = kwargs['task_worker']
with transaction.atomic():
if not mock:
task_worker = TaskWorker.objects.prefetch_related(
'task', 'task__project').get(worker=request.user,
task=task)
task_worker_results = TaskWorkerResult.objects.filter(
task_worker_id=task_worker.id)
if task_status == TaskWorker.STATUS_IN_PROGRESS:
serializer = TaskWorkerResultSerializer(data=template_items,
many=True,
partial=True)
else:
serializer = TaskWorkerResultSerializer(data=template_items,
many=True)
if serializer.is_valid():
task_worker.status = task_status
task_worker.save()
if task_status == TaskWorker.STATUS_SUBMITTED:
redis_publisher = RedisPublisher(
facility='bot', users=[task_worker.task.project.owner])
task = task_worker.task
message = {
"type": "REGULAR",
"payload": {
'project_id':
task_worker.task.project_id,
'project_key':
ProjectSerializer().get_hash_id(
task_worker.task.project),
'task_id':
task_worker.task_id,
'task_group_id':
task_worker.task.group_id,
'taskworker_id':
task_worker.id,
'worker_id':
task_worker.worker_id
}
}
if task.project.is_review:
match_group = MatchGroup.objects.get(batch=task.batch)
if is_final_review(match_group.batch_id):
message = get_review_redis_message(
match_group.id,
ProjectSerializer().get_hash_id(
task_worker.task.project))
message = RedisMessage(json.dumps(message))
redis_publisher.publish_message(message)
if task_worker_results.count() != 0:
serializer.update(task_worker_results,
serializer.validated_data)
else:
serializer.create(task_worker=task_worker)
update_worker_cache.delay([task_worker.worker_id],
constants.TASK_SUBMITTED)
winner_id = task_worker_results[0].result
update_ts_scores(task_worker, winner_id)
if task_status == TaskWorker.STATUS_IN_PROGRESS or saved or mock:
return Response('Success', status.HTTP_200_OK)
elif task_status == TaskWorker.STATUS_SUBMITTED and not saved:
if not auto_accept:
serialized_data = {}
http_status = 204
return Response(serialized_data, http_status)
task_worker_serializer = TaskWorkerSerializer()
instance, http_status = task_worker_serializer.create(
worker=request.user,
project=task_worker.task.project_id)
serialized_data = {}
if http_status == status.HTTP_200_OK:
serialized_data = TaskWorkerSerializer(
instance=instance).data
update_worker_cache.delay([task_worker.worker_id],
constants.TASK_ACCEPTED)
return Response(serialized_data, http_status)
else:
return Response(serializer.errors, status.HTTP_400_BAD_REQUEST)
def submit_results(self, request, mock=False, *args, **kwargs):
task = request.data.get('task', None)
auto_accept = request.data.get('auto_accept', False)
template_items = request.data.get('items', [])
task_status = request.data.get('status', None)
saved = request.data.get('saved')
task_worker = None
if mock:
task_status = TaskWorker.STATUS_SUBMITTED
template_items = kwargs['items']
task_worker = kwargs['task_worker']
with transaction.atomic():
if not mock:
task_worker = TaskWorker.objects.prefetch_related('task', 'task__project').get(worker=request.user,
task=task)
task_worker_results = TaskWorkerResult.objects.filter(task_worker_id=task_worker.id)
if task_status == TaskWorker.STATUS_IN_PROGRESS:
serializer = TaskWorkerResultSerializer(data=template_items, many=True, partial=True)
else:
serializer = TaskWorkerResultSerializer(data=template_items, many=True)
if serializer.is_valid():
task_worker.status = task_status
task_worker.save()
if task_status == TaskWorker.STATUS_SUBMITTED:
redis_publisher = RedisPublisher(facility='bot', users=[task_worker.task.project.owner])
task = task_worker.task
message = {
"type": "REGULAR",
"payload": {
'project_id': task_worker.task.project_id,
'project_key': ProjectSerializer().get_hash_id(task_worker.task.project),
'task_id': task_worker.task_id,
'task_group_id': task_worker.task.group_id,
'taskworker_id': task_worker.id,
'worker_id': task_worker.worker_id
}
}
if task.project.is_review:
match_group = MatchGroup.objects.get(batch=task.batch)
if is_final_review(match_group.batch_id):
message = get_review_redis_message(match_group.id, ProjectSerializer().get_hash_id(
task_worker.task.project))
message = RedisMessage(json.dumps(message))
redis_publisher.publish_message(message)
if task_worker_results.count() != 0:
serializer.update(task_worker_results, serializer.validated_data)
else:
serializer.create(task_worker=task_worker)
update_worker_cache.delay([task_worker.worker_id], constants.TASK_SUBMITTED)
winner_id = task_worker_results[0].result
update_ts_scores(task_worker, winner_id)
if task_status == TaskWorker.STATUS_IN_PROGRESS or saved or mock:
return Response('Success', status.HTTP_200_OK)
elif task_status == TaskWorker.STATUS_SUBMITTED and not saved:
if not auto_accept:
serialized_data = {}
http_status = 204
return Response(serialized_data, http_status)
task_worker_serializer = TaskWorkerSerializer()
instance, http_status = task_worker_serializer.create(
worker=request.user, project=task_worker.task.project_id)
serialized_data = {}
if http_status == status.HTTP_200_OK:
serialized_data = TaskWorkerSerializer(instance=instance).data
update_worker_cache.delay([task_worker.worker_id], constants.TASK_ACCEPTED)
return Response(serialized_data, http_status)
else:
return Response(serializer.errors, status.HTTP_400_BAD_REQUEST)
def submit_results(self, request, *args, **kwargs):
mturk_assignment = self.get_object()
template_items = request.data.get('items', [])
with transaction.atomic():
task_worker_results = TaskWorkerResult.objects.filter(
task_worker_id=mturk_assignment.task_worker.id)
serializer = TaskWorkerResultSerializer(data=template_items,
many=True)
if serializer.is_valid():
if task_worker_results.count() != 0:
serializer.update(task_worker_results,
serializer.validated_data)
else:
serializer.create(task_worker=mturk_assignment.task_worker)
if mturk_assignment.status == TaskWorker.STATUS_SKIPPED:
in_progress_assignment = MTurkAssignment.objects. \
filter(hit=mturk_assignment.hit, assignment_id=mturk_assignment.assignment_id,
status=TaskWorker.STATUS_IN_PROGRESS).first()
if in_progress_assignment is not None and in_progress_assignment.task_worker is not None:
in_progress_assignment.status = TaskWorker.STATUS_SKIPPED
in_progress_assignment.task_worker.status = TaskWorker.STATUS_SKIPPED
in_progress_assignment.task_worker.save()
in_progress_assignment.save()
mturk_assignment.task_worker.task_status = TaskWorker.STATUS_SUBMITTED
mturk_assignment.task_worker.status = TaskWorker.STATUS_SUBMITTED
mturk_assignment.task_worker.save()
mturk_assignment.status = TaskWorker.STATUS_SUBMITTED
mturk_assignment.save()
task_worker = mturk_assignment.task_worker
task_data = task_worker.task.data
redis_publisher = RedisPublisher(
facility='bot', users=[task_worker.task.project.owner])
task = task_worker.task
task_workers = TaskWorker.objects.filter(
task__group_id=task_worker.task.group_id,
status__in=[
TaskWorker.STATUS_ACCEPTED, TaskWorker.STATUS_SUBMITTED
])
message = {
"type": "REGULAR",
"payload": {
'project_id':
task_worker.task.project_id,
'project_key':
ProjectSerializer().get_hash_id(
task_worker.task.project),
'task_id':
task_worker.task_id,
'task_group_id':
task_worker.task.group_id,
'taskworker_id':
task_worker.id,
'worker_id':
task_worker.worker_id,
'expected':
max(task_workers.count(),
task_worker.task.project.repetition),
}
}
if task.project.is_review:
match_group = MatchGroup.objects.get(batch=task.batch)
if is_final_review(task.batch_id):
message = {
"type": "REVIEW",
"payload": {
"match_group_id":
match_group.id,
'project_key':
ProjectSerializer().get_hash_id(
task_worker.task.project),
"is_done":
True
}
}
message = RedisMessage(json.dumps(message))
redis_publisher.publish_message(message)
update_worker_cache.delay([task_worker.worker_id],
constants.TASK_SUBMITTED)
if task.project.is_review:
winner_id = task_worker_results[0].result
update_ts_scores(task_worker, winner_id=winner_id)
if "gold_truth" in task_data:
truth = dict()
truth["message"] = "truth"
truth["truth"] = task_data.get("gold_truth")
return Response(data=truth, status=status.HTTP_200_OK)
return Response(data={'message': 'Success'},
status=status.HTTP_200_OK)
else:
return Response(serializer.errors, status.HTTP_400_BAD_REQUEST)