class PsiTurkRecruiter(Recruiter):
"""Recruit participants from Amazon Mechanical Turk."""
def __init__(self):
"""Set up the connection to MTurk and psiTurk web services."""
# load the configuration options
self.config = PsiturkConfig()
self.config.load_config()
class FakeExperimentServerController(object):
def is_server_running(self):
return "yes"
self.server = FakeExperimentServerController()
# Get keys from environment variables or config file.
self.aws_access_key_id = os.getenv("aws_access_key_id", self.config.get("AWS Access", "aws_access_key_id"))
self.aws_secret_access_key = os.getenv(
"aws_secret_access_key", self.config.get("AWS Access", "aws_secret_access_key")
)
self.aws_region = os.getenv("aws_region", self.config.get("AWS Access", "aws_region"))
def open_recruitment(self, n=1):
"""Open recruitment for the first HIT, unless it's already open."""
from psiturk.amt_services import MTurkServices, RDSServices
from psiturk.psiturk_shell import PsiturkNetworkShell
from psiturk.psiturk_org_services import PsiturkOrgServices
psiturk_access_key_id = os.getenv(
"psiturk_access_key_id", self.config.get("psiTurk Access", "psiturk_access_key_id")
)
psiturk_secret_access_id = os.getenv(
"psiturk_secret_access_id", self.config.get("psiTurk Access", "psiturk_secret_access_id")
)
web_services = PsiturkOrgServices(psiturk_access_key_id, psiturk_secret_access_id)
aws_rds_services = RDSServices(self.aws_access_key_id, self.aws_secret_access_key, self.aws_region)
self.amt_services = MTurkServices(
self.aws_access_key_id,
self.aws_secret_access_key,
self.config.getboolean("Shell Parameters", "launch_in_sandbox_mode"),
)
self.shell = PsiturkNetworkShell(
self.config,
self.amt_services,
aws_rds_services,
web_services,
self.server,
self.config.getboolean("Shell Parameters", "launch_in_sandbox_mode"),
)
try:
participants = Participant.query.all()
assert participants
except Exception:
# Create the first HIT.
self.shell.hit_create(
n,
self.config.get("HIT Configuration", "base_payment"),
self.config.get("HIT Configuration", "duration"),
)
else:
# HIT was already created, no need to recreate it.
print "Reject recruitment reopening: experiment has started."
def recruit_participants(self, n=1):
"""Extend the HIT to recruit more people."""
auto_recruit = os.environ["auto_recruit"] == "true"
if auto_recruit:
print "Starting Wallace's recruit_participants."
hit_id = str(Participant.query.with_entities(Participant.hitid).first().hitid)
print "hit_id is {}.".format(hit_id)
is_sandbox = self.config.getboolean("Shell Parameters", "launch_in_sandbox_mode")
if is_sandbox:
host = "mechanicalturk.sandbox.amazonaws.com"
else:
host = "mechanicalturk.amazonaws.com"
mturkparams = dict(
aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, host=host
)
self.mtc = MTurkConnection(**mturkparams)
self.mtc.extend_hit(hit_id, assignments_increment=int(n or 0))
expiration_increment = self.config.get("HIT Configuration", "duration")
self.mtc.extend_hit(hit_id, expiration_increment=int(float(expiration_increment or 0) * 3600))
else:
print (">>>> auto_recruit set to {}: recruitment suppressed".format(auto_recruit))
def approve_hit(self, assignment_id):
"""Approve the HIT."""
from psiturk.amt_services import MTurkServices
self.amt_services = MTurkServices(
self.aws_access_key_id,
self.aws_secret_access_key,
self.config.getboolean("Shell Parameters", "launch_in_sandbox_mode"),
)
return self.amt_services.approve_worker(assignment_id)
def reward_bonus(self, assignment_id, amount, reason):
"""Reward the Turker with a bonus."""
from psiturk.amt_services import MTurkServices
self.amt_services = MTurkServices(
self.aws_access_key_id,
self.aws_secret_access_key,
self.config.getboolean("Shell Parameters", "launch_in_sandbox_mode"),
)
return self.amt_services.bonus_worker(assignment_id, amount, reason)
def close_recruitment(self):
"""Close recruitment."""
pass
import os
print os.getcwd()
import sys
import boto
from boto.mturk.connection import MTurkConnection
SAND = 0
ACCESS_ID = 'AKIAIM5D5I7RUTGYNI7A'
SECRET_KEY = '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'
HIT = "2JNL8I9NZW6HG96GKYHWCT87ATCVL9"
# https://mechanicalturk.amazonaws.com/?Service=AWSMechanicalTurkRequester
HOST = 'mechanicalturk.amazonaws.com'
mtc = MTurkConnection(aws_access_key_id=ACCESS_ID,
aws_secret_access_key=SECRET_KEY,
host=HOST)
print mtc.get_account_balance()
mtc.extend_hit(hit_id=HIT, assignments_increment=1)
class MTurkClient:
# SETUP
# ===========
def __init__(self,aws_access_key,aws_secret_key,aws_mode):
self.mode = aws_mode
if aws_mode == 'sandbox':
self.host = 'mechanicalturk.sandbox.amazonaws.com'
else:
self.host = 'mechanicalturk.amazonaws.com'
self.c = MTurkConnection(
aws_access_key,
aws_secret_key,
host=self.host)
default_settings = {
'lifetime': DAY,
'duration': 10 * MINUTE,
'approval_delay': DAY,
'title': "[title]",
'description': "[description]",
'keywords': [],
'reward': 0.01,
'max_assignments': 1,
'height': 700,
'qualifications': [],
}
# HITS
# ===========
def create_hit(self,url,extra_settings):
"Eventually, this should take a TEMPLATE and a dictionary of INPUT data that's put into that template. This function would then create an HTML file locally (assuming we're running on a web server) by replacing template {tags} with input values, and then send the URL to the newly created page to MTurk."
settings = self.default_settings.copy()
settings.update(extra_settings)
settings['reward'] = Price(settings['reward'])
settings['qualifications'] = qualification.Qualifications(settings['qualifications'])
settings['keywords'] = ','.join(settings['keywords'])
height = settings.pop('height')
hit = self.c.create_hit(question=ExternalQuestion(url,height),**settings)[0]
#print 'Created hit %s' % hit.HITId
return hit.HITId,hit.HITTypeId
#hit_type=None, # Let Amazon do this automatically
#annotation=None, # Optional annotation for our system to use
#questions=None, # If you want to create multiple HITs at a time? Probably irrelevant for External
#response_groups=None, # Unclear what this does
def get_hit(self,hit_id):
return self.c.get_hit(hit_id)[0]
def hit_results(self,hit_id,type=None): # type in ['Submitted','Approved','Rejected',None]
results = {}
assignments = self.c.get_assignments(hit_id, status=None, page_size=100)
for asst in assignments:
results.setdefault(asst.AssignmentId,{})
answers = asst.answers[0]
for qfa in answers:
field, response = qfa.qid, qfa.fields[0]
results[asst.AssignmentId][field] = response
results[asst.AssignmentId]['worker_id'] = asst.WorkerId
results[asst.AssignmentId]['accept_time'] = datetime.strptime(asst.AcceptTime,"%Y-%m-%dT%H:%M:%SZ")
results[asst.AssignmentId]['submit_time'] = datetime.strptime(asst.SubmitTime,"%Y-%m-%dT%H:%M:%SZ")
return results
# URL of a HIT on MTurk
def hit_url_turk(self,hit_id):
pass
def hit_url_external(self,hit_id):
pass
def extend_hit(self,hit_id,extras):
return self.c.extend_hit(hit_id, extras)
@catcherror
def delete_hit(self,hit_id):
self.c.disable_hit(hit_id)
# Deletes all the HITS on the server. Risky!
def cleanup(self):
for hit in self.c.get_all_hits():
self.delete_hit(hit.HITId)
# ASSIGNMENTS
# ===========
@catcherror
def approve(self, asst_id, feedback=None):
return self.c.approve_assignment(asst_id, feedback)
@catcherror
def reject(self, asst_id, feedback=None):
return self.c.reject_assignment(asst_id, feedback)
def block(self,worker_id,feedback=None):
return self.c.block_worker(worker_id, feedback)
def unblock(self,worker_id,feedback=None):
return self.c.unblock_worker(worker_id, feedback)
def bonus(self,asst,amount,feedback):
return self.c.grant_bonus(asst.worker, asst.asst_id, Price(amount), feedback)
# STATUS / DIAGNOSTICS
# --------------------
def balance(self):
return self.c.get_account_balance()[0]
class MTurkProvider(object):
description = 'This is a task authored by a requester on Daemo, a research crowdsourcing platform. ' \
'Mechanical Turk workers are welcome to do it'
keywords = ['daemo']
countries = ['US', 'CA']
min_hits = 1000
def __init__(self, host, aws_access_key_id, aws_secret_access_key):
self.host = host
self.connection = MTurkConnection(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
host=settings.MTURK_HOST)
self.connection.APIVersion = "2014-08-15"
if not self.host:
raise ValueError("Please provide a host url")
def get_connection(self):
return self.connection
@staticmethod
def _mturk_system_qualifications(qualification):
requirements = []
for item in qualification.items.all():
if item.expression['attribute'] not in [
'location', 'approval_rate', 'total_tasks'
]:
continue
requirement = None
if item.expression['attribute'] == 'location':
op = OP_IN if item.expression['operator'] == 'in' else OP_NOT_IN
requirement = MultiLocaleRequirement(op, [
val.strip() for val in item.expression['value']
if val is not None and val != ''
])
elif item.expression['attribute'] == 'approval_rate':
op = OP_GT if item.expression['operator'] == 'gt' else OP_LT
requirement = PercentAssignmentsApprovedRequirement(
op, item.expression['value'])
elif item.expression['attribute'] == 'total_tasks':
op = OP_GT if item.expression['operator'] == 'gt' else OP_LT
requirement = NumberHitsApprovedRequirement(
op, item.expression['value'])
requirements.append(requirement)
return requirements
def get_qualifications(self, project, boomerang_threshold, add_boomerang):
requirements = []
if project.qualification is not None:
requirements += self._mturk_system_qualifications(
project.qualification)
boomerang_qual, success = self.create_qualification_type(
owner_id=project.owner_id,
project_id=project.group_id,
name='Boomerang Score #{}'.format(project.group_id),
flag=FLAG_Q_BOOMERANG,
description='No description available')
boomerang = None
if boomerang_threshold <= int(settings.BOOMERANG_MIDPOINT * 100):
for i, bucket in enumerate(WAIT_LIST_BUCKETS):
if int(bucket[1] * 100) <= boomerang_threshold:
boomerang_blacklist, success = \
self.create_qualification_type(owner_id=project.owner_id,
name='Boomerang Waitlist #{}-{}'.format(project.group_id, len(
WAIT_LIST_BUCKETS) - i),
flag=FLAG_Q_BOOMERANG,
description='No description available',
deny=True,
project_id=project.group_id,
bucket=bucket)
if success and add_boomerang:
boomerang = BoomerangRequirement(
qualification_type_id=boomerang_blacklist.type_id,
comparator=OP_DNE,
integer_value=None)
requirements.append(boomerang)
else:
boomerang = BoomerangRequirement(
qualification_type_id=boomerang_qual.type_id,
comparator=OP_GTEQ,
integer_value=boomerang_threshold)
if success and add_boomerang:
requirements.append(boomerang)
return Qualifications(requirements), boomerang_qual
def create_hits(self, project, tasks=None, repetition=None):
# if project.min_rating > 0:
# return 'NOOP'
if not tasks:
cursor = connection.cursor()
# noinspection SqlResolve
query = '''
SELECT
max(id) id,
repetition,
group_id,
repetition - sum(existing_assignments) remaining_assignments,
min_rating
FROM (
SELECT
t_rev.id,
t.group_id,
t.min_rating,
p.repetition,
CASE WHEN ma.id IS NULL OR ma.status IN (%(skipped)s, %(rejected)s, %(expired)s)
THEN 0
ELSE 1 END existing_assignments
FROM crowdsourcing_task t
INNER JOIN crowdsourcing_project p ON t.project_id = p.id
INNER JOIN crowdsourcing_task t_rev ON t_rev.group_id = t.group_id
LEFT OUTER JOIN mturk_mturkhit mh ON mh.task_id = t_rev.id
LEFT OUTER JOIN mturk_mturkassignment ma ON ma.hit_id = mh.id
WHERE t.project_id = (%(project_id)s) AND t_rev.exclude_at IS NULL
AND t_rev.deleted_at IS NULL
) t
GROUP BY group_id, repetition, min_rating HAVING sum(existing_assignments) < repetition;
'''
cursor.execute(
query, {
'skipped': TaskWorker.STATUS_SKIPPED,
'rejected': TaskWorker.STATUS_REJECTED,
'expired': TaskWorker.STATUS_EXPIRED,
'project_id': project.id
})
tasks = cursor.fetchall()
rated_workers = Rating.objects.filter(
origin_type=Rating.RATING_REQUESTER).count()
add_boomerang = rated_workers > 0
duration = project.timeout if project.timeout is not None else datetime.timedelta(
hours=24)
lifetime = project.deadline - timezone.now(
) if project.deadline is not None else datetime.timedelta(days=7)
for task in tasks:
question = self.create_external_question(task[0])
mturk_hit = MTurkHIT.objects.filter(task_id=task[0]).first()
qualifications, boomerang_qual = self.get_qualifications(
project=project,
boomerang_threshold=int(round(task[4], 2) * 100),
add_boomerang=add_boomerang)
qualifications_mask = 0
if qualifications is not None:
qualifications_mask = FLAG_Q_LOCALE + FLAG_Q_HITS + FLAG_Q_RATE + FLAG_Q_BOOMERANG
hit_type, success = self.create_hit_type(
title=project.name,
description=self.description,
price=project.price,
duration=duration,
keywords=self.keywords,
approval_delay=datetime.timedelta(days=2),
qual_req=qualifications,
qualifications_mask=qualifications_mask,
boomerang_threshold=int(round(task[4], 2) * 100),
owner_id=project.owner_id,
boomerang_qual=boomerang_qual)
if not success:
return 'FAILURE'
if mturk_hit is None:
try:
hit = self.connection.create_hit(
hit_type=hit_type.string_id,
max_assignments=task[3],
lifetime=lifetime,
question=question)[0]
self.set_notification(hit_type_id=hit.HITTypeId)
mturk_hit = MTurkHIT(hit_id=hit.HITId,
hit_type=hit_type,
task_id=task[0])
except MTurkRequestError as e:
error = e.errors[0][0]
if error == 'AWS.MechanicalTurk.InsufficientFunds':
message = {
"type": "ERROR",
"detail":
"Insufficient funds on your Mechanical Turk account!",
"code": error
}
redis_publisher = RedisPublisher(facility='bot',
users=[project.owner])
message = RedisMessage(json.dumps(message))
redis_publisher.publish_message(message)
return 'FAILED'
else:
if mturk_hit.hit_type_id != hit_type.id:
result, success = self.change_hit_type_of_hit(
hit_id=mturk_hit.hit_id,
hit_type_id=hit_type.string_id)
if success:
mturk_hit.hit_type = hit_type
mturk_hit.save()
return 'SUCCESS'
def create_hit_type(self,
owner_id,
title,
description,
price,
duration,
boomerang_threshold,
keywords=None,
approval_delay=None,
qual_req=None,
qualifications_mask=0,
boomerang_qual=None):
hit_type = MTurkHITType.objects.filter(
owner_id=owner_id,
name=title,
description=description,
price=Decimal(str(price)),
duration=duration,
qualifications_mask=qualifications_mask,
boomerang_threshold=boomerang_threshold).first()
if hit_type is not None:
return hit_type, True
reward = Price(price)
try:
mturk_ht = self.connection.register_hit_type(
title=title,
description=description,
reward=reward,
duration=duration,
keywords=keywords,
approval_delay=approval_delay,
qual_req=qual_req)[0]
hit_type = MTurkHITType(owner_id=owner_id,
name=title,
description=description,
price=Decimal(str(price)),
keywords=keywords,
duration=duration,
qualifications_mask=qualifications_mask,
boomerang_qualification=boomerang_qual,
boomerang_threshold=boomerang_threshold)
hit_type.string_id = mturk_ht.HITTypeId
hit_type.save()
except MTurkRequestError:
return None, False
return hit_type, True
def create_external_question(self, task, frame_height=800):
task_hash = Hashids(salt=settings.SECRET_KEY,
min_length=settings.ID_HASH_MIN_LENGTH)
task_id = task_hash.encode(task)
url = self.host + '/mturk/task/?taskId=' + task_id
question = ExternalQuestion(external_url=url,
frame_height=frame_height)
return question
def update_max_assignments(self, task):
task = Task.objects.get(id=task['id'])
mturk_hit = task.mturk_hit
if not mturk_hit:
raise MTurkHIT.DoesNotExist(
"This task is not associated to any mturk hit")
assignments_completed = task.task_workers.filter(~Q(status__in=[
TaskWorker.STATUS_REJECTED, TaskWorker.STATUS_SKIPPED,
TaskWorker.STATUS_EXPIRED
])).count()
remaining_assignments = task.project.repetition - assignments_completed
if remaining_assignments > 0 and mturk_hit.num_assignments == mturk_hit.mturk_assignments. \
filter(status=TaskWorker.STATUS_SUBMITTED).count() and \
mturk_hit.mturk_assignments.filter(status=TaskWorker.STATUS_IN_PROGRESS).count() == 0:
self.add_assignments(hit_id=mturk_hit.hit_id, increment=1)
self.extend_hit(hit_id=mturk_hit.hit_id)
mturk_hit.status = MTurkHIT.STATUS_IN_PROGRESS
mturk_hit.num_assignments += 1
mturk_hit.save()
elif remaining_assignments == 0:
self.expire_hit(hit_id=mturk_hit.hit_id)
mturk_hit.status = MTurkHIT.STATUS_EXPIRED
mturk_hit.save()
elif remaining_assignments > 0 and \
mturk_hit.status == MTurkHIT.STATUS_EXPIRED:
self.extend_hit(hit_id=mturk_hit.hit_id)
mturk_hit.status = MTurkHIT.STATUS_IN_PROGRESS
return 'SUCCESS'
def get_assignment(self, assignment_id):
try:
return self.connection.get_assignment(assignment_id)[0], True
except MTurkRequestError as e:
error = e.errors[0][0]
if error == 'AWS.MechanicalTurk.InvalidAssignmentState':
return assignment_id, False
return None, False
def set_notification(self, hit_type_id):
self.connection.set_rest_notification(
hit_type=hit_type_id,
url=self.host + '/api/mturk/notification',
event_types=[
'AssignmentReturned', 'AssignmentAbandoned',
'AssignmentAccepted', 'AssignmentSubmitted'
])
def approve_assignment(self, task_worker):
task_worker_obj = TaskWorker.objects.get(id=task_worker['id'])
if hasattr(task_worker_obj, 'mturk_assignments'
) and task_worker_obj.mturk_assignments.first() is not None:
try:
self.connection.approve_assignment(
task_worker_obj.mturk_assignments.first().assignment_id)
except MTurkRequestError:
return False
return True
def reject_assignment(self, task_worker):
task_worker_obj = TaskWorker.objects.get(id=task_worker['id'])
if hasattr(task_worker_obj, 'mturk_assignments'
) and task_worker_obj.mturk_assignments.first() is not None:
try:
self.connection.reject_assignment(
task_worker_obj.mturk_assignments.first().assignment_id)
except MTurkRequestError:
return False
return True
def expire_hit(self, hit_id):
try:
self.connection.expire_hit(hit_id)
except MTurkRequestError:
return False
return True
def disable_hit(self, hit_id):
try:
self.connection.disable_hit(hit_id)
except MTurkRequestError:
return False
return True
def extend_hit(self, hit_id):
try:
self.connection.extend_hit(hit_id=hit_id,
expiration_increment=604800) # 7 days
except MTurkRequestError:
return False
return True
def add_assignments(self, hit_id, increment=1):
try:
self.connection.extend_hit(hit_id=hit_id,
assignments_increment=increment)
except MTurkRequestError:
return False
return True
def test_connection(self):
try:
return self.connection.get_account_balance()[0], True
except MTurkRequestError as e:
error = e.errors[0][0]
if error == 'AWS.NotAuthorized':
return None, False
return None, False
def get_account_balance(self):
try:
return self.connection.get_account_balance()[0]
except MTurkRequestError:
return None
def create_qualification_type(self,
owner_id,
name,
flag,
description,
project_id,
auto_granted=False,
auto_granted_value=None,
deny=False,
bucket=None):
# noinspection SqlResolve
query = '''
SELECT * FROM (
SELECT
task.target_id,
task.username,
round(task.task_w_avg::NUMERIC, 2) rating
--round(coalesce(task.task_w_avg, requester.requester_w_avg,
-- platform.platform_w_avg)::NUMERIC, 2) rating
FROM (
SELECT
target_id,
origin_id,
project_id,
username,
sum(weight * power((%(BOOMERANG_TASK_ALPHA)s), t.row_number))
/ sum(power((%(BOOMERANG_TASK_ALPHA)s), t.row_number)) task_w_avg
FROM (
SELECT
r.id,
r.origin_id,
p.group_id project_id,
weight,
r.target_id,
-1 + row_number()
OVER (PARTITION BY target_id
ORDER BY tw.created_at DESC) AS row_number,
u.username username
FROM crowdsourcing_rating r
INNER JOIN crowdsourcing_task t ON t.id = r.task_id
INNER JOIN crowdsourcing_project p ON p.id = t.project_id
INNER JOIN crowdsourcing_taskworker tw ON t.id = tw.task_id
AND tw.worker_id=r.target_id
INNER JOIN auth_user u ON u.id = r.target_id
WHERE origin_id = (%(origin_id)s) AND origin_type = (%(origin_type)s)) t
GROUP BY origin_id, target_id, project_id, username)
task WHERE task.project_id = (%(project_id)s)
) r
'''
extra_query = 'WHERE rating BETWEEN (%(lower_bound)s) AND (%(upper_bound)s);'
params = {
'origin_type': Rating.RATING_REQUESTER,
'origin_id': owner_id,
'project_id': project_id,
'BOOMERANG_REQUESTER_ALPHA': settings.BOOMERANG_REQUESTER_ALPHA,
'BOOMERANG_PLATFORM_ALPHA': settings.BOOMERANG_PLATFORM_ALPHA,
'BOOMERANG_TASK_ALPHA': settings.BOOMERANG_TASK_ALPHA
}
obj_params = {'upper_bound': 300, 'lower_bound': 100}
if deny and bucket is not None:
query += extra_query
params.update({'upper_bound': bucket[1], 'lower_bound': bucket[0]})
obj_params.update({
'upper_bound': bucket[1] * 100,
'lower_bound': bucket[0] * 100,
'is_blacklist': True
})
cursor = connection.cursor()
cursor.execute(query, params=params)
worker_ratings_raw = cursor.fetchall()
worker_ratings = [{
"worker_id": r[0],
"worker_username": r[1],
"rating": r[2]
} for r in worker_ratings_raw]
qualification = MTurkQualification.objects.filter(owner_id=owner_id,
flag=flag,
name=name).first()
assigned_workers = []
if qualification is None:
try:
qualification_type = self.connection. \
create_qualification_type(name=name, description=description,
status='Active',
auto_granted=auto_granted,
auto_granted_value=auto_granted_value)[0]
qualification = MTurkQualification.objects.create(
owner_id=owner_id,
flag=flag,
name=name,
description=description,
auto_granted=auto_granted,
auto_granted_value=auto_granted_value,
type_id=qualification_type.QualificationTypeId,
**obj_params)
except MTurkRequestError:
return None, False
else:
assigned_workers = MTurkWorkerQualification.objects.values(
'worker').filter(qualification=qualification).values_list(
'worker', flat=True)
for rating in worker_ratings:
user_name = rating["worker_username"].split('.')
if len(user_name) == 2 and user_name[0] == 'mturk':
mturk_worker_id = user_name[1].upper()
if mturk_worker_id not in assigned_workers:
self.assign_qualification(
qualification_type_id=qualification.type_id,
worker_id=mturk_worker_id,
value=int(rating['rating'] * 100))
defaults = {
'qualification': qualification,
'worker': mturk_worker_id,
'score': int(rating['rating'] * 100)
}
MTurkWorkerQualification.objects.update_or_create(
qualification=qualification,
worker=mturk_worker_id,
defaults=defaults)
return qualification, True
def change_hit_type_of_hit(self, hit_id, hit_type_id):
try:
result = self.connection.change_hit_type_of_hit(
hit_id=hit_id, hit_type=hit_type_id)
except MTurkRequestError:
return None, False
return result, True
def update_worker_boomerang(self, project_id, worker_id, task_avg,
requester_avg):
"""
Update boomerang for project
Args:
project_id:
worker_id:
task_avg:
requester_avg
Returns:
str
"""
hit = MTurkHIT.objects.select_related(
'hit_type__boomerang_qualification').filter(
task__project__group_id=project_id).first()
if hit is not None:
qualification = hit.hit_type.boomerang_qualification
worker_qual = MTurkWorkerQualification.objects.filter(
qualification=qualification, worker=worker_id).first()
if worker_qual is not None:
self.update_score(worker_qual,
score=int(task_avg * 100),
override=True)
else:
MTurkWorkerQualification.objects.create(
qualification=qualification,
worker=worker_id,
score=int(task_avg * 100),
overwritten=True)
self.assign_qualification(
qualification_type_id=qualification.type_id,
worker_id=worker_id,
value=int(task_avg * 100))
# other_quals = MTurkWorkerQualification.objects.filter(~Q(qualification=qualification),
# worker=worker_id,
# overwritten=False)
# for q in other_quals:
# self.update_score(q, score=int(requester_avg * 100))
return 'SUCCESS'
def update_score(self, worker_qual, score, override=False):
if worker_qual is None:
return False
try:
self.connection.update_qualification_score(
worker_qual.qualification.type_id, worker_qual.worker, score)
worker_qual.overwritten = override
worker_qual.score = score
worker_qual.save()
except MTurkRequestError:
return False
return True
def assign_qualification(self, qualification_type_id, worker_id, value=1):
"""
Revoke a qualification from a WorkerId
Args:
qualification_type_id:
worker_id:
value
Returns:
bool
"""
try:
self.connection.assign_qualification(qualification_type_id,
worker_id,
value,
send_notification=False)
return True
except MTurkRequestError:
return False
def revoke_qualification(self, qualification_type_id, worker_id):
try:
self.connection.revoke_qualification(
qualification_type_id=qualification_type_id,
subject_id=worker_id)
return True
except MTurkRequestError:
return False
def notify_workers(self, worker_ids, subject, message_text):
try:
self.connection.notify_workers(worker_ids, subject, message_text)
return True
except MTurkRequestError:
return False
hit, = conn.get_hit(args.hit, ['HITDetail', 'HITAssignmentSummary'])
total = int(hit.MaxAssignments)
pending = int(hit.NumberOfAssignmentsPending)
complete = int(hit.NumberOfAssignmentsCompleted)
available = int(hit.NumberOfAssignmentsAvailable)
logging.info("max:%s/pending:%s/complete:%s/remain:%s", total, pending,
complete, available)
# check if we have reached the total
if total >= args.total:
logging.info("MaxAssignments = %s, exiting", total)
break
# compute how many assignments are currently outstanding
current = available + pending
if current < args.concurrent:
diff = min(args.total - total, args.concurrent - current)
logging.info("Extending HIT with %s more assignments", diff)
conn.extend_hit(args.hit, assignments_increment=diff)
# get submitted assignments and approve them
if args.approve:
assignments = conn.get_assignments(args.hit,
status="Submitted",
page_size=100)
for assignment in assignments:
logging.info("Approving assignment %s",
assignment.AssignmentId)
conn.approve_assignment(assignment.AssignmentId, feedback=None)
time.sleep(args.interval)
class PsiTurkRecruiter(Recruiter):
"""Recruit participants from Amazon Mechanical Turk via PsiTurk."""
def __init__(self):
"""Set up the connection to MTurk and psiTurk web services."""
# load the configuration options
self.config = PsiturkConfig()
self.config.load_config()
class FakeExperimentServerController(object):
def is_server_running(self):
return 'yes'
self.server = FakeExperimentServerController()
# Get keys from environment variables or config file.
self.aws_access_key_id = os.getenv(
"aws_access_key_id",
self.config.get("AWS Access", "aws_access_key_id"))
self.aws_secret_access_key = os.getenv(
"aws_secret_access_key",
self.config.get("AWS Access", "aws_secret_access_key"))
self.aws_region = os.getenv(
"aws_region", self.config.get("AWS Access", "aws_region"))
def open_recruitment(self, n=1):
"""Open recruitment for the first HIT, unless it's already open."""
from psiturk.amt_services import MTurkServices, RDSServices
from psiturk.psiturk_shell import PsiturkNetworkShell
from psiturk.psiturk_org_services import PsiturkOrgServices
psiturk_access_key_id = os.getenv(
"psiturk_access_key_id",
self.config.get("psiTurk Access", "psiturk_access_key_id"))
psiturk_secret_access_id = os.getenv(
"psiturk_secret_access_id",
self.config.get("psiTurk Access", "psiturk_secret_access_id"))
web_services = PsiturkOrgServices(psiturk_access_key_id,
psiturk_secret_access_id)
aws_rds_services = RDSServices(self.aws_access_key_id,
self.aws_secret_access_key,
self.aws_region)
self.amt_services = MTurkServices(
self.aws_access_key_id, self.aws_secret_access_key,
self.config.getboolean('Shell Parameters',
'launch_in_sandbox_mode'))
self.shell = PsiturkNetworkShell(
self.config, self.amt_services, aws_rds_services, web_services,
self.server,
self.config.getboolean('Shell Parameters',
'launch_in_sandbox_mode'))
try:
participants = Participant.query.all()
assert (participants)
except Exception:
# Create the first HIT.
self.shell.hit_create(
n, self.config.get('HIT Configuration', 'base_payment'),
self.config.get('HIT Configuration', 'duration'))
else:
# HIT was already created, no need to recreate it.
print "Reject recruitment reopening: experiment has started."
def recruit_participants(self, n=1):
"""Recruit n participants."""
auto_recruit = os.environ['auto_recruit'] == 'true'
if auto_recruit:
print "Starting Dallinger's recruit_participants."
hit_id = str(
Participant.query.with_entities(
Participant.hitid).first().hitid)
print "hit_id is {}.".format(hit_id)
is_sandbox = self.config.getboolean('Shell Parameters',
'launch_in_sandbox_mode')
if is_sandbox:
host = 'mechanicalturk.sandbox.amazonaws.com'
else:
host = 'mechanicalturk.amazonaws.com'
mturkparams = dict(
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
host=host)
self.mtc = MTurkConnection(**mturkparams)
self.mtc.extend_hit(hit_id, assignments_increment=int(n or 0))
expiration_increment = self.config.get('HIT Configuration',
'duration')
self.mtc.extend_hit(hit_id,
expiration_increment=int(
float(expiration_increment or 0) * 3600))
else:
print(">>>> auto_recruit set to {}: recruitment suppressed".format(
auto_recruit))
def approve_hit(self, assignment_id):
"""Approve the HIT."""
from psiturk.amt_services import MTurkServices
self.amt_services = MTurkServices(
self.aws_access_key_id, self.aws_secret_access_key,
self.config.getboolean('Shell Parameters',
'launch_in_sandbox_mode'))
return self.amt_services.approve_worker(assignment_id)
def reward_bonus(self, assignment_id, amount, reason):
"""Reward the Turker with a bonus."""
from psiturk.amt_services import MTurkServices
self.amt_services = MTurkServices(
self.aws_access_key_id, self.aws_secret_access_key,
self.config.getboolean('Shell Parameters',
'launch_in_sandbox_mode'))
return self.amt_services.bonus_worker(assignment_id, amount, reason)
def close_recruitment(self):
"""Close recruitment."""
pass
from boto.mturk.connection import MTurkConnection
print "Content-type: text/html"
print
form = cgi.FieldStorage()
HIT = form.getfirst("ID", "empty")
SAND = form.getfirst("sandbox", "empty")
ACCESS_ID = "AKIAIM5D5I7RUTGYNI7A"
SECRET_KEY = "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"
# HIT="2JNL8I9NZW6HG96GKYHWCT87ATCVL9"
# HOST = 'mechanicalturk.amazonaws.com'
# https://mechanicalturk.amazonaws.com/?Service=AWSMechanicalTurkRequester
if SAND == 1:
print "sandbox"
HOST = "mechanicalturk.sandbox.amazonaws.com"
else:
print "not Sandbox"
HOST = "mechanicalturk.amazonaws.com"
mtc = MTurkConnection(aws_access_key_id=ACCESS_ID, aws_secret_access_key=SECRET_KEY, host=HOST)
print mtc.get_account_balance()
mtc.extend_hit(hit_id=HIT, assignments_increment=1)
class MTurkProvider(object):
description = 'This is a task authored by a requester on Daemo, a research crowdsourcing platform. ' \
'Mechanical Turk workers are welcome to do it'
keywords = ['daemo']
countries = ['US', 'CA']
min_hits = 1000
def __init__(self, host, aws_access_key_id, aws_secret_access_key):
self.host = host
self.connection = MTurkConnection(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
host=settings.MTURK_HOST
)
self.connection.APIVersion = "2014-08-15"
if not self.host:
raise ValueError("Please provide a host url")
def get_connection(self):
return self.connection
@staticmethod
def _mturk_system_qualifications(qualification):
requirements = []
for item in qualification.items.all():
if item.expression['attribute'] not in ['location', 'approval_rate', 'total_tasks']:
continue
requirement = None
if item.expression['attribute'] == 'location':
op = OP_IN if item.expression['operator'] == 'in' else OP_NOT_IN
requirement = MultiLocaleRequirement(op, [val.strip() for val in item.expression['value'] if
val is not None and val != ''])
elif item.expression['attribute'] == 'approval_rate':
op = OP_GT if item.expression['operator'] == 'gt' else OP_LT
requirement = PercentAssignmentsApprovedRequirement(op, item.expression['value'])
elif item.expression['attribute'] == 'total_tasks':
op = OP_GT if item.expression['operator'] == 'gt' else OP_LT
requirement = NumberHitsApprovedRequirement(op, item.expression['value'])
requirements.append(requirement)
return requirements
def get_qualifications(self, project, boomerang_threshold, add_boomerang):
requirements = []
if project.qualification is not None:
requirements += self._mturk_system_qualifications(project.qualification)
boomerang_qual, success = self.create_qualification_type(owner_id=project.owner_id,
project_id=project.group_id,
name='Boomerang Score #{}'.format(project.group_id),
flag=FLAG_Q_BOOMERANG,
description='No description available')
boomerang = None
if boomerang_threshold <= int(settings.BOOMERANG_MIDPOINT * 100):
for i, bucket in enumerate(WAIT_LIST_BUCKETS):
if int(bucket[1] * 100) <= boomerang_threshold:
boomerang_blacklist, success = \
self.create_qualification_type(owner_id=project.owner_id,
name='Boomerang Waitlist #{}-{}'.format(project.group_id, len(
WAIT_LIST_BUCKETS) - i),
flag=FLAG_Q_BOOMERANG,
description='No description available',
deny=True,
project_id=project.group_id,
bucket=bucket)
if success and add_boomerang:
boomerang = BoomerangRequirement(qualification_type_id=boomerang_blacklist.type_id,
comparator=OP_DNE,
integer_value=None)
requirements.append(boomerang)
else:
boomerang = BoomerangRequirement(qualification_type_id=boomerang_qual.type_id, comparator=OP_GTEQ,
integer_value=boomerang_threshold)
if success and add_boomerang:
requirements.append(boomerang)
return Qualifications(requirements), boomerang_qual
def create_hits(self, project, tasks=None, repetition=None):
# if project.min_rating > 0:
# return 'NOOP'
if not tasks:
cursor = connection.cursor()
# noinspection SqlResolve
query = '''
SELECT
max(id) id,
repetition,
group_id,
repetition - sum(existing_assignments) remaining_assignments,
min_rating
FROM (
SELECT
t_rev.id,
t.group_id,
t.min_rating,
p.repetition,
CASE WHEN ma.id IS NULL OR ma.status IN (%(skipped)s, %(rejected)s, %(expired)s)
THEN 0
ELSE 1 END existing_assignments
FROM crowdsourcing_task t
INNER JOIN crowdsourcing_project p ON t.project_id = p.id
INNER JOIN crowdsourcing_task t_rev ON t_rev.group_id = t.group_id
LEFT OUTER JOIN mturk_mturkhit mh ON mh.task_id = t_rev.id
LEFT OUTER JOIN mturk_mturkassignment ma ON ma.hit_id = mh.id
WHERE t.project_id = (%(project_id)s) AND t_rev.exclude_at IS NULL
AND t_rev.deleted_at IS NULL
) t
GROUP BY group_id, repetition, min_rating HAVING sum(existing_assignments) < repetition;
'''
cursor.execute(query, {'skipped': TaskWorker.STATUS_SKIPPED,
'rejected': TaskWorker.STATUS_REJECTED,
'expired': TaskWorker.STATUS_EXPIRED,
'project_id': project.id})
tasks = cursor.fetchall()
rated_workers = Rating.objects.filter(origin_type=Rating.RATING_REQUESTER).count()
add_boomerang = rated_workers > 0
duration = project.timeout if project.timeout is not None else datetime.timedelta(hours=24)
lifetime = project.deadline - timezone.now() if project.deadline is not None else datetime.timedelta(
days=7)
for task in tasks:
question = self.create_external_question(task[0])
mturk_hit = MTurkHIT.objects.filter(task_id=task[0]).first()
qualifications, boomerang_qual = self.get_qualifications(project=project,
boomerang_threshold=int(
round(task[4], 2) * 100),
add_boomerang=add_boomerang)
qualifications_mask = 0
if qualifications is not None:
qualifications_mask = FLAG_Q_LOCALE + FLAG_Q_HITS + FLAG_Q_RATE + FLAG_Q_BOOMERANG
hit_type, success = self.create_hit_type(title=project.name, description=self.description,
price=project.price,
duration=duration, keywords=self.keywords,
approval_delay=datetime.timedelta(days=2),
qual_req=qualifications,
qualifications_mask=qualifications_mask,
boomerang_threshold=int(round(task[4], 2) * 100),
owner_id=project.owner_id, boomerang_qual=boomerang_qual)
if not success:
return 'FAILURE'
if mturk_hit is None:
try:
hit = self.connection.create_hit(hit_type=hit_type.string_id,
max_assignments=task[3],
lifetime=lifetime,
question=question)[0]
self.set_notification(hit_type_id=hit.HITTypeId)
mturk_hit = MTurkHIT(hit_id=hit.HITId, hit_type=hit_type, task_id=task[0])
except MTurkRequestError as e:
error = e.errors[0][0]
if error == 'AWS.MechanicalTurk.InsufficientFunds':
message = {
"type": "ERROR",
"detail": "Insufficient funds on your Mechanical Turk account!",
"code": error
}
redis_publisher = RedisPublisher(facility='bot', users=[project.owner])
message = RedisMessage(json.dumps(message))
redis_publisher.publish_message(message)
return 'FAILED'
else:
if mturk_hit.hit_type_id != hit_type.id:
result, success = self.change_hit_type_of_hit(hit_id=mturk_hit.hit_id,
hit_type_id=hit_type.string_id)
if success:
mturk_hit.hit_type = hit_type
mturk_hit.save()
return 'SUCCESS'
def create_hit_type(self, owner_id, title, description, price, duration, boomerang_threshold, keywords=None,
approval_delay=None, qual_req=None,
qualifications_mask=0, boomerang_qual=None):
hit_type = MTurkHITType.objects.filter(owner_id=owner_id, name=title, description=description,
price=Decimal(str(price)),
duration=duration,
qualifications_mask=qualifications_mask,
boomerang_threshold=boomerang_threshold).first()
if hit_type is not None:
return hit_type, True
reward = Price(price)
try:
mturk_ht = self.connection.register_hit_type(title=title, description=description, reward=reward,
duration=duration, keywords=keywords,
approval_delay=approval_delay,
qual_req=qual_req)[0]
hit_type = MTurkHITType(owner_id=owner_id, name=title, description=description,
price=Decimal(str(price)),
keywords=keywords, duration=duration,
qualifications_mask=qualifications_mask,
boomerang_qualification=boomerang_qual,
boomerang_threshold=boomerang_threshold)
hit_type.string_id = mturk_ht.HITTypeId
hit_type.save()
except MTurkRequestError:
return None, False
return hit_type, True
def create_external_question(self, task, frame_height=800):
task_hash = Hashids(salt=settings.SECRET_KEY, min_length=settings.ID_HASH_MIN_LENGTH)
task_id = task_hash.encode(task)
url = self.host + '/mturk/task/?taskId=' + task_id
question = ExternalQuestion(external_url=url, frame_height=frame_height)
return question
def update_max_assignments(self, task):
task = Task.objects.get(id=task['id'])
mturk_hit = task.mturk_hit
if not mturk_hit:
raise MTurkHIT.DoesNotExist("This task is not associated to any mturk hit")
assignments_completed = task.task_workers.filter(~Q(status__in=[TaskWorker.STATUS_REJECTED,
TaskWorker.STATUS_SKIPPED,
TaskWorker.STATUS_EXPIRED])).count()
remaining_assignments = task.project.repetition - assignments_completed
if remaining_assignments > 0 and mturk_hit.num_assignments == mturk_hit.mturk_assignments. \
filter(status=TaskWorker.STATUS_SUBMITTED).count() and \
mturk_hit.mturk_assignments.filter(status=TaskWorker.STATUS_IN_PROGRESS).count() == 0:
self.add_assignments(hit_id=mturk_hit.hit_id, increment=1)
self.extend_hit(hit_id=mturk_hit.hit_id)
mturk_hit.status = MTurkHIT.STATUS_IN_PROGRESS
mturk_hit.num_assignments += 1
mturk_hit.save()
elif remaining_assignments == 0:
self.expire_hit(hit_id=mturk_hit.hit_id)
mturk_hit.status = MTurkHIT.STATUS_EXPIRED
mturk_hit.save()
elif remaining_assignments > 0 and \
mturk_hit.status == MTurkHIT.STATUS_EXPIRED:
self.extend_hit(hit_id=mturk_hit.hit_id)
mturk_hit.status = MTurkHIT.STATUS_IN_PROGRESS
return 'SUCCESS'
def get_assignment(self, assignment_id):
try:
return self.connection.get_assignment(assignment_id)[0], True
except MTurkRequestError as e:
error = e.errors[0][0]
if error == 'AWS.MechanicalTurk.InvalidAssignmentState':
return assignment_id, False
return None, False
def set_notification(self, hit_type_id):
self.connection.set_rest_notification(hit_type=hit_type_id,
url=self.host + '/api/mturk/notification',
event_types=['AssignmentReturned', 'AssignmentAbandoned',
'AssignmentAccepted', 'AssignmentSubmitted'])
def approve_assignment(self, task_worker):
task_worker_obj = TaskWorker.objects.get(id=task_worker['id'])
if hasattr(task_worker_obj, 'mturk_assignments') and task_worker_obj.mturk_assignments.first() is not None:
try:
self.connection.approve_assignment(task_worker_obj.mturk_assignments.first().assignment_id)
except MTurkRequestError:
return False
return True
def reject_assignment(self, task_worker):
task_worker_obj = TaskWorker.objects.get(id=task_worker['id'])
if hasattr(task_worker_obj, 'mturk_assignments') and task_worker_obj.mturk_assignments.first() is not None:
try:
self.connection.reject_assignment(task_worker_obj.mturk_assignments.first().assignment_id)
except MTurkRequestError:
return False
return True
def expire_hit(self, hit_id):
try:
self.connection.expire_hit(hit_id)
except MTurkRequestError:
return False
return True
def disable_hit(self, hit_id):
try:
self.connection.disable_hit(hit_id)
except MTurkRequestError:
return False
return True
def extend_hit(self, hit_id):
try:
self.connection.extend_hit(hit_id=hit_id, expiration_increment=604800) # 7 days
except MTurkRequestError:
return False
return True
def add_assignments(self, hit_id, increment=1):
try:
self.connection.extend_hit(hit_id=hit_id, assignments_increment=increment)
except MTurkRequestError:
return False
return True
def test_connection(self):
try:
return self.connection.get_account_balance()[0], True
except MTurkRequestError as e:
error = e.errors[0][0]
if error == 'AWS.NotAuthorized':
return None, False
return None, False
def get_account_balance(self):
try:
return self.connection.get_account_balance()[0]
except MTurkRequestError:
return None
def create_qualification_type(self, owner_id, name, flag, description, project_id, auto_granted=False,
auto_granted_value=None, deny=False, bucket=None):
# noinspection SqlResolve
query = '''
SELECT * FROM (
SELECT
task.target_id,
task.username,
round(task.task_w_avg::NUMERIC, 2) rating
--round(coalesce(task.task_w_avg, requester.requester_w_avg,
-- platform.platform_w_avg)::NUMERIC, 2) rating
FROM (
SELECT
target_id,
origin_id,
project_id,
username,
sum(weight * power((%(BOOMERANG_TASK_ALPHA)s), t.row_number))
/ sum(power((%(BOOMERANG_TASK_ALPHA)s), t.row_number)) task_w_avg
FROM (
SELECT
r.id,
r.origin_id,
p.group_id project_id,
weight,
r.target_id,
-1 + row_number()
OVER (PARTITION BY target_id
ORDER BY tw.created_at DESC) AS row_number,
u.username username
FROM crowdsourcing_rating r
INNER JOIN crowdsourcing_task t ON t.id = r.task_id
INNER JOIN crowdsourcing_project p ON p.id = t.project_id
INNER JOIN crowdsourcing_taskworker tw ON t.id = tw.task_id
AND tw.worker_id=r.target_id
INNER JOIN auth_user u ON u.id = r.target_id
WHERE origin_id = (%(origin_id)s) AND origin_type = (%(origin_type)s)) t
GROUP BY origin_id, target_id, project_id, username)
task WHERE task.project_id = (%(project_id)s)
) r
'''
extra_query = 'WHERE rating BETWEEN (%(lower_bound)s) AND (%(upper_bound)s);'
params = {
'origin_type': Rating.RATING_REQUESTER, 'origin_id': owner_id, 'project_id': project_id,
'BOOMERANG_REQUESTER_ALPHA': settings.BOOMERANG_REQUESTER_ALPHA,
'BOOMERANG_PLATFORM_ALPHA': settings.BOOMERANG_PLATFORM_ALPHA,
'BOOMERANG_TASK_ALPHA': settings.BOOMERANG_TASK_ALPHA
}
obj_params = {'upper_bound': 300, 'lower_bound': 100}
if deny and bucket is not None:
query += extra_query
params.update({'upper_bound': bucket[1], 'lower_bound': bucket[0]})
obj_params.update({'upper_bound': bucket[1] * 100, 'lower_bound': bucket[0] * 100, 'is_blacklist': True})
cursor = connection.cursor()
cursor.execute(query, params=params)
worker_ratings_raw = cursor.fetchall()
worker_ratings = [{"worker_id": r[0], "worker_username": r[1], "rating": r[2]} for
r in worker_ratings_raw]
qualification = MTurkQualification.objects.filter(owner_id=owner_id, flag=flag, name=name).first()
assigned_workers = []
if qualification is None:
try:
qualification_type = self.connection. \
create_qualification_type(name=name, description=description,
status='Active',
auto_granted=auto_granted,
auto_granted_value=auto_granted_value)[0]
qualification = MTurkQualification.objects.create(owner_id=owner_id, flag=flag, name=name,
description=description,
auto_granted=auto_granted,
auto_granted_value=auto_granted_value,
type_id=qualification_type.QualificationTypeId,
**obj_params)
except MTurkRequestError:
return None, False
else:
assigned_workers = MTurkWorkerQualification.objects.values('worker').filter(
qualification=qualification).values_list('worker', flat=True)
for rating in worker_ratings:
user_name = rating["worker_username"].split('.')
if len(user_name) == 2 and user_name[0] == 'mturk':
mturk_worker_id = user_name[1].upper()
if mturk_worker_id not in assigned_workers:
self.assign_qualification(
qualification_type_id=qualification.type_id, worker_id=mturk_worker_id,
value=int(rating['rating'] * 100))
defaults = {
'qualification': qualification,
'worker': mturk_worker_id,
'score': int(rating['rating'] * 100)
}
MTurkWorkerQualification.objects.update_or_create(qualification=qualification,
worker=mturk_worker_id,
defaults=defaults)
return qualification, True
def change_hit_type_of_hit(self, hit_id, hit_type_id):
try:
result = self.connection.change_hit_type_of_hit(hit_id=hit_id, hit_type=hit_type_id)
except MTurkRequestError:
return None, False
return result, True
def update_worker_boomerang(self, project_id, worker_id, task_avg, requester_avg):
"""
Update boomerang for project
Args:
project_id:
worker_id:
task_avg:
requester_avg
Returns:
str
"""
hit = MTurkHIT.objects.select_related('hit_type__boomerang_qualification').filter(
task__project__group_id=project_id).first()
if hit is not None:
qualification = hit.hit_type.boomerang_qualification
worker_qual = MTurkWorkerQualification.objects.filter(qualification=qualification,
worker=worker_id).first()
if worker_qual is not None:
self.update_score(worker_qual, score=int(task_avg * 100), override=True)
else:
MTurkWorkerQualification.objects.create(qualification=qualification, worker=worker_id,
score=int(task_avg * 100), overwritten=True)
self.assign_qualification(qualification_type_id=qualification.type_id, worker_id=worker_id,
value=int(task_avg * 100))
# other_quals = MTurkWorkerQualification.objects.filter(~Q(qualification=qualification),
# worker=worker_id,
# overwritten=False)
# for q in other_quals:
# self.update_score(q, score=int(requester_avg * 100))
return 'SUCCESS'
def update_score(self, worker_qual, score, override=False):
if worker_qual is None:
return False
try:
self.connection.update_qualification_score(worker_qual.qualification.type_id, worker_qual.worker, score)
worker_qual.overwritten = override
worker_qual.score = score
worker_qual.save()
except MTurkRequestError:
return False
return True
def assign_qualification(self, qualification_type_id, worker_id,
value=1):
"""
Revoke a qualification from a WorkerId
Args:
qualification_type_id:
worker_id:
value
Returns:
bool
"""
try:
self.connection.assign_qualification(qualification_type_id, worker_id,
value, send_notification=False)
return True
except MTurkRequestError:
return False
def revoke_qualification(self, qualification_type_id, worker_id):
try:
self.connection.revoke_qualification(qualification_type_id=qualification_type_id, subject_id=worker_id)
return True
except MTurkRequestError:
return False
def notify_workers(self, worker_ids, subject, message_text):
try:
self.connection.notify_workers(worker_ids, subject, message_text)
return True
except MTurkRequestError:
return False
zip(assignheader, [
subject, assignid, row['hitid'], row['hittypeid'],
accepttime, submittime, errors, '',
row['assignmentstatus'], 'yes',
'yes' if len(errors) == 0 else ''
]))
# parse errors, extending HIT if asked
if (len(errors) > 0):
numErrors += 1
print(" Error: {0}".format(errors))
if replaceErrors and not asgn['alreadyExtended']:
# try to extend the hit
try:
print(" Extending hit...")
conn.extend_hit(row['hitid'], 1)
print(" Success")
numExtended += 1
asgn['alreadyExtended'] = 'yes'
except AWSConnectionError as reason:
print(" Failure: {0}".format(reason))
# write assignemtn info back to assignments file
assignmentsw.writerow(asgn)
# parse answer sections
for s, sfn, cond in ansSects:
# find sections in row matching section specification
sectkeys = [
rowkey for rowkey in row.keys()
if re.search(s, rowkey)
try:
#if "-e" in sys.argv or "--extend" in sys.argv
if NEWHIT.intersection(argset):
mtc.create_hit(question=question,
title=TITLE,
description=DESCRIPTION,
keywords='Photography',
duration=3600,
reward=payment,
qualifications=qualifications,
max_assignments=assignments,
approval_delay=0)
else:
DynamoHIT = list(mtc.get_all_hits())
if len(DynamoHIT) != 1:
print "Can't identify 1 clear HIT to extend, breaking."
if len(DynamoHIT) == 0:
print "You seem to have no HITs. Please use the -n or --new parameters to make one."
sys.exit(1)
else:
DynamoHIT = DynamoHIT[0]
print "extending HIT {} by {} assignments"\
.format(DynamoHIT.HITId,assignments)
mtc.extend_hit(DynamoHIT.HITId, assignments=assignments)
except MTurkRequestError as e:
print 'request failed'
print e.body
else:
print 'request successful'
# get information about the HIT
hit, = conn.get_hit(args.hit, ['HITDetail', 'HITAssignmentSummary'])
total = int(hit.MaxAssignments)
pending = int(hit.NumberOfAssignmentsPending)
complete = int(hit.NumberOfAssignmentsCompleted)
available = int(hit.NumberOfAssignmentsAvailable)
logging.info(
"max:%s/pending:%s/complete:%s/remain:%s",
total, pending, complete, available)
# check if we have reached the total
if total >= args.total:
logging.info("MaxAssignments = %s, exiting", total)
break
# compute how many assignments are currently outstanding
current = available + pending
if current < args.concurrent:
diff = min(args.total - total, args.concurrent - current)
logging.info("Extending HIT with %s more assignments", diff)
conn.extend_hit(args.hit, assignments_increment=diff)
# get submitted assignments and approve them
if args.approve:
assignments = conn.get_assignments(args.hit, status="Submitted", page_size=100)
for assignment in assignments:
logging.info("Approving assignment %s", assignment.AssignmentId)
conn.approve_assignment(assignment.AssignmentId, feedback=None)
time.sleep(args.interval)
class MTurkServices:
def __init__(self, config):
self.config = config
def get_active_hits(self):
self.connect_to_turk()
# hits = self.mtc.search_hits()
try:
hits = self.mtc.get_all_hits()
except MTurkRequestError:
return(False)
active_hits = [hit for hit in hits if not(hit.expired)]
hits_data = [{'hitid': hit.HITId,
'title': hit.Title,
'status': hit.HITStatus,
'max_assignments': hit.MaxAssignments,
'number_assignments_completed': hit.NumberOfAssignmentsCompleted,
'number_assignments_pending': hit.NumberOfAssignmentsCompleted,
'number_assignments_available': hit.NumberOfAssignmentsAvailable,
'creation_time': hit.CreationTime,
'expiration': hit.Expiration,
} for hit in active_hits]
return(hits_data)
def get_workers(self):
self.connect_to_turk()
try:
hits = self.mtc.search_hits(sort_direction='Descending', page_size=20)
hit_ids = [hit.HITId for hit in hits]
workers_nested = [self.mtc.get_assignments(
hit_id,
status="Submitted",
sort_by='SubmitTime',
page_size=100
) for hit_id in hit_ids]
workers = [val for subl in workers_nested for val in subl] # Flatten nested lists
except MTurkRequestError:
return(False)
completed_workers = [worker for worker in workers if worker.AssignmentStatus == "Submitted"]
worker_data = [{'hitId': worker.HITId,
'assignmentId': worker.AssignmentId,
'workerId': worker.WorkerId,
'submit_time': worker.SubmitTime,
'accept_time': worker.AcceptTime
} for worker in completed_workers]
return(worker_data)
def approve_worker(self, assignment_id):
self.connect_to_turk()
try:
self.mtc.approve_assignment(assignment_id, feedback=None)
except MTurkRequestError:
return(False)
def reject_worker(self, assignment_id):
self.connect_to_turk()
try:
self.mtc.reject_assignment(assignment_id, feedback=None)
except MTurkRequestError:
return(False)
def verify_aws_login(self, key_id, secret_key):
is_sandbox = self.config.getboolean('HIT Configuration', 'using_sandbox')
if is_sandbox:
host = 'mechanicalturk.sandbox.amazonaws.com'
else:
host = 'mechanicalturk.amazonaws.com'
mturkparams = dict(
aws_access_key_id=key_id,
aws_secret_access_key=secret_key,
host=host)
self.mtc = MTurkConnection(**mturkparams)
try:
self.mtc.get_account_balance()
except MTurkRequestError as e:
print(e.error_message)
print('AWS Credentials invalid')
return 0
else:
print('AWS Credentials valid')
return 1
def connect_to_turk(self):
is_sandbox = self.config.getboolean('HIT Configuration', 'using_sandbox')
if is_sandbox:
host = 'mechanicalturk.sandbox.amazonaws.com'
else:
host = 'mechanicalturk.amazonaws.com'
mturkparams = dict(
aws_access_key_id = self.config.get('AWS Access', 'aws_access_key_id'),
aws_secret_access_key = self.config.get('AWS Access', 'aws_secret_access_key'),
host=host)
self.mtc = MTurkConnection(**mturkparams)
def configure_hit(self):
# Configure portal
experimentPortalURL = self.config.get('HIT Configuration', 'question_url')
frameheight = 600
mturkQuestion = ExternalQuestion(experimentPortalURL, frameheight)
# Qualification:
quals = Qualifications()
approve_requirement = self.config.get('HIT Configuration', 'Approve_Requirement')
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if self.config.getboolean('HIT Configuration', 'US_only'):
quals.add(LocaleRequirement("EqualTo", "US"))
# Specify all the HIT parameters
self.paramdict = dict(
hit_type = None,
question = mturkQuestion,
lifetime = datetime.timedelta(hours=self.config.getfloat('HIT Configuration', 'HIT_lifetime')),
max_assignments = self.config.getint('HIT Configuration', 'max_assignments'),
title = self.config.get('HIT Configuration', 'title'),
description = self.config.get('HIT Configuration', 'description'),
keywords = self.config.get('HIT Configuration', 'keywords'),
reward = self.config.getfloat('HIT Configuration', 'reward'),
duration = datetime.timedelta(hours=self.config.getfloat('HIT Configuration', 'duration')),
approval_delay = None,
questions = None,
qualifications = quals
)
def is_signed_up(self):
access_key_id = self.config.get('AWS Access', 'aws_access_key_id')
access_key = self.config.get('AWS Access', 'aws_secret_access_key')
return (access_key_id != 'YourAccessKeyId') and \
(access_key != 'YourSecreteAccessKey')
def check_balance(self):
if self.is_signed_up():
self.connect_to_turk()
return(self.mtc.get_account_balance()[0])
else:
return('-')
# TODO (if valid AWS credentials haven't been provided then connect_to_turk() will
# fail, not error checking here and elsewhere)
def create_hit(self):
self.connect_to_turk()
self.configure_hit()
myhit = self.mtc.create_hit(**self.paramdict)[0]
self.hitid = myhit.HITId
# TODO(Jay): Have a wrapper around functions that serializes them.
# Default output should not be serialized.
def expire_hit(self, hitid):
self.connect_to_turk()
self.mtc.expire_hit(hitid)
def extend_hit(self, hitid, assignments_increment=None, expiration_increment=None):
self.connect_to_turk()
self.mtc.extend_hit(hitid, assignments_increment=int(assignments_increment))
self.mtc.extend_hit(hitid, expiration_increment=int(expiration_increment)*60)
def get_summary(self):
try:
balance = self.check_balance()
summary = jsonify(balance=str(balance))
return(summary)
except MTurkRequestError as e:
print(e.error_message)
return(False)
