def test_check_credentials_no_creds_set_raises(self, with_mock):
creds = {
'aws_access_key_id': '',
'aws_secret_access_key': '',
'region_name': 'us-east-1'
}
service = MTurkService(**creds)
with pytest.raises(MTurkServiceException):
service.check_credentials()
def test_check_credentials_no_creds_set_raises(self, with_mock):
creds = {
"aws_access_key_id": "",
"aws_secret_access_key": "",
"region_name": "us-east-1",
}
service = MTurkService(**creds)
with pytest.raises(MTurkServiceException):
service.check_credentials()
class MTurkRecruiter(object):
"""Recruit participants from Amazon Mechanical Turk"""
@classmethod
def from_current_config(cls):
config = get_config()
if not config.ready:
config.load_config()
ad_url = '{}/ad'.format(get_base_url())
hit_domain = os.getenv('HOST')
return cls(config, hit_domain, ad_url)
def __init__(self, config, hit_domain, ad_url):
self.config = config
self.ad_url = ad_url
self.hit_domain = hit_domain
self.mturkservice = MTurkService(
self.config.get('aws_access_key_id'),
self.config.get('aws_secret_access_key'),
(self.config.get('mode') == "sandbox"))
def open_recruitment(self, n=1):
"""Open a connection to AWS MTurk and create a HIT."""
if self.is_in_progress:
# Already started... do nothing.
return None
if self.config.get('mode') == 'debug':
raise MTurkRecruiterException("Can't run a HIT in debug mode")
if self.hit_domain is None:
raise MTurkRecruiterException("Can't run a HIT from localhost")
self.mturkservice.check_credentials()
hit_request = {
'max_assignments': n,
'title': self.config.get('title'),
'description': self.config.get('description'),
'keywords': self.config.get('amt_keywords'),
'reward': self.config.get('base_payment'),
'duration_hours': self.config.get('duration'),
'lifetime_days': self.config.get('lifetime'),
'ad_url': self.ad_url,
'notification_url': self.config.get('notification_url'),
'approve_requirement': self.config.get('approve_requirement'),
'us_only': self.config.get('us_only'),
}
hit_info = self.mturkservice.create_hit(**hit_request)
if self.config.get('mode') == "sandbox":
lookup_url = "https://workersandbox.mturk.com/mturk/preview?groupId={type_id}"
else:
lookup_url = "https://worker.mturk.com/mturk/preview?groupId={type_id}"
return lookup_url.format(**hit_info)
def recruit_participants(self, n=1):
"""Recruit n new participants to an existing HIT"""
if not self.config.get('auto_recruit', False):
logger.info('auto_recruit is False: recruitment suppressed')
return
hit_id = self.current_hit_id()
if hit_id is None:
logger.info('no HIT in progress: recruitment aborted')
return
return self.mturkservice.extend_hit(
hit_id, number=n, duration_hours=self.config.get('duration'))
def reward_bonus(self, assignment_id, amount, reason):
"""Reward the Turker for a specified assignment with a bonus."""
return self.mturkservice.grant_bonus(assignment_id, amount, reason)
@property
def is_in_progress(self):
return bool(Participant.query.first())
def current_hit_id(self):
any_participant_record = Participant.query.with_entities(
Participant.hit_id).first()
if any_participant_record is not None:
return str(any_participant_record.hit_id)
def approve_hit(self, assignment_id):
return self.mturkservice.approve_assignment(assignment_id)
def close_recruitment(self):
"""Clean up once the experiment is complete.
This does nothing, because the fact that this is called means
that all MTurk HITs that were created were already completed.
"""
pass
class MTurkRecruiter(Recruiter):
"""Recruit participants from Amazon Mechanical Turk"""
nickname = "mturk"
extra_routes = mturk_routes
def __init__(self, *args, **kwargs):
super(MTurkRecruiter, self).__init__()
self.config = get_config()
base_url = get_base_url()
self.ad_url = "{}/ad?recruiter={}".format(base_url, self.nickname)
self.notification_url = "{}/mturk-sns-listener".format(base_url)
self.hit_domain = os.getenv("HOST")
self.mturkservice = MTurkService(
aws_access_key_id=self.config.get("aws_access_key_id"),
aws_secret_access_key=self.config.get("aws_secret_access_key"),
region_name=self.config.get("aws_region"),
sandbox=self.config.get("mode") != "live",
)
self.notifies_admin = admin_notifier(self.config)
self.mailer = get_mailer(self.config)
self.store = kwargs.get("store") or RedisStore()
self._validate_config()
def _validate_config(self):
mode = self.config.get("mode")
if mode not in ("sandbox", "live"):
raise MTurkRecruiterException(
'"{}" is not a valid mode for MTurk recruitment. '
'The value of "mode" must be either "sandbox" or "live"'.
format(mode))
def exit_response(self, experiment, participant):
return flask.render_template(
"exit_recruiter_mturk.html",
hitid=participant.hit_id,
assignmentid=participant.assignment_id,
workerid=participant.worker_id,
external_submit_url=self.external_submission_url,
)
@property
def external_submission_url(self):
"""On experiment completion, participants are returned to
the Mechanical Turk site to submit their HIT, which in turn triggers
notifications to the /mturk-sns-listener route.
"""
if self.is_sandbox:
return "https://workersandbox.mturk.com/mturk/externalSubmit"
return "https://www.mturk.com/mturk/externalSubmit"
def open_recruitment(self, n=1):
"""Open a connection to AWS MTurk and create a HIT."""
logger.info("Opening MTurk recruitment for {} participants".format(n))
if self.is_in_progress:
raise MTurkRecruiterException(
"Tried to open_recruitment on already open recruiter.")
if self.hit_domain is None:
raise MTurkRecruiterException("Can't run a HIT from localhost")
self.mturkservice.check_credentials()
hit_request = {
"experiment_id":
self.config.get("id"),
"max_assignments":
n,
"title":
"{} ({})".format(self.config.get("title"),
heroku_tools.app_name(self.config.get("id"))),
"description":
self.config.get("description"),
"keywords":
self._config_to_list("keywords"),
"reward":
self.config.get("base_payment"),
"duration_hours":
self.config.get("duration"),
"lifetime_days":
self.config.get("lifetime"),
"question":
MTurkQuestions.external(self.ad_url),
"notification_url":
self.notification_url,
"annotation":
self.config.get("id"),
"qualifications":
self._build_required_hit_qualifications(),
}
hit_info = self.mturkservice.create_hit(**hit_request)
self._record_current_hit_id(hit_info["id"])
url = hit_info["worker_url"]
return {
"items": [url],
"message": "HIT now published to Amazon Mechanical Turk",
}
def assign_experiment_qualifications(self, worker_id, qualifications):
"""Assigns MTurk Qualifications to a worker.
@param worker_id string the MTurk worker ID
@param qualifications list of dict w/ `name`, `description` and
(optional) `score` keys
"""
by_name = {qual["name"]: qual for qual in qualifications}
result = self._ensure_mturk_qualifications(qualifications)
for qual in result["new_qualifications"]:
score = by_name[qual["name"]].get("score")
if score is not None:
self.mturkservice.assign_qualification(qual["id"], worker_id,
qual["score"])
else:
self.mturkservice.increment_qualification_score(
qual["id"], worker_id)
for name in result["existing_qualifications"]:
score = by_name[name].get("score")
if score is not None:
self.mturkservice.assign_named_qualification(
name, worker_id, score)
else:
self.mturkservice.increment_named_qualification_score(
name, worker_id)
def compensate_worker(self, worker_id, email, dollars, notify=True):
"""Pay a worker by means of a special HIT that only they can see."""
qualification = self.mturkservice.create_qualification_type(
name="Dallinger Compensation Qualification - {}".format(
generate_random_id()),
description=(
"You have received a qualification to allow you to complete a "
"compensation HIT from Dallinger for ${}.".format(dollars)),
)
qid = qualification["id"]
self.mturkservice.assign_qualification(qid,
worker_id,
1,
notify=notify)
hit_request = {
"experiment_id": "(compensation only)",
"max_assignments": 1,
"title": "Dallinger Compensation HIT",
"description": "For compenation only; no task required.",
"keywords": [],
"reward": float(dollars),
"duration_hours": 1,
"lifetime_days": 3,
"question": MTurkQuestions.compensation(sandbox=self.is_sandbox),
"qualifications": [MTurkQualificationRequirements.must_have(qid)],
"do_subscribe": False,
}
hit_info = self.mturkservice.create_hit(**hit_request)
if email is not None:
message = {
"subject":
"Dallinger Compensation HIT",
"sender":
self.config.get("dallinger_email_address"),
"recipients": [email],
"body":
("A special compensation HIT is available for you to complete on MTurk.\n\n"
"Title: {title}\n"
"Reward: ${reward:.2f}\n"
"URL: {worker_url}").format(**hit_info),
}
self.mailer.send(**message)
else:
message = {}
return {
"hit": hit_info,
"qualification": qualification,
"email": message
}
def recruit(self, n=1):
"""Recruit n new participants to an existing HIT"""
logger.info("Recruiting {} MTurk participants".format(n))
if not self.config.get("auto_recruit"):
logger.info("auto_recruit is False: recruitment suppressed")
return
hit_id = self.current_hit_id()
if hit_id is None:
logger.info("no HIT in progress: recruitment aborted")
return
try:
return self.mturkservice.extend_hit(
hit_id, number=n, duration_hours=self.config.get("duration"))
except MTurkServiceException as ex:
logger.exception(str(ex))
def notify_duration_exceeded(self, participants, reference_time):
"""The participant has exceed the maximum time for the activity,
defined in the "duration" config value. We need find out the assignment
status on MTurk and act based on this.
"""
unsubmitted = []
for participant in participants:
summary = ParticipationTime(participant, reference_time,
self.config)
status = self._mturk_status_for(participant)
if status == "Approved":
participant.status = "approved"
session.commit()
elif status == "Rejected":
participant.status = "rejected"
session.commit()
elif status == "Submitted":
self._resend_submitted_rest_notification_for(participant)
self._message_researcher(self._resubmitted_msg(summary))
logger.warning(
"Error - submitted notification for participant {} missed. "
"A replacement notification was created and sent, "
"but proceed with caution.".format(participant.id))
else:
self._send_notification_missing_rest_notification_for(
participant)
unsubmitted.append(summary)
disable_hit = self.config.get("disable_when_duration_exceeded")
if disable_hit and unsubmitted:
self._disable_autorecruit()
self.close_recruitment()
pick_one = unsubmitted[0]
# message the researcher about the one of the participants:
self._message_researcher(self._cancelled_msg(pick_one))
# Attempt to force-expire the hit via boto. It's possible
# that the HIT won't exist if the HIT has been deleted manually.
try:
self.mturkservice.expire_hit(pick_one.participant.hit_id)
except MTurkServiceException as ex:
logger.exception(ex)
def rejects_questionnaire_from(self, participant):
"""Mechanical Turk participants submit their HITs on the MTurk site
(see external_submission_url), and MTurk then sends a notification
to Dallinger which is used to mark the assignment completed.
If a HIT has already been submitted, it's too late to submit the
questionnaire.
"""
if participant.status != "working":
return ("This participant has already sumbitted their HIT "
"on MTurk and can no longer submit the questionnaire")
def submitted_event(self):
"""MTurk will send its own notification when the worker
completes the HIT on that service.
"""
return None
def reward_bonus(self, assignment_id, amount, reason):
"""Reward the Turker for a specified assignment with a bonus."""
try:
return self.mturkservice.grant_bonus(assignment_id, amount, reason)
except MTurkServiceException as ex:
logger.exception(str(ex))
@property
def is_in_progress(self):
"""Does an MTurk HIT for the current experiment ID already exist?"""
return self.current_hit_id() is not None
def current_hit_id(self):
"""Return the ID of the HIT associated with the active experiment ID
if any such HIT exists.
"""
return self.store.get(self.hit_id_storage_key)
def approve_hit(self, assignment_id):
try:
return self.mturkservice.approve_assignment(assignment_id)
except MTurkServiceException as ex:
logger.exception(str(ex))
def close_recruitment(self):
"""Clean up once the experiment is complete.
This may be called before all users have finished so uses the
expire_hit rather than the disable_hit API call. This allows people
who have already picked up the hit to complete it as normal.
"""
logger.info(CLOSE_RECRUITMENT_LOG_PREFIX + " mturk")
# We are not expiring the hit currently as notifications are failing
# TODO: Reinstate this
# try:
# return self.mturkservice.expire_hit(
# self.current_hit_id(),
# )
# except MTurkServiceException as ex:
# logger.exception(str(ex))
@property
def is_sandbox(self):
return self.config.get("mode") == "sandbox"
@property
def hit_id_storage_key(self):
experiment_id = self.config.get("id")
return "{}:{}".format(self.__class__.__name__, experiment_id)
def _build_required_hit_qualifications(self):
# The Qualications an MTurk worker must have, or in the case of the
# blocklist, not have, in order for them to see and accept the HIT.
quals = []
reqs = MTurkQualificationRequirements
if self.config.get("approve_requirement") is not None:
quals.append(
reqs.min_approval(self.config.get("approve_requirement")))
if self.config.get("us_only"):
quals.append(reqs.restrict_to_countries(["US"]))
for item in self._config_to_list("mturk_qualification_blocklist"):
qtype = self.mturkservice.get_qualification_type_by_name(item)
if qtype:
quals.append(reqs.must_not_have(qtype["id"]))
if self.config.get("mturk_qualification_requirements",
None) is not None:
explicit_qualifications = json.loads(
self.config.get("mturk_qualification_requirements"))
quals.extend(explicit_qualifications)
return quals
def _record_current_hit_id(self, hit_id):
self.store.set(self.hit_id_storage_key, hit_id)
def _confirm_sns_subscription(self, token, topic):
self.mturkservice.confirm_subscription(token=token, topic=topic)
def _report_event_notification(self, events):
q = _get_queue()
for event in events:
event_type = event.get("EventType")
assignment_id = event.get("AssignmentId")
participant_id = None
q.enqueue(worker_function, event_type, assignment_id,
participant_id)
def _mturk_status_for(self, participant):
try:
assignment = self.mturkservice.get_assignment(
participant.assignment_id)
status = assignment["status"]
except Exception:
status = None
return status
def _disable_autorecruit(self):
heroku_app = heroku_tools.HerokuApp(
self.config.get("heroku_app_id_root"))
args = json.dumps({"auto_recruit": "false"})
headers = heroku_tools.request_headers(
self.config.get("heroku_auth_token"))
requests.patch(heroku_app.config_url, data=args, headers=headers)
def _resend_submitted_rest_notification_for(self, participant):
q = _get_queue()
q.enqueue(worker_function, "AssignmentSubmitted",
participant.assignment_id, None)
def _send_notification_missing_rest_notification_for(self, participant):
q = _get_queue()
q.enqueue(worker_function, "NotificationMissing",
participant.assignment_id, None)
def _config_to_list(self, key):
# At some point we'll support lists, so all service code supports them,
# but the config system only supports strings for now, so we convert:
as_string = self.config.get(key, None)
if as_string is None:
return []
return [item.strip() for item in as_string.split(",") if item.strip()]
def _ensure_mturk_qualifications(self, qualifications):
"""Create MTurk Qualifications for names that don't already exist,
but also return names that already do.
"""
result = {"new_qualifications": [], "existing_qualifications": []}
for qual in qualifications:
name = qual["name"]
desc = qual["description"]
try:
result["new_qualifications"].append({
"name":
name,
"id":
self.mturkservice.create_qualification_type(name,
desc)["id"],
"available":
False,
})
except DuplicateQualificationNameError:
result["existing_qualifications"].append(name)
# We need to make sure the new qualifications are actually ready
# for assignment, as there's a small delay.
for tries in range(5):
for new in result["new_qualifications"]:
if new["available"]:
continue
try:
self.mturkservice.get_qualification_type_by_name(
new["name"])
except QualificationNotFoundException:
logger.warn(
"Did not find qualification {}. Trying again...".
format(new["name"]))
time.sleep(1)
else:
new["available"] = True
if all([n["available"] for n in result["new_qualifications"]]):
break
unavailable = [
q for q in result["new_qualifications"] if not q["available"]
]
if unavailable:
logger.warn(
"After several attempts, some qualifications are still not ready "
"for assignment: {}".format(", ".join(unavailable)))
# Return just the available among the new ones
result["new_qualifications"] = [
q for q in result["new_qualifications"] if q["available"]
]
return result
def _resubmitted_msg(self, summary):
templates = MTurkHITMessages.by_flavor(summary,
self.config.get("whimsical"))
return templates.resubmitted_msg()
def _cancelled_msg(self, summary):
templates = MTurkHITMessages.by_flavor(summary,
self.config.get("whimsical"))
return templates.hit_cancelled_msg()
def _message_researcher(self, message):
try:
self.notifies_admin.send(message["subject"], message["body"])
except MessengerError as ex:
logger.exception(ex)
class MTurkRecruiter(Recruiter):
"""Recruit participants from Amazon Mechanical Turk"""
nickname = 'mturk'
experiment_qualification_desc = 'Experiment-specific qualification'
group_qualification_desc = 'Experiment group qualification'
def __init__(self):
super(MTurkRecruiter, self).__init__()
self.config = get_config()
self.ad_url = '{}/ad?recruiter={}'.format(
get_base_url(),
self.nickname,
)
self.hit_domain = os.getenv('HOST')
self.mturkservice = MTurkService(
self.config.get('aws_access_key_id'),
self.config.get('aws_secret_access_key'),
self.config.get('aws_region'),
self.config.get('mode') != "live")
self._validate_config()
def _validate_config(self):
mode = self.config.get('mode')
if mode not in ('sandbox', 'live'):
raise MTurkRecruiterException(
'"{}" is not a valid mode for MTurk recruitment. '
'The value of "mode" must be either "sandbox" or "live"'.
format(mode))
@property
def external_submission_url(self):
"""On experiment completion, participants are returned to
the Mechanical Turk site to submit their HIT, which in turn triggers
notifications to the /notifications route.
"""
if self.config.get('mode') == "sandbox":
return "https://workersandbox.mturk.com/mturk/externalSubmit"
return "https://www.mturk.com/mturk/externalSubmit"
@property
def qualifications(self):
quals = {self.config.get('id'): self.experiment_qualification_desc}
group_name = self.config.get('group_name', None)
if group_name:
quals[group_name] = self.group_qualification_desc
return quals
def open_recruitment(self, n=1):
"""Open a connection to AWS MTurk and create a HIT."""
if self.is_in_progress:
# Already started... do nothing.
return None
if self.hit_domain is None:
raise MTurkRecruiterException("Can't run a HIT from localhost")
self.mturkservice.check_credentials()
if self.config.get('assign_qualifications'):
self._create_mturk_qualifications()
hit_request = {
'max_assignments': n,
'title': self.config.get('title'),
'description': self.config.get('description'),
'keywords': self._config_to_list('keywords'),
'reward': self.config.get('base_payment'),
'duration_hours': self.config.get('duration'),
'lifetime_days': self.config.get('lifetime'),
'ad_url': self.ad_url,
'notification_url': self.config.get('notification_url'),
'approve_requirement': self.config.get('approve_requirement'),
'us_only': self.config.get('us_only'),
'blacklist': self._config_to_list('qualification_blacklist'),
'annotation': self.config.get('id'),
}
hit_info = self.mturkservice.create_hit(**hit_request)
if self.config.get('mode') == "sandbox":
lookup_url = "https://workersandbox.mturk.com/mturk/preview?groupId={type_id}"
else:
lookup_url = "https://worker.mturk.com/mturk/preview?groupId={type_id}"
return {
'items': [
lookup_url.format(**hit_info),
],
'message': 'HIT now published to Amazon Mechanical Turk'
}
def recruit(self, n=1):
"""Recruit n new participants to an existing HIT"""
if not self.config.get('auto_recruit', False):
logger.info('auto_recruit is False: recruitment suppressed')
return
hit_id = self.current_hit_id()
if hit_id is None:
logger.info('no HIT in progress: recruitment aborted')
return
try:
return self.mturkservice.extend_hit(
hit_id, number=n, duration_hours=self.config.get('duration'))
except MTurkServiceException as ex:
logger.exception(str(ex))
def notify_completed(self, participant):
"""Assign a Qualification to the Participant for the experiment ID,
and for the configured group_name, if it's been set.
"""
if not self.config.get('assign_qualifications'):
return
worker_id = participant.worker_id
for name in self.qualifications:
try:
self.mturkservice.increment_qualification_score(
name, worker_id)
except QualificationNotFoundException as ex:
logger.exception(ex)
def rejects_questionnaire_from(self, participant):
"""Mechanical Turk participants submit their HITs on the MTurk site
(see external_submission_url), and MTurk then sends a notification
to Dallinger which is used to mark the assignment completed.
If a HIT has already been submitted, it's too late to submit the
questionnaire.
"""
if participant.status != "working":
return ("This participant has already sumbitted their HIT "
"on MTurk and can no longer submit the questionnaire")
def submitted_event(self):
"""MTurk will send its own notification when the worker
completes the HIT on that service.
"""
return None
def reward_bonus(self, assignment_id, amount, reason):
"""Reward the Turker for a specified assignment with a bonus."""
try:
return self.mturkservice.grant_bonus(assignment_id, amount, reason)
except MTurkServiceException as ex:
logger.exception(str(ex))
@property
def is_in_progress(self):
return bool(Participant.query.first())
def current_hit_id(self):
any_participant_record = Participant.query.with_entities(
Participant.hit_id).first()
if any_participant_record is not None:
return str(any_participant_record.hit_id)
def approve_hit(self, assignment_id):
try:
return self.mturkservice.approve_assignment(assignment_id)
except MTurkServiceException as ex:
logger.exception(str(ex))
def close_recruitment(self):
"""Clean up once the experiment is complete.
This may be called before all users have finished so uses the
expire_hit rather than the disable_hit API call. This allows people
who have already picked up the hit to complete it as normal.
"""
logger.info(CLOSE_RECRUITMENT_LOG_PREFIX)
# We are not expiring the hit currently as notifications are failing
# TODO: Reinstate this
# try:
# return self.mturkservice.expire_hit(
# self.current_hit_id(),
# )
# except MTurkServiceException as ex:
# logger.exception(str(ex))
def _config_to_list(self, key):
# At some point we'll support lists, so all service code supports them,
# but the config system only supports strings for now, so we convert:
as_string = self.config.get(key, '')
return [item.strip() for item in as_string.split(',') if item.strip()]
def _create_mturk_qualifications(self):
"""Create MTurk Qualification for experiment ID, and for group_name
if it's been set. Qualifications with these names already exist, but
it's faster to try and fail than to check, then try.
"""
for name, desc in self.qualifications.items():
try:
self.mturkservice.create_qualification_type(name, desc)
except DuplicateQualificationNameError:
pass
class MTurkRecruiter(Recruiter):
"""Recruit participants from Amazon Mechanical Turk"""
nickname = "mturk"
extra_routes = mturk_routes
experiment_qualification_desc = "Experiment-specific qualification"
group_qualification_desc = "Experiment group qualification"
def __init__(self):
super(MTurkRecruiter, self).__init__()
self.config = get_config()
base_url = get_base_url()
self.ad_url = "{}/ad?recruiter={}".format(base_url, self.nickname)
self.notification_url = "{}/mturk-sns-listener".format(base_url)
self.hit_domain = os.getenv("HOST")
self.mturkservice = MTurkService(
aws_access_key_id=self.config.get("aws_access_key_id"),
aws_secret_access_key=self.config.get("aws_secret_access_key"),
region_name=self.config.get("aws_region"),
sandbox=self.config.get("mode") != "live",
)
self.notifies_admin = admin_notifier(self.config)
self.mailer = get_mailer(self.config)
self._validate_config()
def _validate_config(self):
mode = self.config.get("mode")
if mode not in ("sandbox", "live"):
raise MTurkRecruiterException(
'"{}" is not a valid mode for MTurk recruitment. '
'The value of "mode" must be either "sandbox" or "live"'.
format(mode))
@property
def external_submission_url(self):
"""On experiment completion, participants are returned to
the Mechanical Turk site to submit their HIT, which in turn triggers
notifications to the /notifications route.
"""
if self.is_sandbox:
return "https://workersandbox.mturk.com/mturk/externalSubmit"
return "https://www.mturk.com/mturk/externalSubmit"
@property
def qualifications(self):
quals = {self.config.get("id"): self.experiment_qualification_desc}
group_name = self.config.get("group_name", None)
if group_name:
quals[group_name] = self.group_qualification_desc
return quals
def open_recruitment(self, n=1):
"""Open a connection to AWS MTurk and create a HIT."""
logger.info("Opening MTurk recruitment for {} participants".format(n))
if self.is_in_progress:
raise MTurkRecruiterException(
"Tried to open_recruitment on already open recruiter.")
if self.hit_domain is None:
raise MTurkRecruiterException("Can't run a HIT from localhost")
self.mturkservice.check_credentials()
if self.config.get("assign_qualifications"):
self._create_mturk_qualifications()
hit_request = {
"experiment_id": self.config.get("id"),
"max_assignments": n,
"title": self.config.get("title"),
"description": self.config.get("description"),
"keywords": self._config_to_list("keywords"),
"reward": self.config.get("base_payment"),
"duration_hours": self.config.get("duration"),
"lifetime_days": self.config.get("lifetime"),
"question": MTurkQuestions.external(self.ad_url),
"notification_url": self.notification_url,
"annotation": self.config.get("id"),
"qualifications": self._build_hit_qualifications(),
}
hit_info = self.mturkservice.create_hit(**hit_request)
url = hit_info["worker_url"]
return {
"items": [url],
"message": "HIT now published to Amazon Mechanical Turk",
}
def compensate_worker(self, worker_id, email, dollars, notify=True):
"""Pay a worker by means of a special HIT that only they can see.
"""
qualification = self.mturkservice.create_qualification_type(
name="Dallinger Compensation Qualification - {}".format(
generate_random_id()),
description=(
"You have received a qualification to allow you to complete a "
"compensation HIT from Dallinger for ${}.".format(dollars)),
)
qid = qualification["id"]
self.mturkservice.assign_qualification(qid,
worker_id,
1,
notify=notify)
hit_request = {
"experiment_id": "(compensation only)",
"max_assignments": 1,
"title": "Dallinger Compensation HIT",
"description": "For compenation only; no task required.",
"keywords": [],
"reward": float(dollars),
"duration_hours": 1,
"lifetime_days": 3,
"question": MTurkQuestions.compensation(sandbox=self.is_sandbox),
"qualifications": [MTurkQualificationRequirements.must_have(qid)],
"do_subscribe": False,
}
hit_info = self.mturkservice.create_hit(**hit_request)
if email is not None:
message = {
"subject":
"Dallinger Compensation HIT",
"sender":
self.config.get("dallinger_email_address"),
"recipients": [email],
"body":
("A special compenstation HIT is available for you to complete on MTurk.\n\n"
"Title: {title}\n"
"Reward: ${reward:.2f}\n"
"URL: {worker_url}").format(**hit_info),
}
self.mailer.send(**message)
else:
message = {}
return {
"hit": hit_info,
"qualification": qualification,
"email": message
}
def recruit(self, n=1):
"""Recruit n new participants to an existing HIT"""
logger.info("Recruiting {} MTurk participants".format(n))
if not self.config.get("auto_recruit"):
logger.info("auto_recruit is False: recruitment suppressed")
return
hit_id = self.current_hit_id()
if hit_id is None:
logger.info("no HIT in progress: recruitment aborted")
return
try:
return self.mturkservice.extend_hit(
hit_id, number=n, duration_hours=self.config.get("duration"))
except MTurkServiceException as ex:
logger.exception(str(ex))
def notify_completed(self, participant):
"""Assign a Qualification to the Participant for the experiment ID,
and for the configured group_name, if it's been set.
Overrecruited participants don't receive qualifications, since they
haven't actually completed the experiment. This allows them to remain
eligible for future runs.
"""
if participant.status == "overrecruited" or not self.qualification_active:
return
worker_id = participant.worker_id
for name in self.qualifications:
try:
self.mturkservice.increment_qualification_score(
name, worker_id)
except QualificationNotFoundException as ex:
logger.exception(ex)
def notify_duration_exceeded(self, participants, reference_time):
"""The participant has exceed the maximum time for the activity,
defined in the "duration" config value. We need find out the assignment
status on MTurk and act based on this.
"""
unsubmitted = []
for participant in participants:
summary = ParticipationTime(participant, reference_time,
self.config)
status = self._mturk_status_for(participant)
if status == "Approved":
participant.status = "approved"
session.commit()
elif status == "Rejected":
participant.status = "rejected"
session.commit()
elif status == "Submitted":
self._resend_submitted_rest_notification_for(participant)
self._message_researcher(self._resubmitted_msg(summary))
logger.warning(
"Error - submitted notification for participant {} missed. "
"A replacement notification was created and sent, "
"but proceed with caution.".format(participant.id))
else:
self._send_notification_missing_rest_notification_for(
participant)
unsubmitted.append(summary)
if unsubmitted:
self._disable_autorecruit()
self.close_recruitment()
pick_one = unsubmitted[0]
# message the researcher about the one of the participants:
self._message_researcher(self._cancelled_msg(pick_one))
# Attempt to force-expire the hit via boto. It's possible
# that the HIT won't exist if the HIT has been deleted manually.
try:
self.mturkservice.expire_hit(pick_one.participant.hit_id)
except MTurkServiceException as ex:
logger.exception(ex)
def rejects_questionnaire_from(self, participant):
"""Mechanical Turk participants submit their HITs on the MTurk site
(see external_submission_url), and MTurk then sends a notification
to Dallinger which is used to mark the assignment completed.
If a HIT has already been submitted, it's too late to submit the
questionnaire.
"""
if participant.status != "working":
return ("This participant has already sumbitted their HIT "
"on MTurk and can no longer submit the questionnaire")
def submitted_event(self):
"""MTurk will send its own notification when the worker
completes the HIT on that service.
"""
return None
def reward_bonus(self, assignment_id, amount, reason):
"""Reward the Turker for a specified assignment with a bonus."""
try:
return self.mturkservice.grant_bonus(assignment_id, amount, reason)
except MTurkServiceException as ex:
logger.exception(str(ex))
@property
def is_in_progress(self):
"""Does an MTurk HIT for the current experiment ID already exist?"""
experiment_id = self.config.get("id")
hits = self.mturkservice.get_hits(
hit_filter=lambda h: h["annotation"] == experiment_id)
for _ in hits:
return True
return False
@property
def qualification_active(self):
return bool(self.config.get("assign_qualifications"))
def current_hit_id(self):
"""Return the ID of the most recent HIT with our experiment ID
in the annotation, if any such HITs exist.
"""
experiment_id = self.config.get("id")
hits = list(
self.mturkservice.get_hits(
hit_filter=lambda h: h["annotation"] == experiment_id))
if not hits:
return None
if len(hits) == 1:
return hits[0]["id"]
# This is unlikely, but we might have more than one HIT if one was created
# directly via the MTurk UI.
hit_ids = [h["id"] for h in sorted(hits, key=lambda k: k["created"])]
most_recent = hit_ids[-1]
logger.warn(
"More than one HIT found annotated with experiment ID {}: ({}). "
"Using {}, as it is the most recently created.".format(
experiment_id, ", ".join(hit_ids), most_recent))
return most_recent
def approve_hit(self, assignment_id):
try:
return self.mturkservice.approve_assignment(assignment_id)
except MTurkServiceException as ex:
logger.exception(str(ex))
def close_recruitment(self):
"""Clean up once the experiment is complete.
This may be called before all users have finished so uses the
expire_hit rather than the disable_hit API call. This allows people
who have already picked up the hit to complete it as normal.
"""
logger.info(CLOSE_RECRUITMENT_LOG_PREFIX + " mturk")
# We are not expiring the hit currently as notifications are failing
# TODO: Reinstate this
# try:
# return self.mturkservice.expire_hit(
# self.current_hit_id(),
# )
# except MTurkServiceException as ex:
# logger.exception(str(ex))
@property
def is_sandbox(self):
return self.config.get("mode") == "sandbox"
def _build_hit_qualifications(self):
quals = []
reqs = MTurkQualificationRequirements
if self.config.get("approve_requirement") is not None:
quals.append(
reqs.min_approval(self.config.get("approve_requirement")))
if self.config.get("us_only"):
quals.append(reqs.restrict_to_countries(["US"]))
for item in self._config_to_list("mturk_qualification_blocklist"):
qtype = self.mturkservice.get_qualification_type_by_name(item)
if qtype:
quals.append(reqs.must_not_have(qtype["id"]))
if self.config.get("mturk_qualification_requirements",
None) is not None:
explicit_qualifications = json.loads(
self.config.get("mturk_qualification_requirements"))
quals.extend(explicit_qualifications)
return quals
def _confirm_sns_subscription(self, token, topic):
self.mturkservice.confirm_subscription(token=token, topic=topic)
def _report_event_notification(self, events):
q = _get_queue()
for event in events:
event_type = event.get("EventType")
assignment_id = event.get("AssignmentId")
participant_id = None
q.enqueue(worker_function, event_type, assignment_id,
participant_id)
def _mturk_status_for(self, participant):
try:
assignment = self.mturkservice.get_assignment(
participant.assignment_id)
status = assignment["status"]
except Exception:
status = None
return status
def _disable_autorecruit(self):
heroku_app = heroku_tools.HerokuApp(
self.config.get("heroku_app_id_root"))
args = json.dumps({"auto_recruit": "false"})
headers = heroku_tools.request_headers(
self.config.get("heroku_auth_token"))
requests.patch(heroku_app.config_url, data=args, headers=headers)
def _resend_submitted_rest_notification_for(self, participant):
q = _get_queue()
q.enqueue(worker_function, "AssignmentSubmitted",
participant.assignment_id, None)
def _send_notification_missing_rest_notification_for(self, participant):
q = _get_queue()
q.enqueue(worker_function, "NotificationMissing",
participant.assignment_id, None)
def _config_to_list(self, key):
# At some point we'll support lists, so all service code supports them,
# but the config system only supports strings for now, so we convert:
as_string = self.config.get(key, None)
if as_string is None:
return []
return [item.strip() for item in as_string.split(",") if item.strip()]
def _create_mturk_qualifications(self):
"""Create MTurk Qualification for experiment ID, and for group_name
if it's been set. Qualifications with these names already exist, but
it's faster to try and fail than to check, then try.
"""
for name, desc in self.qualifications.items():
try:
self.mturkservice.create_qualification_type(name, desc)
except DuplicateQualificationNameError:
pass
def _resubmitted_msg(self, summary):
templates = MTurkHITMessages.by_flavor(summary,
self.config.get("whimsical"))
return templates.resubmitted_msg()
def _cancelled_msg(self, summary):
templates = MTurkHITMessages.by_flavor(summary,
self.config.get("whimsical"))
return templates.hit_cancelled_msg()
def _message_researcher(self, message):
try:
self.notifies_admin.send(message["subject"], message["body"])
except MessengerError as ex:
logger.exception(ex)
class MTurkRecruiter(Recruiter):
"""Recruit participants from Amazon Mechanical Turk"""
experiment_qualification_desc = 'Experiment-specific qualification'
group_qualification_desc = 'Experiment group qualification'
def __init__(self):
super(MTurkRecruiter, self).__init__()
self.config = get_config()
self.ad_url = '{}/ad'.format(get_base_url())
self.hit_domain = os.getenv('HOST')
self.mturkservice = MTurkService(
self.config.get('aws_access_key_id'),
self.config.get('aws_secret_access_key'),
self.config.get('mode') != u"live"
)
self._validate_conifg()
def _validate_conifg(self):
mode = self.config.get('mode')
if mode not in (u'sandbox', u'live'):
raise MTurkRecruiterException(
'"{}" is not a valid mode for MTurk recruitment. '
'The value of "mode" must be either "sandbox" or "live"'.format(mode)
)
@property
def external_submission_url(self):
"""On experiment completion, participants are returned to
the Mechanical Turk site to submit their HIT, which in turn triggers
notifications to the /notifications route.
"""
if self.config.get('mode') == "sandbox":
return "https://workersandbox.mturk.com/mturk/externalSubmit"
return "https://www.mturk.com/mturk/externalSubmit"
@property
def qualifications(self):
quals = {self.config.get('id'): self.experiment_qualification_desc}
group_name = self.config.get('group_name', None)
if group_name:
quals[group_name] = self.group_qualification_desc
return quals
def open_recruitment(self, n=1):
"""Open a connection to AWS MTurk and create a HIT."""
if self.is_in_progress:
# Already started... do nothing.
return None
if self.hit_domain is None:
raise MTurkRecruiterException("Can't run a HIT from localhost")
self.mturkservice.check_credentials()
self._create_mturk_qualifications()
hit_request = {
'max_assignments': n,
'title': self.config.get('title'),
'description': self.config.get('description'),
'keywords': self._config_to_list('keywords'),
'reward': self.config.get('base_payment'),
'duration_hours': self.config.get('duration'),
'lifetime_days': self.config.get('lifetime'),
'ad_url': self.ad_url,
'notification_url': self.config.get('notification_url'),
'approve_requirement': self.config.get('approve_requirement'),
'us_only': self.config.get('us_only'),
'blacklist': self._config_to_list('qualification_blacklist'),
}
hit_info = self.mturkservice.create_hit(**hit_request)
if self.config.get('mode') == "sandbox":
lookup_url = "https://workersandbox.mturk.com/mturk/preview?groupId={type_id}"
else:
lookup_url = "https://worker.mturk.com/mturk/preview?groupId={type_id}"
return {
'items': [lookup_url.format(**hit_info), ],
'message': 'HIT now published to Amazon Mechanical Turk'
}
def recruit(self, n=1):
"""Recruit n new participants to an existing HIT"""
if not self.config.get('auto_recruit', False):
logger.info('auto_recruit is False: recruitment suppressed')
return
hit_id = self.current_hit_id()
if hit_id is None:
logger.info('no HIT in progress: recruitment aborted')
return
try:
return self.mturkservice.extend_hit(
hit_id,
number=n,
duration_hours=self.config.get('duration')
)
except MTurkServiceException as ex:
logger.exception(ex.message)
def notify_recruited(self, participant):
pass
def notify_using(self, participant):
"""Assign a Qualification to the Participant for the experiment ID,
and for the configured group_name, if it's been set.
"""
worker_id = participant.worker_id
for name in self.qualifications:
try:
self.mturkservice.increment_qualification_score(
name, worker_id
)
except QualificationNotFoundException, ex:
logger.exception(ex)
class MTurkRecruiter(Recruiter):
"""Recruit participants from Amazon Mechanical Turk"""
nickname = "mturk"
experiment_qualification_desc = "Experiment-specific qualification"
group_qualification_desc = "Experiment group qualification"
def __init__(self):
super(MTurkRecruiter, self).__init__()
self.config = get_config()
self.ad_url = "{}/ad?recruiter={}".format(get_base_url(),
self.nickname)
self.hit_domain = os.getenv("HOST")
self.mturkservice = MTurkService(
self.config.get("aws_access_key_id"),
self.config.get("aws_secret_access_key"),
self.config.get("aws_region"),
self.config.get("mode") != "live",
)
self.messenger = get_messenger(self.config)
self._validate_config()
def _validate_config(self):
mode = self.config.get("mode")
if mode not in ("sandbox", "live"):
raise MTurkRecruiterException(
'"{}" is not a valid mode for MTurk recruitment. '
'The value of "mode" must be either "sandbox" or "live"'.
format(mode))
@property
def external_submission_url(self):
"""On experiment completion, participants are returned to
the Mechanical Turk site to submit their HIT, which in turn triggers
notifications to the /notifications route.
"""
if self.config.get("mode") == "sandbox":
return "https://workersandbox.mturk.com/mturk/externalSubmit"
return "https://www.mturk.com/mturk/externalSubmit"
@property
def qualifications(self):
quals = {self.config.get("id"): self.experiment_qualification_desc}
group_name = self.config.get("group_name", None)
if group_name:
quals[group_name] = self.group_qualification_desc
return quals
def open_recruitment(self, n=1):
"""Open a connection to AWS MTurk and create a HIT."""
logger.info("Opening MTurk recruitment for {} participants".format(n))
if self.is_in_progress:
raise MTurkRecruiterException(
"Tried to open_recruitment on already open recruiter.")
if self.hit_domain is None:
raise MTurkRecruiterException("Can't run a HIT from localhost")
self.mturkservice.check_credentials()
if self.config.get("assign_qualifications"):
self._create_mturk_qualifications()
hit_request = {
"max_assignments": n,
"title": self.config.get("title"),
"description": self.config.get("description"),
"keywords": self._config_to_list("keywords"),
"reward": self.config.get("base_payment"),
"duration_hours": self.config.get("duration"),
"lifetime_days": self.config.get("lifetime"),
"ad_url": self.ad_url,
"notification_url": self.config.get("notification_url"),
"approve_requirement": self.config.get("approve_requirement"),
"us_only": self.config.get("us_only"),
"blacklist": self._config_to_list("qualification_blacklist"),
"annotation": self.config.get("id"),
}
hit_info = self.mturkservice.create_hit(**hit_request)
if self.config.get("mode") == "sandbox":
lookup_url = (
"https://workersandbox.mturk.com/mturk/preview?groupId={type_id}"
)
else:
lookup_url = "https://worker.mturk.com/mturk/preview?groupId={type_id}"
return {
"items": [lookup_url.format(**hit_info)],
"message": "HIT now published to Amazon Mechanical Turk",
}
def recruit(self, n=1):
"""Recruit n new participants to an existing HIT"""
logger.info("Recruiting {} MTurk participants".format(n))
if not self.config.get("auto_recruit"):
logger.info("auto_recruit is False: recruitment suppressed")
return
hit_id = self.current_hit_id()
if hit_id is None:
logger.info("no HIT in progress: recruitment aborted")
return
try:
return self.mturkservice.extend_hit(
hit_id, number=n, duration_hours=self.config.get("duration"))
except MTurkServiceException as ex:
logger.exception(str(ex))
def notify_completed(self, participant):
"""Assign a Qualification to the Participant for the experiment ID,
and for the configured group_name, if it's been set.
Overrecruited participants don't receive qualifications, since they
haven't actually completed the experiment. This allows them to remain
eligible for future runs.
"""
if participant.status == "overrecruited" or not self.qualification_active:
return
worker_id = participant.worker_id
for name in self.qualifications:
try:
self.mturkservice.increment_qualification_score(
name, worker_id)
except QualificationNotFoundException as ex:
logger.exception(ex)
def notify_duration_exceeded(self, participants, reference_time):
"""The participant has exceed the maximum time for the activity,
defined in the "duration" config value. We need find out the assignment
status on MTurk and act based on this.
"""
unsubmitted = []
for participant in participants:
summary = ParticipationTime(participant, reference_time,
self.config)
status = self._mturk_status_for(participant)
if status == "Approved":
participant.status = "approved"
session.commit()
elif status == "Rejected":
participant.status = "rejected"
session.commit()
elif status == "Submitted":
self._resend_submitted_rest_notification_for(participant)
self._message_researcher(self._resubmitted_msg(summary))
logger.warning(
"Error - submitted notification for participant {} missed. "
"A replacement notification was created and sent, "
"but proceed with caution.".format(participant.id))
else:
self._send_notification_missing_rest_notification_for(
participant)
unsubmitted.append(summary)
if unsubmitted:
self._disable_autorecruit()
self.close_recruitment()
pick_one = unsubmitted[0]
# message the researcher about the one of the participants:
self._message_researcher(self._cancelled_msg(pick_one))
# Attempt to force-expire the hit via boto. It's possible
# that the HIT won't exist if the HIT has been deleted manually.
try:
self.mturkservice.expire_hit(pick_one.participant.hit_id)
except MTurkServiceException as ex:
logger.exception(ex)
def rejects_questionnaire_from(self, participant):
"""Mechanical Turk participants submit their HITs on the MTurk site
(see external_submission_url), and MTurk then sends a notification
to Dallinger which is used to mark the assignment completed.
If a HIT has already been submitted, it's too late to submit the
questionnaire.
"""
if participant.status != "working":
return ("This participant has already sumbitted their HIT "
"on MTurk and can no longer submit the questionnaire")
def submitted_event(self):
"""MTurk will send its own notification when the worker
completes the HIT on that service.
"""
return None
def reward_bonus(self, assignment_id, amount, reason):
"""Reward the Turker for a specified assignment with a bonus."""
try:
return self.mturkservice.grant_bonus(assignment_id, amount, reason)
except MTurkServiceException as ex:
logger.exception(str(ex))
@property
def is_in_progress(self):
# Has this recruiter resulted in any participants?
return bool(
Participant.query.filter_by(recruiter_id=self.nickname).first())
@property
def qualification_active(self):
return bool(self.config.get("assign_qualifications"))
def current_hit_id(self):
any_participant_record = (Participant.query.with_entities(
Participant.hit_id).filter_by(recruiter_id=self.nickname).first())
if any_participant_record is not None:
return str(any_participant_record.hit_id)
def approve_hit(self, assignment_id):
try:
return self.mturkservice.approve_assignment(assignment_id)
except MTurkServiceException as ex:
logger.exception(str(ex))
def close_recruitment(self):
"""Clean up once the experiment is complete.
This may be called before all users have finished so uses the
expire_hit rather than the disable_hit API call. This allows people
who have already picked up the hit to complete it as normal.
"""
logger.info(CLOSE_RECRUITMENT_LOG_PREFIX + " mturk")
# We are not expiring the hit currently as notifications are failing
# TODO: Reinstate this
# try:
# return self.mturkservice.expire_hit(
# self.current_hit_id(),
# )
# except MTurkServiceException as ex:
# logger.exception(str(ex))
def _mturk_status_for(self, participant):
try:
assignment = self.mturkservice.get_assignment(
participant.assignment_id)
status = assignment["status"]
except Exception:
status = None
return status
def _disable_autorecruit(self):
heroku_app = heroku_tools.HerokuApp(self.config.get("id"))
args = json.dumps({"auto_recruit": "false"})
headers = heroku_tools.request_headers(
self.config.get("heroku_auth_token"))
requests.patch(heroku_app.config_url, data=args, headers=headers)
def _resend_submitted_rest_notification_for(self, participant):
notification_url = self.config.get("notification_url")
args = {
"Event.1.EventType": "AssignmentSubmitted",
"Event.1.AssignmentId": participant.assignment_id,
}
requests.post(notification_url, data=args)
def _send_notification_missing_rest_notification_for(self, participant):
notification_url = self.config.get("notification_url")
args = {
"Event.1.EventType": "NotificationMissing",
"Event.1.AssignmentId": participant.assignment_id,
}
requests.post(notification_url, data=args)
def _config_to_list(self, key):
# At some point we'll support lists, so all service code supports them,
# but the config system only supports strings for now, so we convert:
as_string = self.config.get(key, None)
if as_string is None:
return []
return [item.strip() for item in as_string.split(",") if item.strip()]
def _create_mturk_qualifications(self):
"""Create MTurk Qualification for experiment ID, and for group_name
if it's been set. Qualifications with these names already exist, but
it's faster to try and fail than to check, then try.
"""
for name, desc in self.qualifications.items():
try:
self.mturkservice.create_qualification_type(name, desc)
except DuplicateQualificationNameError:
pass
def _resubmitted_msg(self, summary):
templates = MTurkHITMessages.by_flavor(summary,
self.config.get("whimsical"))
return templates.resubmitted_msg()
def _cancelled_msg(self, summary):
templates = MTurkHITMessages.by_flavor(summary,
self.config.get("whimsical"))
return templates.hit_cancelled_msg()
def _message_researcher(self, message):
try:
self.messenger.send(message)
except MessengerError as ex:
logger.exception(ex)