class MTurkRecruiter(object): """Recruit participants from Amazon Mechanical Turk""" @classmethod def from_current_config(cls): config = get_config() if not config.ready: config.load_config() ad_url = '{}/ad'.format(get_base_url()) hit_domain = os.getenv('HOST') return cls(config, hit_domain, ad_url) def __init__(self, config, hit_domain, ad_url): self.config = config self.ad_url = ad_url self.hit_domain = hit_domain self.mturkservice = MTurkService( self.config.get('aws_access_key_id'), self.config.get('aws_secret_access_key'), (self.config.get('mode') == "sandbox")) def open_recruitment(self, n=1): """Open a connection to AWS MTurk and create a HIT.""" if self.is_in_progress: # Already started... do nothing. return None if self.config.get('mode') == 'debug': raise MTurkRecruiterException("Can't run a HIT in debug mode") if self.hit_domain is None: raise MTurkRecruiterException("Can't run a HIT from localhost") self.mturkservice.check_credentials() hit_request = { 'max_assignments': n, 'title': self.config.get('title'), 'description': self.config.get('description'), 'keywords': self.config.get('amt_keywords'), 'reward': self.config.get('base_payment'), 'duration_hours': self.config.get('duration'), 'lifetime_days': self.config.get('lifetime'), 'ad_url': self.ad_url, 'notification_url': self.config.get('notification_url'), 'approve_requirement': self.config.get('approve_requirement'), 'us_only': self.config.get('us_only'), } hit_info = self.mturkservice.create_hit(**hit_request) if self.config.get('mode') == "sandbox": lookup_url = "https://workersandbox.mturk.com/mturk/preview?groupId={type_id}" else: lookup_url = "https://worker.mturk.com/mturk/preview?groupId={type_id}" return lookup_url.format(**hit_info) def recruit_participants(self, n=1): """Recruit n new participants to an existing HIT""" if not self.config.get('auto_recruit', False): logger.info('auto_recruit is False: recruitment suppressed') return hit_id = self.current_hit_id() if hit_id is None: logger.info('no HIT in progress: recruitment aborted') return return self.mturkservice.extend_hit( hit_id, number=n, duration_hours=self.config.get('duration')) def reward_bonus(self, assignment_id, amount, reason): """Reward the Turker for a specified assignment with a bonus.""" return self.mturkservice.grant_bonus(assignment_id, amount, reason) @property def is_in_progress(self): return bool(Participant.query.first()) def current_hit_id(self): any_participant_record = Participant.query.with_entities( Participant.hit_id).first() if any_participant_record is not None: return str(any_participant_record.hit_id) def approve_hit(self, assignment_id): return self.mturkservice.approve_assignment(assignment_id) def close_recruitment(self): """Clean up once the experiment is complete. This does nothing, because the fact that this is called means that all MTurk HITs that were created were already completed. """ pass
class MTurkRecruiter(Recruiter): """Recruit participants from Amazon Mechanical Turk""" nickname = "mturk" extra_routes = mturk_routes def __init__(self, *args, **kwargs): super(MTurkRecruiter, self).__init__() self.config = get_config() base_url = get_base_url() self.ad_url = "{}/ad?recruiter={}".format(base_url, self.nickname) self.notification_url = "{}/mturk-sns-listener".format(base_url) self.hit_domain = os.getenv("HOST") self.mturkservice = MTurkService( aws_access_key_id=self.config.get("aws_access_key_id"), aws_secret_access_key=self.config.get("aws_secret_access_key"), region_name=self.config.get("aws_region"), sandbox=self.config.get("mode") != "live", ) self.notifies_admin = admin_notifier(self.config) self.mailer = get_mailer(self.config) self.store = kwargs.get("store") or RedisStore() self._validate_config() def _validate_config(self): mode = self.config.get("mode") if mode not in ("sandbox", "live"): raise MTurkRecruiterException( '"{}" is not a valid mode for MTurk recruitment. ' 'The value of "mode" must be either "sandbox" or "live"'. format(mode)) def exit_response(self, experiment, participant): return flask.render_template( "exit_recruiter_mturk.html", hitid=participant.hit_id, assignmentid=participant.assignment_id, workerid=participant.worker_id, external_submit_url=self.external_submission_url, ) @property def external_submission_url(self): """On experiment completion, participants are returned to the Mechanical Turk site to submit their HIT, which in turn triggers notifications to the /mturk-sns-listener route. """ if self.is_sandbox: return "https://workersandbox.mturk.com/mturk/externalSubmit" return "https://www.mturk.com/mturk/externalSubmit" def open_recruitment(self, n=1): """Open a connection to AWS MTurk and create a HIT.""" logger.info("Opening MTurk recruitment for {} participants".format(n)) if self.is_in_progress: raise MTurkRecruiterException( "Tried to open_recruitment on already open recruiter.") if self.hit_domain is None: raise MTurkRecruiterException("Can't run a HIT from localhost") self.mturkservice.check_credentials() hit_request = { "experiment_id": self.config.get("id"), "max_assignments": n, "title": "{} ({})".format(self.config.get("title"), heroku_tools.app_name(self.config.get("id"))), "description": self.config.get("description"), "keywords": self._config_to_list("keywords"), "reward": self.config.get("base_payment"), "duration_hours": self.config.get("duration"), "lifetime_days": self.config.get("lifetime"), "question": MTurkQuestions.external(self.ad_url), "notification_url": self.notification_url, "annotation": self.config.get("id"), "qualifications": self._build_required_hit_qualifications(), } hit_info = self.mturkservice.create_hit(**hit_request) self._record_current_hit_id(hit_info["id"]) url = hit_info["worker_url"] return { "items": [url], "message": "HIT now published to Amazon Mechanical Turk", } def assign_experiment_qualifications(self, worker_id, qualifications): """Assigns MTurk Qualifications to a worker. @param worker_id string the MTurk worker ID @param qualifications list of dict w/ `name`, `description` and (optional) `score` keys """ by_name = {qual["name"]: qual for qual in qualifications} result = self._ensure_mturk_qualifications(qualifications) for qual in result["new_qualifications"]: score = by_name[qual["name"]].get("score") if score is not None: self.mturkservice.assign_qualification(qual["id"], worker_id, qual["score"]) else: self.mturkservice.increment_qualification_score( qual["id"], worker_id) for name in result["existing_qualifications"]: score = by_name[name].get("score") if score is not None: self.mturkservice.assign_named_qualification( name, worker_id, score) else: self.mturkservice.increment_named_qualification_score( name, worker_id) def compensate_worker(self, worker_id, email, dollars, notify=True): """Pay a worker by means of a special HIT that only they can see.""" qualification = self.mturkservice.create_qualification_type( name="Dallinger Compensation Qualification - {}".format( generate_random_id()), description=( "You have received a qualification to allow you to complete a " "compensation HIT from Dallinger for ${}.".format(dollars)), ) qid = qualification["id"] self.mturkservice.assign_qualification(qid, worker_id, 1, notify=notify) hit_request = { "experiment_id": "(compensation only)", "max_assignments": 1, "title": "Dallinger Compensation HIT", "description": "For compenation only; no task required.", "keywords": [], "reward": float(dollars), "duration_hours": 1, "lifetime_days": 3, "question": MTurkQuestions.compensation(sandbox=self.is_sandbox), "qualifications": [MTurkQualificationRequirements.must_have(qid)], "do_subscribe": False, } hit_info = self.mturkservice.create_hit(**hit_request) if email is not None: message = { "subject": "Dallinger Compensation HIT", "sender": self.config.get("dallinger_email_address"), "recipients": [email], "body": ("A special compensation HIT is available for you to complete on MTurk.\n\n" "Title: {title}\n" "Reward: ${reward:.2f}\n" "URL: {worker_url}").format(**hit_info), } self.mailer.send(**message) else: message = {} return { "hit": hit_info, "qualification": qualification, "email": message } def recruit(self, n=1): """Recruit n new participants to an existing HIT""" logger.info("Recruiting {} MTurk participants".format(n)) if not self.config.get("auto_recruit"): logger.info("auto_recruit is False: recruitment suppressed") return hit_id = self.current_hit_id() if hit_id is None: logger.info("no HIT in progress: recruitment aborted") return try: return self.mturkservice.extend_hit( hit_id, number=n, duration_hours=self.config.get("duration")) except MTurkServiceException as ex: logger.exception(str(ex)) def notify_duration_exceeded(self, participants, reference_time): """The participant has exceed the maximum time for the activity, defined in the "duration" config value. We need find out the assignment status on MTurk and act based on this. """ unsubmitted = [] for participant in participants: summary = ParticipationTime(participant, reference_time, self.config) status = self._mturk_status_for(participant) if status == "Approved": participant.status = "approved" session.commit() elif status == "Rejected": participant.status = "rejected" session.commit() elif status == "Submitted": self._resend_submitted_rest_notification_for(participant) self._message_researcher(self._resubmitted_msg(summary)) logger.warning( "Error - submitted notification for participant {} missed. " "A replacement notification was created and sent, " "but proceed with caution.".format(participant.id)) else: self._send_notification_missing_rest_notification_for( participant) unsubmitted.append(summary) disable_hit = self.config.get("disable_when_duration_exceeded") if disable_hit and unsubmitted: self._disable_autorecruit() self.close_recruitment() pick_one = unsubmitted[0] # message the researcher about the one of the participants: self._message_researcher(self._cancelled_msg(pick_one)) # Attempt to force-expire the hit via boto. It's possible # that the HIT won't exist if the HIT has been deleted manually. try: self.mturkservice.expire_hit(pick_one.participant.hit_id) except MTurkServiceException as ex: logger.exception(ex) def rejects_questionnaire_from(self, participant): """Mechanical Turk participants submit their HITs on the MTurk site (see external_submission_url), and MTurk then sends a notification to Dallinger which is used to mark the assignment completed. If a HIT has already been submitted, it's too late to submit the questionnaire. """ if participant.status != "working": return ("This participant has already sumbitted their HIT " "on MTurk and can no longer submit the questionnaire") def submitted_event(self): """MTurk will send its own notification when the worker completes the HIT on that service. """ return None def reward_bonus(self, assignment_id, amount, reason): """Reward the Turker for a specified assignment with a bonus.""" try: return self.mturkservice.grant_bonus(assignment_id, amount, reason) except MTurkServiceException as ex: logger.exception(str(ex)) @property def is_in_progress(self): """Does an MTurk HIT for the current experiment ID already exist?""" return self.current_hit_id() is not None def current_hit_id(self): """Return the ID of the HIT associated with the active experiment ID if any such HIT exists. """ return self.store.get(self.hit_id_storage_key) def approve_hit(self, assignment_id): try: return self.mturkservice.approve_assignment(assignment_id) except MTurkServiceException as ex: logger.exception(str(ex)) def close_recruitment(self): """Clean up once the experiment is complete. This may be called before all users have finished so uses the expire_hit rather than the disable_hit API call. This allows people who have already picked up the hit to complete it as normal. """ logger.info(CLOSE_RECRUITMENT_LOG_PREFIX + " mturk") # We are not expiring the hit currently as notifications are failing # TODO: Reinstate this # try: # return self.mturkservice.expire_hit( # self.current_hit_id(), # ) # except MTurkServiceException as ex: # logger.exception(str(ex)) @property def is_sandbox(self): return self.config.get("mode") == "sandbox" @property def hit_id_storage_key(self): experiment_id = self.config.get("id") return "{}:{}".format(self.__class__.__name__, experiment_id) def _build_required_hit_qualifications(self): # The Qualications an MTurk worker must have, or in the case of the # blocklist, not have, in order for them to see and accept the HIT. quals = [] reqs = MTurkQualificationRequirements if self.config.get("approve_requirement") is not None: quals.append( reqs.min_approval(self.config.get("approve_requirement"))) if self.config.get("us_only"): quals.append(reqs.restrict_to_countries(["US"])) for item in self._config_to_list("mturk_qualification_blocklist"): qtype = self.mturkservice.get_qualification_type_by_name(item) if qtype: quals.append(reqs.must_not_have(qtype["id"])) if self.config.get("mturk_qualification_requirements", None) is not None: explicit_qualifications = json.loads( self.config.get("mturk_qualification_requirements")) quals.extend(explicit_qualifications) return quals def _record_current_hit_id(self, hit_id): self.store.set(self.hit_id_storage_key, hit_id) def _confirm_sns_subscription(self, token, topic): self.mturkservice.confirm_subscription(token=token, topic=topic) def _report_event_notification(self, events): q = _get_queue() for event in events: event_type = event.get("EventType") assignment_id = event.get("AssignmentId") participant_id = None q.enqueue(worker_function, event_type, assignment_id, participant_id) def _mturk_status_for(self, participant): try: assignment = self.mturkservice.get_assignment( participant.assignment_id) status = assignment["status"] except Exception: status = None return status def _disable_autorecruit(self): heroku_app = heroku_tools.HerokuApp( self.config.get("heroku_app_id_root")) args = json.dumps({"auto_recruit": "false"}) headers = heroku_tools.request_headers( self.config.get("heroku_auth_token")) requests.patch(heroku_app.config_url, data=args, headers=headers) def _resend_submitted_rest_notification_for(self, participant): q = _get_queue() q.enqueue(worker_function, "AssignmentSubmitted", participant.assignment_id, None) def _send_notification_missing_rest_notification_for(self, participant): q = _get_queue() q.enqueue(worker_function, "NotificationMissing", participant.assignment_id, None) def _config_to_list(self, key): # At some point we'll support lists, so all service code supports them, # but the config system only supports strings for now, so we convert: as_string = self.config.get(key, None) if as_string is None: return [] return [item.strip() for item in as_string.split(",") if item.strip()] def _ensure_mturk_qualifications(self, qualifications): """Create MTurk Qualifications for names that don't already exist, but also return names that already do. """ result = {"new_qualifications": [], "existing_qualifications": []} for qual in qualifications: name = qual["name"] desc = qual["description"] try: result["new_qualifications"].append({ "name": name, "id": self.mturkservice.create_qualification_type(name, desc)["id"], "available": False, }) except DuplicateQualificationNameError: result["existing_qualifications"].append(name) # We need to make sure the new qualifications are actually ready # for assignment, as there's a small delay. for tries in range(5): for new in result["new_qualifications"]: if new["available"]: continue try: self.mturkservice.get_qualification_type_by_name( new["name"]) except QualificationNotFoundException: logger.warn( "Did not find qualification {}. Trying again...". format(new["name"])) time.sleep(1) else: new["available"] = True if all([n["available"] for n in result["new_qualifications"]]): break unavailable = [ q for q in result["new_qualifications"] if not q["available"] ] if unavailable: logger.warn( "After several attempts, some qualifications are still not ready " "for assignment: {}".format(", ".join(unavailable))) # Return just the available among the new ones result["new_qualifications"] = [ q for q in result["new_qualifications"] if q["available"] ] return result def _resubmitted_msg(self, summary): templates = MTurkHITMessages.by_flavor(summary, self.config.get("whimsical")) return templates.resubmitted_msg() def _cancelled_msg(self, summary): templates = MTurkHITMessages.by_flavor(summary, self.config.get("whimsical")) return templates.hit_cancelled_msg() def _message_researcher(self, message): try: self.notifies_admin.send(message["subject"], message["body"]) except MessengerError as ex: logger.exception(ex)
class MTurkRecruiter(Recruiter): """Recruit participants from Amazon Mechanical Turk""" nickname = "mturk" extra_routes = mturk_routes experiment_qualification_desc = "Experiment-specific qualification" group_qualification_desc = "Experiment group qualification" def __init__(self): super(MTurkRecruiter, self).__init__() self.config = get_config() base_url = get_base_url() self.ad_url = "{}/ad?recruiter={}".format(base_url, self.nickname) self.notification_url = "{}/mturk-sns-listener".format(base_url) self.hit_domain = os.getenv("HOST") self.mturkservice = MTurkService( aws_access_key_id=self.config.get("aws_access_key_id"), aws_secret_access_key=self.config.get("aws_secret_access_key"), region_name=self.config.get("aws_region"), sandbox=self.config.get("mode") != "live", ) self.notifies_admin = admin_notifier(self.config) self.mailer = get_mailer(self.config) self._validate_config() def _validate_config(self): mode = self.config.get("mode") if mode not in ("sandbox", "live"): raise MTurkRecruiterException( '"{}" is not a valid mode for MTurk recruitment. ' 'The value of "mode" must be either "sandbox" or "live"'. format(mode)) @property def external_submission_url(self): """On experiment completion, participants are returned to the Mechanical Turk site to submit their HIT, which in turn triggers notifications to the /notifications route. """ if self.is_sandbox: return "https://workersandbox.mturk.com/mturk/externalSubmit" return "https://www.mturk.com/mturk/externalSubmit" @property def qualifications(self): quals = {self.config.get("id"): self.experiment_qualification_desc} group_name = self.config.get("group_name", None) if group_name: quals[group_name] = self.group_qualification_desc return quals def open_recruitment(self, n=1): """Open a connection to AWS MTurk and create a HIT.""" logger.info("Opening MTurk recruitment for {} participants".format(n)) if self.is_in_progress: raise MTurkRecruiterException( "Tried to open_recruitment on already open recruiter.") if self.hit_domain is None: raise MTurkRecruiterException("Can't run a HIT from localhost") self.mturkservice.check_credentials() if self.config.get("assign_qualifications"): self._create_mturk_qualifications() hit_request = { "experiment_id": self.config.get("id"), "max_assignments": n, "title": self.config.get("title"), "description": self.config.get("description"), "keywords": self._config_to_list("keywords"), "reward": self.config.get("base_payment"), "duration_hours": self.config.get("duration"), "lifetime_days": self.config.get("lifetime"), "question": MTurkQuestions.external(self.ad_url), "notification_url": self.notification_url, "annotation": self.config.get("id"), "qualifications": self._build_hit_qualifications(), } hit_info = self.mturkservice.create_hit(**hit_request) url = hit_info["worker_url"] return { "items": [url], "message": "HIT now published to Amazon Mechanical Turk", } def compensate_worker(self, worker_id, email, dollars, notify=True): """Pay a worker by means of a special HIT that only they can see. """ qualification = self.mturkservice.create_qualification_type( name="Dallinger Compensation Qualification - {}".format( generate_random_id()), description=( "You have received a qualification to allow you to complete a " "compensation HIT from Dallinger for ${}.".format(dollars)), ) qid = qualification["id"] self.mturkservice.assign_qualification(qid, worker_id, 1, notify=notify) hit_request = { "experiment_id": "(compensation only)", "max_assignments": 1, "title": "Dallinger Compensation HIT", "description": "For compenation only; no task required.", "keywords": [], "reward": float(dollars), "duration_hours": 1, "lifetime_days": 3, "question": MTurkQuestions.compensation(sandbox=self.is_sandbox), "qualifications": [MTurkQualificationRequirements.must_have(qid)], "do_subscribe": False, } hit_info = self.mturkservice.create_hit(**hit_request) if email is not None: message = { "subject": "Dallinger Compensation HIT", "sender": self.config.get("dallinger_email_address"), "recipients": [email], "body": ("A special compenstation HIT is available for you to complete on MTurk.\n\n" "Title: {title}\n" "Reward: ${reward:.2f}\n" "URL: {worker_url}").format(**hit_info), } self.mailer.send(**message) else: message = {} return { "hit": hit_info, "qualification": qualification, "email": message } def recruit(self, n=1): """Recruit n new participants to an existing HIT""" logger.info("Recruiting {} MTurk participants".format(n)) if not self.config.get("auto_recruit"): logger.info("auto_recruit is False: recruitment suppressed") return hit_id = self.current_hit_id() if hit_id is None: logger.info("no HIT in progress: recruitment aborted") return try: return self.mturkservice.extend_hit( hit_id, number=n, duration_hours=self.config.get("duration")) except MTurkServiceException as ex: logger.exception(str(ex)) def notify_completed(self, participant): """Assign a Qualification to the Participant for the experiment ID, and for the configured group_name, if it's been set. Overrecruited participants don't receive qualifications, since they haven't actually completed the experiment. This allows them to remain eligible for future runs. """ if participant.status == "overrecruited" or not self.qualification_active: return worker_id = participant.worker_id for name in self.qualifications: try: self.mturkservice.increment_qualification_score( name, worker_id) except QualificationNotFoundException as ex: logger.exception(ex) def notify_duration_exceeded(self, participants, reference_time): """The participant has exceed the maximum time for the activity, defined in the "duration" config value. We need find out the assignment status on MTurk and act based on this. """ unsubmitted = [] for participant in participants: summary = ParticipationTime(participant, reference_time, self.config) status = self._mturk_status_for(participant) if status == "Approved": participant.status = "approved" session.commit() elif status == "Rejected": participant.status = "rejected" session.commit() elif status == "Submitted": self._resend_submitted_rest_notification_for(participant) self._message_researcher(self._resubmitted_msg(summary)) logger.warning( "Error - submitted notification for participant {} missed. " "A replacement notification was created and sent, " "but proceed with caution.".format(participant.id)) else: self._send_notification_missing_rest_notification_for( participant) unsubmitted.append(summary) if unsubmitted: self._disable_autorecruit() self.close_recruitment() pick_one = unsubmitted[0] # message the researcher about the one of the participants: self._message_researcher(self._cancelled_msg(pick_one)) # Attempt to force-expire the hit via boto. It's possible # that the HIT won't exist if the HIT has been deleted manually. try: self.mturkservice.expire_hit(pick_one.participant.hit_id) except MTurkServiceException as ex: logger.exception(ex) def rejects_questionnaire_from(self, participant): """Mechanical Turk participants submit their HITs on the MTurk site (see external_submission_url), and MTurk then sends a notification to Dallinger which is used to mark the assignment completed. If a HIT has already been submitted, it's too late to submit the questionnaire. """ if participant.status != "working": return ("This participant has already sumbitted their HIT " "on MTurk and can no longer submit the questionnaire") def submitted_event(self): """MTurk will send its own notification when the worker completes the HIT on that service. """ return None def reward_bonus(self, assignment_id, amount, reason): """Reward the Turker for a specified assignment with a bonus.""" try: return self.mturkservice.grant_bonus(assignment_id, amount, reason) except MTurkServiceException as ex: logger.exception(str(ex)) @property def is_in_progress(self): """Does an MTurk HIT for the current experiment ID already exist?""" experiment_id = self.config.get("id") hits = self.mturkservice.get_hits( hit_filter=lambda h: h["annotation"] == experiment_id) for _ in hits: return True return False @property def qualification_active(self): return bool(self.config.get("assign_qualifications")) def current_hit_id(self): """Return the ID of the most recent HIT with our experiment ID in the annotation, if any such HITs exist. """ experiment_id = self.config.get("id") hits = list( self.mturkservice.get_hits( hit_filter=lambda h: h["annotation"] == experiment_id)) if not hits: return None if len(hits) == 1: return hits[0]["id"] # This is unlikely, but we might have more than one HIT if one was created # directly via the MTurk UI. hit_ids = [h["id"] for h in sorted(hits, key=lambda k: k["created"])] most_recent = hit_ids[-1] logger.warn( "More than one HIT found annotated with experiment ID {}: ({}). " "Using {}, as it is the most recently created.".format( experiment_id, ", ".join(hit_ids), most_recent)) return most_recent def approve_hit(self, assignment_id): try: return self.mturkservice.approve_assignment(assignment_id) except MTurkServiceException as ex: logger.exception(str(ex)) def close_recruitment(self): """Clean up once the experiment is complete. This may be called before all users have finished so uses the expire_hit rather than the disable_hit API call. This allows people who have already picked up the hit to complete it as normal. """ logger.info(CLOSE_RECRUITMENT_LOG_PREFIX + " mturk") # We are not expiring the hit currently as notifications are failing # TODO: Reinstate this # try: # return self.mturkservice.expire_hit( # self.current_hit_id(), # ) # except MTurkServiceException as ex: # logger.exception(str(ex)) @property def is_sandbox(self): return self.config.get("mode") == "sandbox" def _build_hit_qualifications(self): quals = [] reqs = MTurkQualificationRequirements if self.config.get("approve_requirement") is not None: quals.append( reqs.min_approval(self.config.get("approve_requirement"))) if self.config.get("us_only"): quals.append(reqs.restrict_to_countries(["US"])) for item in self._config_to_list("mturk_qualification_blocklist"): qtype = self.mturkservice.get_qualification_type_by_name(item) if qtype: quals.append(reqs.must_not_have(qtype["id"])) if self.config.get("mturk_qualification_requirements", None) is not None: explicit_qualifications = json.loads( self.config.get("mturk_qualification_requirements")) quals.extend(explicit_qualifications) return quals def _confirm_sns_subscription(self, token, topic): self.mturkservice.confirm_subscription(token=token, topic=topic) def _report_event_notification(self, events): q = _get_queue() for event in events: event_type = event.get("EventType") assignment_id = event.get("AssignmentId") participant_id = None q.enqueue(worker_function, event_type, assignment_id, participant_id) def _mturk_status_for(self, participant): try: assignment = self.mturkservice.get_assignment( participant.assignment_id) status = assignment["status"] except Exception: status = None return status def _disable_autorecruit(self): heroku_app = heroku_tools.HerokuApp( self.config.get("heroku_app_id_root")) args = json.dumps({"auto_recruit": "false"}) headers = heroku_tools.request_headers( self.config.get("heroku_auth_token")) requests.patch(heroku_app.config_url, data=args, headers=headers) def _resend_submitted_rest_notification_for(self, participant): q = _get_queue() q.enqueue(worker_function, "AssignmentSubmitted", participant.assignment_id, None) def _send_notification_missing_rest_notification_for(self, participant): q = _get_queue() q.enqueue(worker_function, "NotificationMissing", participant.assignment_id, None) def _config_to_list(self, key): # At some point we'll support lists, so all service code supports them, # but the config system only supports strings for now, so we convert: as_string = self.config.get(key, None) if as_string is None: return [] return [item.strip() for item in as_string.split(",") if item.strip()] def _create_mturk_qualifications(self): """Create MTurk Qualification for experiment ID, and for group_name if it's been set. Qualifications with these names already exist, but it's faster to try and fail than to check, then try. """ for name, desc in self.qualifications.items(): try: self.mturkservice.create_qualification_type(name, desc) except DuplicateQualificationNameError: pass def _resubmitted_msg(self, summary): templates = MTurkHITMessages.by_flavor(summary, self.config.get("whimsical")) return templates.resubmitted_msg() def _cancelled_msg(self, summary): templates = MTurkHITMessages.by_flavor(summary, self.config.get("whimsical")) return templates.hit_cancelled_msg() def _message_researcher(self, message): try: self.notifies_admin.send(message["subject"], message["body"]) except MessengerError as ex: logger.exception(ex)
class MTurkRecruiter(Recruiter): """Recruit participants from Amazon Mechanical Turk""" nickname = 'mturk' experiment_qualification_desc = 'Experiment-specific qualification' group_qualification_desc = 'Experiment group qualification' def __init__(self): super(MTurkRecruiter, self).__init__() self.config = get_config() self.ad_url = '{}/ad?recruiter={}'.format( get_base_url(), self.nickname, ) self.hit_domain = os.getenv('HOST') self.mturkservice = MTurkService( self.config.get('aws_access_key_id'), self.config.get('aws_secret_access_key'), self.config.get('aws_region'), self.config.get('mode') != "live") self._validate_config() def _validate_config(self): mode = self.config.get('mode') if mode not in ('sandbox', 'live'): raise MTurkRecruiterException( '"{}" is not a valid mode for MTurk recruitment. ' 'The value of "mode" must be either "sandbox" or "live"'. format(mode)) @property def external_submission_url(self): """On experiment completion, participants are returned to the Mechanical Turk site to submit their HIT, which in turn triggers notifications to the /notifications route. """ if self.config.get('mode') == "sandbox": return "https://workersandbox.mturk.com/mturk/externalSubmit" return "https://www.mturk.com/mturk/externalSubmit" @property def qualifications(self): quals = {self.config.get('id'): self.experiment_qualification_desc} group_name = self.config.get('group_name', None) if group_name: quals[group_name] = self.group_qualification_desc return quals def open_recruitment(self, n=1): """Open a connection to AWS MTurk and create a HIT.""" if self.is_in_progress: # Already started... do nothing. return None if self.hit_domain is None: raise MTurkRecruiterException("Can't run a HIT from localhost") self.mturkservice.check_credentials() if self.config.get('assign_qualifications'): self._create_mturk_qualifications() hit_request = { 'max_assignments': n, 'title': self.config.get('title'), 'description': self.config.get('description'), 'keywords': self._config_to_list('keywords'), 'reward': self.config.get('base_payment'), 'duration_hours': self.config.get('duration'), 'lifetime_days': self.config.get('lifetime'), 'ad_url': self.ad_url, 'notification_url': self.config.get('notification_url'), 'approve_requirement': self.config.get('approve_requirement'), 'us_only': self.config.get('us_only'), 'blacklist': self._config_to_list('qualification_blacklist'), 'annotation': self.config.get('id'), } hit_info = self.mturkservice.create_hit(**hit_request) if self.config.get('mode') == "sandbox": lookup_url = "https://workersandbox.mturk.com/mturk/preview?groupId={type_id}" else: lookup_url = "https://worker.mturk.com/mturk/preview?groupId={type_id}" return { 'items': [ lookup_url.format(**hit_info), ], 'message': 'HIT now published to Amazon Mechanical Turk' } def recruit(self, n=1): """Recruit n new participants to an existing HIT""" if not self.config.get('auto_recruit', False): logger.info('auto_recruit is False: recruitment suppressed') return hit_id = self.current_hit_id() if hit_id is None: logger.info('no HIT in progress: recruitment aborted') return try: return self.mturkservice.extend_hit( hit_id, number=n, duration_hours=self.config.get('duration')) except MTurkServiceException as ex: logger.exception(str(ex)) def notify_completed(self, participant): """Assign a Qualification to the Participant for the experiment ID, and for the configured group_name, if it's been set. """ if not self.config.get('assign_qualifications'): return worker_id = participant.worker_id for name in self.qualifications: try: self.mturkservice.increment_qualification_score( name, worker_id) except QualificationNotFoundException as ex: logger.exception(ex) def rejects_questionnaire_from(self, participant): """Mechanical Turk participants submit their HITs on the MTurk site (see external_submission_url), and MTurk then sends a notification to Dallinger which is used to mark the assignment completed. If a HIT has already been submitted, it's too late to submit the questionnaire. """ if participant.status != "working": return ("This participant has already sumbitted their HIT " "on MTurk and can no longer submit the questionnaire") def submitted_event(self): """MTurk will send its own notification when the worker completes the HIT on that service. """ return None def reward_bonus(self, assignment_id, amount, reason): """Reward the Turker for a specified assignment with a bonus.""" try: return self.mturkservice.grant_bonus(assignment_id, amount, reason) except MTurkServiceException as ex: logger.exception(str(ex)) @property def is_in_progress(self): return bool(Participant.query.first()) def current_hit_id(self): any_participant_record = Participant.query.with_entities( Participant.hit_id).first() if any_participant_record is not None: return str(any_participant_record.hit_id) def approve_hit(self, assignment_id): try: return self.mturkservice.approve_assignment(assignment_id) except MTurkServiceException as ex: logger.exception(str(ex)) def close_recruitment(self): """Clean up once the experiment is complete. This may be called before all users have finished so uses the expire_hit rather than the disable_hit API call. This allows people who have already picked up the hit to complete it as normal. """ logger.info(CLOSE_RECRUITMENT_LOG_PREFIX) # We are not expiring the hit currently as notifications are failing # TODO: Reinstate this # try: # return self.mturkservice.expire_hit( # self.current_hit_id(), # ) # except MTurkServiceException as ex: # logger.exception(str(ex)) def _config_to_list(self, key): # At some point we'll support lists, so all service code supports them, # but the config system only supports strings for now, so we convert: as_string = self.config.get(key, '') return [item.strip() for item in as_string.split(',') if item.strip()] def _create_mturk_qualifications(self): """Create MTurk Qualification for experiment ID, and for group_name if it's been set. Qualifications with these names already exist, but it's faster to try and fail than to check, then try. """ for name, desc in self.qualifications.items(): try: self.mturkservice.create_qualification_type(name, desc) except DuplicateQualificationNameError: pass
class MTurkRecruiter(Recruiter): """Recruit participants from Amazon Mechanical Turk""" nickname = "mturk" experiment_qualification_desc = "Experiment-specific qualification" group_qualification_desc = "Experiment group qualification" def __init__(self): super(MTurkRecruiter, self).__init__() self.config = get_config() self.ad_url = "{}/ad?recruiter={}".format(get_base_url(), self.nickname) self.hit_domain = os.getenv("HOST") self.mturkservice = MTurkService( self.config.get("aws_access_key_id"), self.config.get("aws_secret_access_key"), self.config.get("aws_region"), self.config.get("mode") != "live", ) self.messenger = get_messenger(self.config) self._validate_config() def _validate_config(self): mode = self.config.get("mode") if mode not in ("sandbox", "live"): raise MTurkRecruiterException( '"{}" is not a valid mode for MTurk recruitment. ' 'The value of "mode" must be either "sandbox" or "live"'. format(mode)) @property def external_submission_url(self): """On experiment completion, participants are returned to the Mechanical Turk site to submit their HIT, which in turn triggers notifications to the /notifications route. """ if self.config.get("mode") == "sandbox": return "https://workersandbox.mturk.com/mturk/externalSubmit" return "https://www.mturk.com/mturk/externalSubmit" @property def qualifications(self): quals = {self.config.get("id"): self.experiment_qualification_desc} group_name = self.config.get("group_name", None) if group_name: quals[group_name] = self.group_qualification_desc return quals def open_recruitment(self, n=1): """Open a connection to AWS MTurk and create a HIT.""" logger.info("Opening MTurk recruitment for {} participants".format(n)) if self.is_in_progress: raise MTurkRecruiterException( "Tried to open_recruitment on already open recruiter.") if self.hit_domain is None: raise MTurkRecruiterException("Can't run a HIT from localhost") self.mturkservice.check_credentials() if self.config.get("assign_qualifications"): self._create_mturk_qualifications() hit_request = { "max_assignments": n, "title": self.config.get("title"), "description": self.config.get("description"), "keywords": self._config_to_list("keywords"), "reward": self.config.get("base_payment"), "duration_hours": self.config.get("duration"), "lifetime_days": self.config.get("lifetime"), "ad_url": self.ad_url, "notification_url": self.config.get("notification_url"), "approve_requirement": self.config.get("approve_requirement"), "us_only": self.config.get("us_only"), "blacklist": self._config_to_list("qualification_blacklist"), "annotation": self.config.get("id"), } hit_info = self.mturkservice.create_hit(**hit_request) if self.config.get("mode") == "sandbox": lookup_url = ( "https://workersandbox.mturk.com/mturk/preview?groupId={type_id}" ) else: lookup_url = "https://worker.mturk.com/mturk/preview?groupId={type_id}" return { "items": [lookup_url.format(**hit_info)], "message": "HIT now published to Amazon Mechanical Turk", } def recruit(self, n=1): """Recruit n new participants to an existing HIT""" logger.info("Recruiting {} MTurk participants".format(n)) if not self.config.get("auto_recruit"): logger.info("auto_recruit is False: recruitment suppressed") return hit_id = self.current_hit_id() if hit_id is None: logger.info("no HIT in progress: recruitment aborted") return try: return self.mturkservice.extend_hit( hit_id, number=n, duration_hours=self.config.get("duration")) except MTurkServiceException as ex: logger.exception(str(ex)) def notify_completed(self, participant): """Assign a Qualification to the Participant for the experiment ID, and for the configured group_name, if it's been set. Overrecruited participants don't receive qualifications, since they haven't actually completed the experiment. This allows them to remain eligible for future runs. """ if participant.status == "overrecruited" or not self.qualification_active: return worker_id = participant.worker_id for name in self.qualifications: try: self.mturkservice.increment_qualification_score( name, worker_id) except QualificationNotFoundException as ex: logger.exception(ex) def notify_duration_exceeded(self, participants, reference_time): """The participant has exceed the maximum time for the activity, defined in the "duration" config value. We need find out the assignment status on MTurk and act based on this. """ unsubmitted = [] for participant in participants: summary = ParticipationTime(participant, reference_time, self.config) status = self._mturk_status_for(participant) if status == "Approved": participant.status = "approved" session.commit() elif status == "Rejected": participant.status = "rejected" session.commit() elif status == "Submitted": self._resend_submitted_rest_notification_for(participant) self._message_researcher(self._resubmitted_msg(summary)) logger.warning( "Error - submitted notification for participant {} missed. " "A replacement notification was created and sent, " "but proceed with caution.".format(participant.id)) else: self._send_notification_missing_rest_notification_for( participant) unsubmitted.append(summary) if unsubmitted: self._disable_autorecruit() self.close_recruitment() pick_one = unsubmitted[0] # message the researcher about the one of the participants: self._message_researcher(self._cancelled_msg(pick_one)) # Attempt to force-expire the hit via boto. It's possible # that the HIT won't exist if the HIT has been deleted manually. try: self.mturkservice.expire_hit(pick_one.participant.hit_id) except MTurkServiceException as ex: logger.exception(ex) def rejects_questionnaire_from(self, participant): """Mechanical Turk participants submit their HITs on the MTurk site (see external_submission_url), and MTurk then sends a notification to Dallinger which is used to mark the assignment completed. If a HIT has already been submitted, it's too late to submit the questionnaire. """ if participant.status != "working": return ("This participant has already sumbitted their HIT " "on MTurk and can no longer submit the questionnaire") def submitted_event(self): """MTurk will send its own notification when the worker completes the HIT on that service. """ return None def reward_bonus(self, assignment_id, amount, reason): """Reward the Turker for a specified assignment with a bonus.""" try: return self.mturkservice.grant_bonus(assignment_id, amount, reason) except MTurkServiceException as ex: logger.exception(str(ex)) @property def is_in_progress(self): # Has this recruiter resulted in any participants? return bool( Participant.query.filter_by(recruiter_id=self.nickname).first()) @property def qualification_active(self): return bool(self.config.get("assign_qualifications")) def current_hit_id(self): any_participant_record = (Participant.query.with_entities( Participant.hit_id).filter_by(recruiter_id=self.nickname).first()) if any_participant_record is not None: return str(any_participant_record.hit_id) def approve_hit(self, assignment_id): try: return self.mturkservice.approve_assignment(assignment_id) except MTurkServiceException as ex: logger.exception(str(ex)) def close_recruitment(self): """Clean up once the experiment is complete. This may be called before all users have finished so uses the expire_hit rather than the disable_hit API call. This allows people who have already picked up the hit to complete it as normal. """ logger.info(CLOSE_RECRUITMENT_LOG_PREFIX + " mturk") # We are not expiring the hit currently as notifications are failing # TODO: Reinstate this # try: # return self.mturkservice.expire_hit( # self.current_hit_id(), # ) # except MTurkServiceException as ex: # logger.exception(str(ex)) def _mturk_status_for(self, participant): try: assignment = self.mturkservice.get_assignment( participant.assignment_id) status = assignment["status"] except Exception: status = None return status def _disable_autorecruit(self): heroku_app = heroku_tools.HerokuApp(self.config.get("id")) args = json.dumps({"auto_recruit": "false"}) headers = heroku_tools.request_headers( self.config.get("heroku_auth_token")) requests.patch(heroku_app.config_url, data=args, headers=headers) def _resend_submitted_rest_notification_for(self, participant): notification_url = self.config.get("notification_url") args = { "Event.1.EventType": "AssignmentSubmitted", "Event.1.AssignmentId": participant.assignment_id, } requests.post(notification_url, data=args) def _send_notification_missing_rest_notification_for(self, participant): notification_url = self.config.get("notification_url") args = { "Event.1.EventType": "NotificationMissing", "Event.1.AssignmentId": participant.assignment_id, } requests.post(notification_url, data=args) def _config_to_list(self, key): # At some point we'll support lists, so all service code supports them, # but the config system only supports strings for now, so we convert: as_string = self.config.get(key, None) if as_string is None: return [] return [item.strip() for item in as_string.split(",") if item.strip()] def _create_mturk_qualifications(self): """Create MTurk Qualification for experiment ID, and for group_name if it's been set. Qualifications with these names already exist, but it's faster to try and fail than to check, then try. """ for name, desc in self.qualifications.items(): try: self.mturkservice.create_qualification_type(name, desc) except DuplicateQualificationNameError: pass def _resubmitted_msg(self, summary): templates = MTurkHITMessages.by_flavor(summary, self.config.get("whimsical")) return templates.resubmitted_msg() def _cancelled_msg(self, summary): templates = MTurkHITMessages.by_flavor(summary, self.config.get("whimsical")) return templates.hit_cancelled_msg() def _message_researcher(self, message): try: self.messenger.send(message) except MessengerError as ex: logger.exception(ex)