def test_open_recruitment_with_blocklist(self, recruiter):
recruiter.config.set("mturk_qualification_blocklist", u"foo, bar")
# Our fake response will always return the same QualificationType ID
recruiter.mturkservice.get_qualification_type_by_name.return_value = {
"id": u"fake id"
}
recruiter.open_recruitment(n=1)
recruiter.mturkservice.create_hit.assert_called_once_with(
question=MTurkQuestions.external(
ad_url="http://fake-domain/ad?recruiter=mturk"),
description="fake HIT description",
duration_hours=1.0,
experiment_id="TEST_EXPERIMENT_UID",
lifetime_days=1,
keywords=[u"kw1", u"kw2", u"kw3"],
max_assignments=1,
notification_url="http://fake-domain{}".format(SNS_ROUTE_PATH),
reward=0.01,
title="fake experiment title",
annotation="TEST_EXPERIMENT_UID",
qualifications=[
MTurkQualificationRequirements.min_approval(95),
MTurkQualificationRequirements.restrict_to_countries(["US"]),
MTurkQualificationRequirements.must_not_have(u"fake id"),
MTurkQualificationRequirements.must_not_have(u"fake id"),
],
)
def test_worker_cannot_see_hit_when_blacklist_in_qualifications(
self, with_cleanup, worker_id, qtype
):
with_cleanup.assign_qualification(qtype["id"], worker_id, score=1)
print(
'MANUAL STEP: Check for qualification: "{}". (May be delay)'.format(
qtype["name"]
)
)
input("Any key to continue...")
hit = with_cleanup.create_hit(
**standard_hit_config(
title="Dallinger: Blacklist",
qualifications=[
MTurkQualificationRequirements.must_not_have(qtype["id"])
],
lifetime_days=0.25,
)
)
print(
'MANUAL STEP: Should NOT be able to see "{}"" as available HIT'.format(
hit["title"]
)
)
input("Any key to continue...")
pass
def test_create_hit_with_qualification(self, with_cleanup, qtype):
qual = MTurkQualificationRequirements.must_not_have(
qualification_id=qtype["id"]
)
hit = with_cleanup.create_hit(**standard_hit_config(qualifications=[qual]))
assert hit["status"] == "Assignable"
assert hit["qualification_type_ids"] == [qtype["id"]]
def test_open_recruitment_single_recruitee_builds_hit(self, recruiter):
recruiter.open_recruitment(n=1)
recruiter.mturkservice.create_hit.assert_called_once_with(
question=MTurkQuestions.external(
ad_url="http://fake-domain/ad?recruiter=mturk"),
description=u"fake HIT description",
duration_hours=1.0,
experiment_id="TEST_EXPERIMENT_UID",
keywords=[u"kw1", u"kw2", u"kw3"],
lifetime_days=1,
max_assignments=1,
notification_url=u"http://fake-domain{}".format(SNS_ROUTE_PATH),
reward=0.01,
title=u"fake experiment title",
annotation="TEST_EXPERIMENT_UID",
qualifications=[
MTurkQualificationRequirements.min_approval(95),
MTurkQualificationRequirements.restrict_to_countries(["US"]),
],
)
def compensate_worker(self, worker_id, email, dollars, notify=True):
"""Pay a worker by means of a special HIT that only they can see.
"""
qualification = self.mturkservice.create_qualification_type(
name="Dallinger Compensation Qualification - {}".format(
generate_random_id()),
description=(
"You have received a qualification to allow you to complete a "
"compensation HIT from Dallinger for ${}.".format(dollars)),
)
qid = qualification["id"]
self.mturkservice.assign_qualification(qid,
worker_id,
1,
notify=notify)
hit_request = {
"experiment_id": "(compensation only)",
"max_assignments": 1,
"title": "Dallinger Compensation HIT",
"description": "For compenation only; no task required.",
"keywords": [],
"reward": float(dollars),
"duration_hours": 1,
"lifetime_days": 3,
"question": MTurkQuestions.compensation(sandbox=self.is_sandbox),
"qualifications": [MTurkQualificationRequirements.must_have(qid)],
"do_subscribe": False,
}
hit_info = self.mturkservice.create_hit(**hit_request)
if email is not None:
message = {
"subject":
"Dallinger Compensation HIT",
"sender":
self.config.get("dallinger_email_address"),
"recipients": [email],
"body":
("A special compenstation HIT is available for you to complete on MTurk.\n\n"
"Title: {title}\n"
"Reward: ${reward:.2f}\n"
"URL: {worker_url}").format(**hit_info),
}
self.mailer.send(**message)
else:
message = {}
return {
"hit": hit_info,
"qualification": qualification,
"email": message
}
def test_open_recruitment_with_more_than_pool_size_uses_requested_count(
self, recruiter):
num_recruits = recruiter.pool_size + 1
recruiter.open_recruitment(n=num_recruits)
recruiter.mturkservice.create_hit.assert_called_once_with(
question=MTurkQuestions.external(
ad_url="http://fake-domain/ad?recruiter=mturklarge"),
description="fake HIT description",
duration_hours=1.0,
experiment_id="TEST_EXPERIMENT_UID",
keywords=["kw1", "kw2", "kw3"],
lifetime_days=1,
max_assignments=num_recruits,
notification_url="http://fake-domain{}".format(SNS_ROUTE_PATH),
reward=0.01,
title="fake experiment title",
annotation="TEST_EXPERIMENT_UID",
qualifications=[
MTurkQualificationRequirements.min_approval(95),
MTurkQualificationRequirements.restrict_to_countries(["US"]),
],
)
def standard_hit_config(**kwargs):
defaults = {
"experiment_id": "some-experiment-id",
"lifetime_days": 0.0004, # 34 seconds (30 is minimum)
"max_assignments": 1,
"notification_url": "https://url-of-notification-route",
"title": "Test Title",
"keywords": ["testkw1", "testkw2"],
"reward": 0.01,
"question": MTurkQuestions.external(ad_url="https://url-of-ad-route"),
"duration_hours": 0.25,
"qualifications": [
MTurkQualificationRequirements.min_approval(95),
MTurkQualificationRequirements.restrict_to_countries(["US"]),
],
"do_subscribe": False,
}
defaults.update(**kwargs)
# Use fixed description, since this is how we clean up:
defaults["description"] = TEST_HIT_DESCRIPTION + system_marker()
return defaults
def test_open_recruitment_with_explicit_qualifications(self, recruiter):
recruiter.config.set(
"mturk_qualification_requirements",
u"""
[
{
"QualificationTypeId":"789RVWYBAZW00EXAMPLE",
"Comparator":"In",
"IntegerValues":[10, 20, 30]
}
]
""",
)
recruiter.open_recruitment(n=1)
recruiter.mturkservice.create_hit.assert_called_once_with(
question=MTurkQuestions.external(
ad_url="http://fake-domain/ad?recruiter=mturk"),
description="fake HIT description",
duration_hours=1.0,
experiment_id="TEST_EXPERIMENT_UID",
lifetime_days=1,
keywords=[u"kw1", u"kw2", u"kw3"],
max_assignments=1,
notification_url="http://fake-domain{}".format(SNS_ROUTE_PATH),
reward=0.01,
title="fake experiment title",
annotation="TEST_EXPERIMENT_UID",
qualifications=[
MTurkQualificationRequirements.min_approval(95),
MTurkQualificationRequirements.restrict_to_countries(["US"]),
{
"QualificationTypeId": "789RVWYBAZW00EXAMPLE",
"Comparator": "In",
"IntegerValues": [10, 20, 30],
},
],
)
def test_create_hit_lifecycle(self, with_cleanup, qtype, worker_id):
result = with_cleanup.get_qualification_type_by_name(qtype["name"])
assert qtype == result
with_cleanup.assign_qualification(qtype["id"], worker_id, score=2)
workers = with_cleanup.get_workers_with_qualification(qtype["id"])
assert worker_id in [w["id"] for w in workers]
result = with_cleanup.increment_qualification_score(qtype["name"], worker_id)
assert result["score"] == 3
qualifications = (MTurkQualificationRequirements.must_have(qtype["id"]),)
config = standard_hit_config(
max_assignments=2,
annotation="test-annotation",
qualifications=qualifications,
)
hit = with_cleanup.create_hit(**config)
assert hit["status"] == "Assignable"
assert hit["max_assignments"] == 2
assert hit["annotation"] == "test-annotation"
# There is a lag before extension is possible
sleep_secs = 2
max_wait = 60
time.sleep(sleep_secs)
start = time.time()
updated = None
while not updated and time.time() - start < max_wait:
try:
updated = with_cleanup.extend_hit(
hit["id"], number=1, duration_hours=0.25
)
except MTurkServiceException:
time.sleep(sleep_secs)
if updated is None:
pytest.fail("HIT was never updated")
else:
assert updated["max_assignments"] == 3
assert with_cleanup.disable_hit(
hit_id=hit["id"], experiment_id=config["experiment_id"]
)