def test_locale_qualification_in_notin(self):
self.set_http_response(status_code=200, body=MOCK_SERVER_RESPONSE)
q = ExternalQuestion(external_url="http://samplesite",
frame_height=800)
keywords = ['boto', 'test', 'doctest']
title = "Boto External Question Test"
annotation = 'An annotation from boto external question test'
qualifications = Qualifications()
test_requirement1 = LocaleRequirement(comparator='In',
locale=[('US', 'CA')])
test_requirement2 = LocaleRequirement(comparator='NotIn',
locale=[('US', 'WA'), 'CA'])
qualifications.add(test_requirement1)
qualifications.add(test_requirement2)
create_hit_rs = self.service_connection.create_hit(
question=q,
lifetime=60 * 65,
max_assignments=2,
title=title,
keywords=keywords,
reward=0.05,
duration=60 * 6,
approval_delay=60 * 60,
annotation=annotation,
qualifications=qualifications)
self.assert_request_parameters(
{
'QualificationRequirement.1.Comparator':
'In',
'QualificationRequirement.1.LocaleValue.1.Country':
'US',
'QualificationRequirement.1.LocaleValue.1.Subdivision':
'CA',
'QualificationRequirement.1.QualificationTypeId':
'00000000000000000071',
'QualificationRequirement.2.Comparator':
'NotIn',
'QualificationRequirement.2.LocaleValue.1.Country':
'US',
'QualificationRequirement.2.LocaleValue.1.Subdivision':
'WA',
'QualificationRequirement.2.LocaleValue.2.Country':
'CA',
'QualificationRequirement.2.QualificationTypeId':
'00000000000000000071'
},
ignore_params_values=[
'AWSAccessKeyId', 'SignatureVersion', 'Timestamp', 'Title',
'Question', 'AssignmentDurationInSeconds',
'RequesterAnnotation', 'Version', 'LifetimeInSeconds',
'AutoApprovalDelayInSeconds', 'Reward.1.Amount', 'Description',
'MaxAssignments', 'Reward.1.CurrencyCode', 'Keywords',
'Operation'
])
self.assertEquals(create_hit_rs.status, True)
def create_campaign_hits(campaignid):
"""
Create HITs for a campaign.
"""
campaign = Campaign.query.filter_by(id=campaignid).first()
options = campaign.options
terms = campaign.terms
# Determine the number of quizzes that need to be generated
numquizzes = int(math.ceil(float(len(terms)) / campaign.terms_per_quiz))
# A set of questions will be built for each "page"
for pagenum in range(numquizzes):
conn = create_connection()
question_form = create_question_form(campaign.title, campaign.title,"categorization")
# Add the term questions to the page / quiz
for i in range(campaign.terms_per_quiz):
termnum = i + (pagenum * campaign.terms_per_quiz)
if termnum < len(terms):
term = terms[termnum]
# Build up an answer dictionary for use with the question form
answers = [(cgi.escape(option.option_text),
("%s|%s" % (option.id, cgi.escape(option.option_text))))
for option in options]
# A question should be appended for each term under consideration
questiontext = campaign.question.replace("[term]", cgi.escape(term.term))
identifier = "%s|%s|%s" % (campaign.id, term.id, cgi.escape(term.term))
question_form.append(create_question(identifier,
questiontext,
answers))
# Create a locale requirement that answerers must be in the US
qualifications = Qualifications()
lr = LocaleRequirement("EqualTo","US")
qualifications.add(lr)
print lr.get_as_params()
# Now that the questions have been added, create the HIT
conn.create_hit(questions=question_form,
max_assignments=campaign.times_per_term,
title=campaign.title,
description=campaign.title,
duration=60*30, # allot 30 minutes per quiz
qualifications=qualifications, # require the answerers be in the US
reward=campaign.reward_per_quiz)
# Update the campaign to prevent multiple generation attempts
campaign.job_generated = True
db.session.add(campaign)
db.session.commit()
class Production(Config):
DEBUG = False
DEVELOPMENT = False
TESTING = False
MTURK_HOST = 'mechanicalturk.amazonaws.com'
MTURK_EXTERNAL_SUBMIT = 'https://www.mturk.com/mturk/externalSubmit'
QUALIFICATIONS = Qualifications([
NumberHitsApprovedRequirement('GreaterThan', 500),
PercentAssignmentsApprovedRequirement('GreaterThan', 95),
LocaleRequirement('EqualTo', 'US')
])
CELERYBEAT_SCHEDULE = {
'delete_temp_files': {
'task': 'delete_temp_files',
'schedule': timedelta(seconds=86400),
'args': ()
},
'run_gather': {
'task': 'run_gather',
'schedule': timedelta(seconds=60),
'args': ()
},
}
TACKBP_NW_09_CORPUS_URL = 'https://s3-us-west-2.amazonaws.com/tac-kbp-2009/sentences.meta-sentencesonly-no-liu-et-al-naacl2016-test-set-270k'
def getHITType():
# Changing this will add another hit type and might mess up later fetches...
# Only change if you know what you are doing...
_mtc = MTurkConnection(host=_host)
_title = "Guess the Closest Part of a Person!"
_description = "Help us find out which body part of a person is closest to the camera that took the picture."
_keywords = "person, people, image, images, object, objects, depth, comparisons, human3.6m"
_reward = _mtc.get_price_as_price(0.1)
_duration = 60 * 15
_approval_delay = 60 * 60 * 24 * 10
_qualifications = Qualifications()
_qualifications.add(
PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo',
98,
required_to_preview=True))
_qualifications.add(
NumberHitsApprovedRequirement('GreaterThanOrEqualTo',
100,
required_to_preview=True))
_qualifications.add(
LocaleRequirement('EqualTo', 'US', required_to_preview=True))
return _mtc.register_hit_type(title=_title,
description=_description,
reward=_reward,
duration=_duration,
keywords=_keywords,
approval_delay=_approval_delay,
qual_req=_qualifications)
def getHITType():
# Changing this will add another hit type and might mess up later fetches...
# Only change if you know what you are doing...
_mtc = MTurkConnection(host=_host)
_title = "Select Person-Object Interactions in Images"
_description = "Please click on all the objects (or other people) that the highlighted person is interacting with."
_reward = _mtc.get_price_as_price(0.10)
_duration = 60 * 10
_keywords = "person, people, image, images, object, objects, actions, interactions"
_approval_delay = 60 * 60 * 24 * 5
_qualifications = Qualifications()
_qualifications.add(
PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo',
98,
required_to_preview=True))
_qualifications.add(
NumberHitsApprovedRequirement('GreaterThanOrEqualTo',
100,
required_to_preview=True))
_qualifications.add(
LocaleRequirement('EqualTo', 'US', required_to_preview=True))
return _mtc.register_hit_type(title=_title,
description=_description,
reward=_reward,
duration=_duration,
keywords=_keywords,
approval_delay=_approval_delay,
qual_req=_qualifications)
def configure_hit(self, hit_config):
# configure question_url based on the id
experimentPortalURL = hit_config['ad_location']
frameheight = 600
mturkQuestion = ExternalQuestion(experimentPortalURL, frameheight)
# Qualification:
quals = Qualifications()
approve_requirement = hit_config['approve_requirement']
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if hit_config['us_only']:
quals.add(LocaleRequirement("EqualTo", "US"))
# Specify all the HIT parameters
self.paramdict = dict(hit_type=None,
question=mturkQuestion,
lifetime=hit_config['lifetime'],
max_assignments=hit_config['max_assignments'],
title=hit_config['title'],
description=hit_config['description'],
keywords=hit_config['keywords'],
reward=hit_config['reward'],
duration=hit_config['duration'],
approval_delay=None,
questions=None,
qualifications=quals)
def build_hit_qualifications(self, approve_requirement, restrict_to_usa,
blacklist):
"""Translate restrictions/qualifications to boto Qualifications objects
@blacklist is a list of names for Qualifications workers must
not already hold in order to see and accept the HIT.
"""
quals = Qualifications()
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if restrict_to_usa:
quals.add(LocaleRequirement("EqualTo", "US"))
if blacklist is not None:
for item in blacklist:
qtype = self.get_qualification_type_by_name(item)
if qtype:
quals.add(
Requirement(qtype['id'],
"DoesNotExist",
required_to_preview=True))
return quals
def configure_hit(self):
# Configure portal
experimentPortalURL = self.config.get('HIT Configuration', 'question_url')
frameheight = 600
mturkQuestion = ExternalQuestion(experimentPortalURL, frameheight)
# Qualification:
quals = Qualifications()
approve_requirement = self.config.get('HIT Configuration', 'Approve_Requirement')
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if self.config.getboolean('HIT Configuration', 'US_only'):
quals.add(LocaleRequirement("EqualTo", "US"))
# Specify all the HIT parameters
self.paramdict = dict(
hit_type = None,
question = mturkQuestion,
lifetime = datetime.timedelta(hours=self.config.getfloat('HIT Configuration', 'HIT_lifetime')),
max_assignments = self.config.getint('HIT Configuration', 'max_assignments'),
title = self.config.get('HIT Configuration', 'title'),
description = self.config.get('HIT Configuration', 'description'),
keywords = self.config.get('HIT Configuration', 'keywords'),
reward = self.config.getfloat('HIT Configuration', 'reward'),
duration = datetime.timedelta(hours=self.config.getfloat('HIT Configuration', 'duration')),
approval_delay = None,
questions = None,
qualifications = quals
)
def configure_hit(self, hit_config):
''' Configure HIT '''
# configure question_url based on the id
experiment_portal_url = hit_config['ad_location']
frame_height = 600
mturk_question = ExternalQuestion(experiment_portal_url, frame_height)
# Qualification:
quals = Qualifications()
approve_requirement = hit_config['approve_requirement']
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if hit_config['us_only']:
quals.add(LocaleRequirement("EqualTo", "US"))
# Create a HIT type for this HIT.
hit_type = self.mtc.register_hit_type(hit_config['title'],
hit_config['description'],
hit_config['reward'],
hit_config['duration'],
keywords=hit_config['keywords'],
approval_delay=None,
qual_req=quals)[0]
# Check the config file to see if notifications are wanted.
config = PsiturkConfig()
config.load_config()
try:
url = config.get('Server Parameters', 'notification_url')
all_event_types = [
"AssignmentAccepted",
"AssignmentAbandoned",
"AssignmentReturned",
"AssignmentSubmitted",
"HITReviewable",
"HITExpired",
]
self.mtc.set_rest_notification(hit_type.HITTypeId,
url,
event_types=all_event_types)
except:
pass
# Specify all the HIT parameters
self.param_dict = dict(hit_type=hit_type.HITTypeId,
question=mturk_question,
lifetime=hit_config['lifetime'],
max_assignments=hit_config['max_assignments'],
questions=None,
response_groups=[
'Minimal', 'HITDetail', 'HITQuestion',
'HITAssignmentSummary'
])
def build_hit_qualifications(self, approve_requirement, restrict_to_usa):
"""Translate restrictions/qualifications to boto Qualifications objects"""
quals = Qualifications()
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if restrict_to_usa:
quals.add(LocaleRequirement("EqualTo", "US"))
return quals
def get_requirement(qualification_type_id):
"""
Function to set the requirements. This is optional.
:return: list
"""
qualifications = Qualifications()
qualifications.add(
PercentAssignmentsApprovedRequirement(comparator="GreaterThan",
integer_value="90"))
qualifications.add(LocaleRequirement("EqualTo", "US"))
# qualifications.add(Requirement(qualification_type_id=qualification_type_id, comparator="EqualTo", integer_value="1"))
return qualifications
def build_qualifications(locales=None):
"""
Creates a single qualification that workers have a > 95% acceptance rate.
:return: boto qualification obj.
"""
qualifications = Qualifications()
requirements = [
PercentAssignmentsApprovedRequirement(comparator="GreaterThan",
integer_value="95")
]
if locales:
loc_req = LocaleRequirement(comparator='In', locale=locales)
requirements.append(loc_req)
_ = [qualifications.add(req) for req in requirements]
return qualifications
def update_hit(settings):
global hit
hit_quals = Qualifications()
old_hit_type_id = hit["type_id"]
settings_quals = settings["qualifications"]
## TODO: master worker, custom quals, utility for creating qualifications?
if (settings_quals):
if settings_quals["location"]:
hit_quals.add(
LocaleRequirement("EqualTo", settings_quals["location"]))
if settings_quals["approval_percentage"]:
hit_quals.add(
PercentAssignmentsApprovedRequirement(
"GreaterThanOrEqualTo",
settings_quals["approval_percentage"]))
request_settings = dict(
title=settings["title"],
description=settings["description"],
keywords=settings["keywords"],
reward=settings["reward"],
approval_delay=timedelta(seconds=settings["auto_approval_delay"]),
duration=timedelta(seconds=settings["assignment_duration"]),
qual_req=hit_quals)
try:
new_hit_type_id = mtc.register_hit_type(
**request_settings)[0].HITTypeId
if new_hit_type_id == old_hit_type_id:
prints("Settings haven't changed; not updating")
sys.exit(1)
change_existing_hit_type_result = mtc.change_hit_type_of_hit(
hit["id"], new_hit_type_id)
except MTurkRequestError as e:
print("Error\n")
pp(e.__dict__)
hit_modes[mode]["type_id"] = new_hit_type_id
## write new_hit_type_id to hit_modes.json
with open("hit_modes.json", 'w') as f:
json.dump(hit_modes, f, indent=4, separators=(',', ': '))
prints("Updated from type:", old_hit_type_id, "to type:", new_hit_type_id)
def createSeveralHits(numberOfSubjects):
# HOST = 'mechanicalturk.sandbox.amazonaws.com'
# HOST = 'mechanicalturk.amazonaws.com'
global connection
global layoutID
url = 'THE LINK TO THE STUDY'
title = "TITLE OF THE STUDY"
description = "DESCRIPTION OF THE STUDY"
keywords = ["KEYWORDS"]
amount = 0.4
# game = ExternalQuestion(url, frame_height)
remaining = numberOfSubjects
create_hit_result = None
timestr = time.strftime("%Y%m%d-%H%M%S")
myfile=open("./HITIDs/"+timestr + '.txt', 'w') # Saves all the created hits under HITIDS. The name of the file is based on the date and time of creation.
####Setting Worker Requirements:
workerRequirements=Qualifications()
req1= PercentAssignmentsApprovedRequirement(comparator = "GreaterThan", integer_value = "96",required_to_preview=True)
workerRequirements.add(req1)
req2= LocaleRequirement(comparator = "EqualTo",locale ="US",required_to_preview=True)
workerRequirements.add(req2)
req3=NumberHitsApprovedRequirement(comparator = "GreaterThan", integer_value = "50",required_to_preview=True)
workerRequirements.add(req3)
while remaining>0:
no_assignments = min(9, remaining)
create_hit_result = connection.create_hit(
title=title,
description=description,
keywords=keywords,
max_assignments=no_assignments,
hit_layout=layoutID,
lifetime=6*60*60,
duration=3*60*60,
approval_delay=3*60*60*24,
reward=Price(amount=amount),
qualifications=workerRequirements)
remaining -= no_assignments
myfile.write(create_hit_result[0].HITId+"\n")
print "No. Assignments: ", no_assignments
print "Results:", create_hit_result
print "hit ID:", create_hit_result[0].HITId
def submit_hit(self,
urls,
title,
description,
keywords=None,
duration=datetime.timedelta(seconds=300),
amount=0.0,
response_groups=('Minimal', 'HITDetail'),
num_assignments=5):
quals = Qualifications()
#mturk workers with high apporval in the US.
approve_requirement = 95
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
quals.add(LocaleRequirement("EqualTo", "US"))
hit_type = self.mturk.register_hit_type(
title,
description,
boto.mturk.price.Price(amount=amount),
duration,
keywords,
qual_req=quals,
approval_delay=datetime.timedelta(seconds=100000))
for requirement in quals.requirements:
requirement.required_to_preview = True
for url in urls:
try:
questionform = boto.mturk.question.ExternalQuestion(
url, self.frame_height)
create_hit_result = self.mturk.create_hit(
hit_type=hit_type[0].HITTypeId,
question=questionform,
max_assignments=num_assignments,
response_groups=('Minimal', 'HITDetail'),
qualifications=quals,
lifetime=datetime.timedelta(days=90))
assert create_hit_result.status
except:
print "Could not finish making hits. Check that you have enough money in the account!"
def __init__(self, host, n=10):
mturkparams = dict(
aws_access_key_id=config.get('AWS Access', 'aws_access_key_id'),
aws_secret_access_key=config.get('AWS Access',
'aws_secret_access_key'),
host=host)
self.mtc = MTurkConnection(**mturkparams)
# Configure portal
experimentPortalURL = config.get('HIT Configuration', 'question_url')
frameheight = 600
mturkQuestion = ExternalQuestion(experimentPortalURL, frameheight)
# Qualification:
quals = Qualifications()
approve_requirement = config.getint('HIT Configuration',
'Approve_Requirement')
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if config.getboolean('HIT Configuration', 'US_only'):
quals.add(LocaleRequirement("EqualTo", "US"))
# Specify all the HIT parameters
self.paramdict = dict(
hit_type=None,
question=mturkQuestion,
lifetime=datetime.timedelta(
hours=config.getfloat('HIT Configuration', 'HIT_lifetime')),
max_assignments=config.getint('HIT Configuration',
'max_assignments'),
title=config.get('HIT Configuration', 'title'),
description=config.get('HIT Configuration', 'description'),
keywords=config.get('HIT Configuration', 'keywords'),
reward=config.getfloat('HIT Configuration', 'reward'),
duration=datetime.timedelta(
hours=config.getfloat('HIT Configuration', 'duration')),
approval_delay=None,
questions=None,
qualifications=quals)
class DevelopmentConfig(Config):
DEVELOPMENT = True
DEBUG = True
TESTING = True
MTURK_HOST = 'mechanicalturk.sandbox.amazonaws.com'
MTURK_EXTERNAL_SUBMIT = 'https://workersandbox.mturk.com/mturk/externalSubmit'
#CELERY_REDIS_MAX_CONNECTIONS = 5
QUALIFICATIONS = Qualifications([LocaleRequirement('EqualTo', 'US')])
CELERYBEAT_SCHEDULE = {
'delete_temp_files': {
'task': 'delete_temp_files',
'schedule': timedelta(seconds=86400),
'args': ()
},
'run_gather': {
'task': 'run_gather',
'schedule': timedelta(seconds=20),
'args': ()
},
}
TACKBP_NW_09_CORPUS_URL = 'https://s3-us-west-2.amazonaws.com/tac-kbp-2009/sentences.meta-sentencesonly-no-liu-et-al-naacl2016-test-set-2k'
def createHIT():
# TODO: fix height here
url = 'https://www.zeerak.net/279akz/getTask/'
external_question = ExternalQuestion(url, 1200)
qualifications = Qualifications()
qualifications.add(LocaleRequirement("EqualTo", "US"))
response = mtc.create_hit(
title='Read a short article and learn something new!',
keywords='read, react, share, news',
description='How would you share this on social media? Just one task!',
duration=1800,
question=external_question,
reward=0.30,
max_assignments=145,
qualifications=qualifications,
# response_groups='', # batches??
)
# The response included several fields that will be helpful later
hit_type_Id = response[0].HITTypeId
hit_Id = response[0].HITId
available_balance = mtc.get_account_balance()
print("-------------------------------------------------------")
print("Your HIT has been created. You can see it at this link:")
print(">>>>>> https://workersandbox.mturk.com/mturk/preview?groupId={}".
format(hit_type_Id))
print("Your HIT Id is: {}".format(hit_Id))
print("And see results here:")
print(">>>>>> {}".format(mturk_environment['manage']))
print("Your account balance is {}".format(available_balance))
print("-------------------------------------------------------")
return
def read_config(self):
'''Return (properties, variables_list)'''
filename = self._get_filename('config')
parser = RawConfigParser()
parser.read(filename)
properties = {}
if parser.has_option('properties', 'hittypeid'):
properties['hittypeid'] = parser.get('properties', 'hittypeid')
else:
# Create a new HIT Type ID if not present
for key in ('title', 'description', 'keywords'):
properties[key] = parser.get('properties', key)
properties['reward'] = Price(parser.getfloat('properties', 'reward'))
for key in ('assignmentduration', 'autoapprovaldelay'):
properties[key] = self._parse_time(parser.get('timing', key))
# Qualifications
requirements = []
if parser.has_option('qualifications', 'locale'):
requirements.append(LocaleRequirement(
'EqualTo', parser.get('qualifications', 'locale'), True))
for key in RecordWrapper.QUALIFICATIONS:
if parser.has_option('qualifications', key):
value = parser.get('qualifications', key)
comparator = ''.join(x for x in value if not x.isdigit())
value = int(value[len(comparator):])
requirements.append(RecordWrapper.QUALIFICATIONS[key](
RecordWrapper.COMPARATORS[comparator], value, True))
properties['qualifications'] = Qualifications(requirements)
# Other properties
properties['annotation'] = parser.get('properties', 'annotation')
properties['assignments'] = parser.get('properties', 'assignments')
try:
properties['assignments'] = int(properties['assignments'])
except ValueError:
properties['assignments'] = self.read_assignment_amounts(properties['assignments'])
properties['hitlifetime'] = self._parse_time(parser.get('timing', 'hitlifetime'))
# Question
properties['url'] = parser.get('question', 'url')
properties['frameheight'] = parser.get('question', 'frameheight')
# Input
n = parser.getint('input', 'numhits')
if isinstance(properties['assignments'], list):
assert len(properties['assignments']) == n, (len(properties['assignments']), n)
variables_list = [dict() for i in xrange(n)]
for key in parser.options('input'):
if key != 'numhits':
value = parser.get('input', key)
if value[0] == '[':
value = json.loads(value)
assert len(value) == n
for i in xrange(n):
variables_list[i][key] = value[i]
elif '-' in value:
start, end = [int(x) for x in value.split('-')]
assert end - start + 1 == n
for i in xrange(n):
variables_list[i][key] = start + i
else:
for i in xrange(n):
variables_list[i][key] = value
return properties, variables_list
'raven.contrib.django.raven_compat',
'tests',
'tests.demo',
]
mturk_hit_settings = {
'keywords': ['easy', 'bonus', 'choice', 'study'],
'title': 'Title for your experiment',
'description': 'Description for your experiment',
'frame_height': 500,
'preview_template': 'global/MTurkPreview.html',
'minutes_allotted_per_assignment': 60,
'expiration_hours': 7*24, # 7 days
# to prevent retakes
'grant_qualification_id': 'YOUR_QUALIFICATION_ID_HERE',
'qualification_requirements': [
LocaleRequirement("EqualTo", "US"),
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo", 50),
NumberHitsApprovedRequirement("GreaterThanOrEqualTo", 5),
# Requirement('YOUR_QUALIFICATION_ID_HERE', 'DoesNotExist'),
]
}
SESSION_CONFIG_DEFAULTS = {
'real_world_currency_per_point': 0.01,
'participation_fee': 10.00,
'doc': "",
'mturk_hit_settings': mturk_hit_settings,
}
def create_hit(href, hit):
if hit.type == 'MULTIPLE_URLS': # the max_workers was not set in this form
hit.max_workers = hit.size1 + hit.size2 + hit.size3 + hit.size4 + hit.size5 + \
hit.size6 + hit.size7 + hit.size8 + hit.size9 + hit.size10
hostURL = SANDBOX_HOST if hit.sandbox else HOST
qualifications = Qualifications()
if hit.hit_approval > 0:
qualifications.add(
NumberHitsApprovedRequirement("GreaterThan", hit.hit_approval,
False))
if hit.hit_approval_rate > 0:
qualifications.add(
PercentAssignmentsApprovedRequirement("GreaterThan",
hit.hit_approval_rate,
False))
if hit.accepted_hit_rate > 0:
qualifications.add(
PercentAssignmentsSubmittedRequirement("GreaterThan",
hit.accepted_hit_rate,
False))
if hit.returned_hit_rate > 0:
qualifications.add(
PercentAssignmentsReturnedRequirement("GreaterThan",
hit.returned_hit_rate,
False))
if hit.abandoned_hit_rate > 0:
qualifications.add(
PercentAssignmentsAbandonedRequirement("LessThan",
hit.abandoned_hit_rate,
False))
if hit.rejected_hit_rate > 0:
qualifications.add(
PercentAssignmentsRejectedRequirement("LessThan",
hit.rejected_hit_rate,
False))
if hit.locale_qualification is not None and hit.locale_qualification != 'all':
qualifications.add(
LocaleRequirement("EqualTo", hit.locale_qualification, False))
connection = MTurkConnection(aws_access_key_id=hit.aws_access_key,
aws_secret_access_key=hit.aws_secret_key,
host=hostURL,
is_secure=True)
return Response(
connection.create_hit(
question=ExternalQuestion(href, hit.frame_height),
lifetime=hit.lifetime,
max_assignments=hit.max_workers,
title=hit.title,
description=hit.description,
keywords=['turktime'] # TODO
,
reward=hit.reward,
duration=hit.duration,
approval_delay=hit.approval_delay,
qualifications=qualifications,
response_groups=[
'Minimal', 'HITDetail', 'HITQuestion', 'HITAssignmentSummary'
]))
HOST = {
True: "mechanicalturk.sandbox.amazonaws.com",
False: "mechanicalturk.amazonaws.com"
}[SANDBOX]
NUM_ITERATIONS = 10
EXPERIMENT_URL = """https://sophiaray.github.io/quadmods/abstract.html?shapeCond=Rh,S&condition=label-order"""
mtc = MTurkConnection(aws_access_key_id=AK,
aws_secret_access_key=SK,
host=HOST)
quals = Qualifications()
quals.add(PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 95))
quals.add(NumberHitsApprovedRequirement('GreaterThanOrEqualTo', 1))
quals.add(LocaleRequirement('EqualTo', 'US'))
new_hit = mtc.create_hit(
hit_type=None,
question=ExternalQuestion(EXPERIMENT_URL, 600),
lifetime=2 * 60 *
60, # Amount of time HIT will be available to accept unless 'max_assignments' are accepted before
max_assignments=NUM_ITERATIONS,
title='$3 for 15 min. | Concept learning | University of Louisville',
description=
'Participate in a simple psychological experiment on concept learning. The complete duration should be approximately 15 minutes (reward is estimated according to $12/hr).',
keywords='concepts, learning',
reward=3.0,
duration=45 *
60, # Maximum amount of time turkers are allowed to spend on HIT
approval_delay=1 * 60 *
def create_hit(settings):
global hit
## make sure there isn't already a hit
if (hit is not None):
sys.exit("Error: it looks like you already created the hit in %s mode (HIT ID stored in hit_modes.json)" % mode)
hit_quals = Qualifications()
settings_quals = settings["qualifications"]
## TODO: master worker, custom quals, utility for creating qualifications?
if (settings_quals):
if settings_quals["location"]:
hit_quals.add(LocaleRequirement("EqualTo", settings_quals["location"]))
if settings_quals["approval_percentage"]:
hit_quals.add(PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
settings_quals["approval_percentage"]))
if "custom" in settings_quals and settings_quals["custom"]:
hit_quals.add(Requirement(settings_quals["custom"]["QualificationTypeID"], settings_quals["custom"]["Comparator"],settings_quals["custom"]["IntegerValue"]))
prints(
"Your settings are:",
"",
dict_str(settings_raw)
)
if "http:" in settings["url"]:
sys.exit("Error: inside settings.json, your url is set to use 'http:'. It needs to use 'https:'")
## todo: some static analysis
if dialogue_mode=="verbose":
prints(
"",
"Are these settings okay? (yes/no)")
confirm_settings = raw_input("> ")
else:
confirm_settings = "yes"
if "n" in confirm_settings:
sys.exit()
if dialogue_mode=="verbose":
prints(
"",
"How many assignments do you want to start with?",
"(you can always add more later using cosub add)")
max_assignments = None
while max_assignments is None:
try:
max_assignments = int(raw_input("> "))
except ValueError:
prints("Couldn't understand answer. Try entering a positive integer (e.g., 20)")
else:
prints("You will start with " + str(defaultNSs) + " assignments.")
max_assignments = defaultNSs
if mode == "production":
reward = settings["reward"]
cost = max_assignments * float(reward)
fee = 0.4 if max_assignments > 9 else 0.2
fee_str = "40%" if fee == 0.4 else "20%"
prints(
"The cost will be $%.2f -- %s assignments * $%.2f/assignment + %s fee" % (cost, max_assignments, reward, fee_str)
)
if dialogue_mode=="verbose":
prints("Is this okay? (yes/no)")
confirm_cost = raw_input("> ")
if "n" in confirm_cost:
sys.exit()
else:
print("(This won't cost anything because you're in sandbox mode)")
## TODO: implement bounds checking for assignments
if dialogue_mode=="verbose":
prints(
"",
"How long do you want to collect data for?",
"You can give an answer in seconds, minutes, hours, days, or weeks.",
"(and you can always add more time using cosub add)")
lifetime_seconds = None
while lifetime_seconds is None:
lifetime = raw_input("> ")
lifetime_seconds = timeparse(lifetime)
if not lifetime_seconds:
prints("Couldn't understand answer; try an answer in one of these formats:",
" 2 weeks",
" 3 days",
" 12 hours",
" 30 minutes")
else:
prints("You will collect data for " + defaultDuration + ".")
lifetime_seconds = timeparse(defaultDuration)
## TODO: implement bounds checking for time (30 seconds - 1 year)
prints("","Creating HIT...","")
request_settings = dict(
title = settings["title"],
description = settings["description"],
keywords = settings["keywords"],
question = question.ExternalQuestion(settings["url"], settings["frame_height"]),
max_assignments = max_assignments,
reward = settings["reward"],
approval_delay = timedelta(seconds = settings["auto_approval_delay"]),
duration = timedelta(seconds = settings["assignment_duration"]),
lifetime = timedelta(seconds = lifetime_seconds),
qualifications = hit_quals
)
try:
create_result = mtc.create_hit(**request_settings)[0]
except MTurkRequestError as e:
print("Error\n")
pp(e.__dict__)
sys.exit(1)
hit = {
"id": create_result.HITId,
# hit_group_id = hit.HITGroupId,
"type_id": create_result.HITTypeId
}
hit_modes[mode] = hit
print("Successfully created HIT")
## write hit and HITTypeId into even-odd.json
with open("hit_modes.json", 'w') as new_settings_file:
json.dump(hit_modes, new_settings_file, indent=4, separators=(',', ': '))
print("Wrote HIT ID and HIT Type ID to hit_modes.json")
prints(
"- The number of initial assignments is set to %s" % request_settings["max_assignments"],
"- The initial HIT lifetime is set to %s" % humane_timedelta(request_settings["lifetime"]))
prints(
"",
"Manage HIT: ",
HOST_requester + "/mturk/manageHIT?HITId=" + hit["id"])
prints(
"",
"View HIT: ",
HOST_worker + "/mturk/preview?groupId=" + hit["type_id"],
"")
logger.write({'Action': 'Create', 'Data': settings_raw })
if 'input' in hitdata['question']:
qurls = [hitdata['question']['url'].format(**row) for row in hitdata['question']['input']]
else:
qurls = [hitdata['question']['url']]
questions = [ExternalQuestion(url, hitdata['question']['height']) for url in qurls]
quals = Qualifications()
if 'builtin' in hitdata['qualifications']:
for b in hitdata['qualifications']['builtin']:
if b['qualification'] == 'AdultRequirement':
assert b['value'] in (0, 1), 'value must be 0 or 1, not {}'.format(b['value'])
q = AdultRequirement(b['comparator'], b['value'], b['private'])
elif b['qualification'] == 'LocaleRequirement':
q = LocaleRequirement(b['comparator'], b['locale'], b['private'])
else:
q = {'NumberHitsApprovedRequirement': NumberHitsApprovedRequirement,
'PercentAssignmentsAbandonedRequirement': PercentAssignmentsAbandonedRequirement,
'PercentAssignmentsApprovedRequirement': PercentAssignmentsApprovedRequirement,
'PercentAssignmentsRejectedRequirement': PercentAssignmentsRejectedRequirement,
'PercentAssignmentsReturnedRequirement': PercentAssignmentsReturnedRequirement,
'PercentAssignmentsSubmittedRequirement': PercentAssignmentsSubmittedRequirement
}[b['qualification']](b['comparator'], b['value'], b['private'])
quals.add(q)
if 'custom' in hitdata['qualifications']:
for c in hitdata['qualifications']['custom']:
optional = {}
if 'value' in c:
optional['integer_value'] = c['value']
operation = "RegisterHITType"
description = "Translate 10 words from " + language_name + " to English"
title = "Translate 10 words from " + language_name + " to English"
keywords = "translation, vocabulary, dictionary, " + language_name + ", English, language, research, JHU"
approval_delay = 60 * 60 * 24 * 7 #7 days
reward = 0.15
duration = 60 * 60
qualifications = Qualifications()
qualifications.add(
PercentAssignmentsApprovedRequirement(comparator="GreaterThan",
integer_value="85"))
if language_prefix not in indic:
qualifications.add(
LocaleRequirement(comparator="NotEqualTo", locale="IN"))
response = mturk_conn.register_hit_type(title, description, reward,
duration, keywords,
approval_delay, qualifications)
new_mturk_hittype_id = response[0].HITTypeId
item = {
"hittype_id": hittype_id,
"mturk_hittype_id": mturk_hittype_id,
"new_mturk_hittype_id": new_mturk_hittype_id,
"typename": typename,
"fulltypename": fulltypename
}
q.put(item)
def send_hit(self):
# First check for qualifications
qualifications = Qualifications()
if self.qualification_adult:
qualifications.add(AdultRequirement("EqualTo", 1))
else:
qualifications.add(AdultRequirement("EqualTo", 0))
if self.qualification_custom not in [None, ""]:
qualifications.add(
Requirement(self.qualification_custom,
self.qualification_custom_operator,
self.qualification_custom_value,
required_to_preview=True))
if self.qualification_number_hits_approved is not None:
qual_number_hits = NumberHitsApprovedRequirement(
"GreaterThan", self.qualification_number_hits_approved)
qualifications.add(qual_number_hits)
if self.qualification_percent_assignments_approved is not None:
qual_perc_approved = PercentAssignmentsApprovedRequirement(
"GreaterThan", self.qualification_percent_assignments_approved)
qualifications.add(qual_perc_approved)
if self.qualification_locale != 'None':
qualifications.add(
LocaleRequirement("EqualTo", self.qualification_locale))
# Domain name must be https
url = "%s/turk/%s" % (DOMAIN_NAME, self.id)
frame_height = 900
questionform = ExternalQuestion(url, frame_height)
if len(qualifications.requirements) > 0:
result = self.connection.create_hit(
title=self.title,
description=self.description,
keywords=self.keywords,
duration=datetime.timedelta(self.assignment_duration_in_hours /
24.0),
lifetime=datetime.timedelta(self.lifetime_in_hours / 24.0),
max_assignments=self.max_assignments,
question=questionform,
qualifications=qualifications,
reward=Price(amount=self.reward),
response_groups=('Minimal', 'HITDetail'))[0]
else:
result = self.connection.create_hit(
title=self.title,
description=self.description,
keywords=self.keywords,
duration=datetime.timedelta(self.assignment_duration_in_hours /
24.0),
lifetime=datetime.timedelta(self.lifetime_in_hours / 24.0),
max_assignments=self.max_assignments,
question=questionform,
reward=Price(amount=self.reward),
response_groups=('Minimal', 'HITDetail'))[0]
# Update our hit object with the aws HIT
self.mturk_id = result.HITId
# When we generate the hit, we won't have any assignments to update
self.update(mturk_hit=result)
mtc = MTurkConnection(
aws_access_key_id="EXAMPLE",
aws_secret_access_key="EXAMPLE",
host=HOST)
# Configure portal
experimentPortalURL = "http://smash.psych.nyu.edu:5001/mturk"
frameheight = 600
mturkQuestion = ExternalQuestion( experimentPortalURL, 600 )
# Qualification:
quals = Qualifications();
quals.add( PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo", "95") )
quals.add( LocaleRequirement("EqualTo", "US") )
#quals.add( NumberHitsApprovedRequirement("GreaterThanOrEqualTo", "100") )
# Specify all the HIT parameters
paramdict = dict(
hit_type = None,
question = mturkQuestion,
lifetime = datetime.timedelta(1), # How long the HIT will be available
max_assignments = 50, # Total times it will be assigned, not max per turker
title = "Paid volunteers needed for an online experiment in Psychology",
description = "Learn to categorize a set of cards over a series of training trials.",
keywords = "New York University, psychology experiment, category learning",
reward = 0.75,
duration = datetime.timedelta(hours=2),
approval_delay = None,
annotation = None, # Do we need this? Not clear on what it is.