def send_task(self):
if self.osm_from is not None and self.osm_to is not None and self.reference is not None:
title = 'Find the corresponding product'
description = ('Select the most appropriate answer')
keywords = 'images, selecting, products, matching, match, selection'
self.save_task()
question = ExternalQuestion('http://www.dalliz.com/mturk/key/%s'%(self.key), 1106)
qualifications = Qualifications()
qualifications.add(PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 95))
#--------------- CREATE THE HIT -------------------
a = self.mtc.create_hit(
question=question,
title=title,
description=description,
keywords=keywords,
reward=0.01,
max_assignments=10,
approval_delay=datetime.timedelta(seconds=3600*24*30), # auto-approve timeout
duration = datetime.timedelta(seconds=3600*24*30))
def __init__(self):
if SANDBOX:
host = 'mechanicalturk.sandbox.amazonaws.com'
else:
host = 'mechanicalturk.amazonaws.com'
login_params = {
'aws_access_key_id': ACCESS_KEY_ID,
'aws_secret_access_key': SECRET_ACCESS_KEY,
'host': host
}
self.connection = MTurkConnection(**login_params)
question = ExternalQuestion(EXTERNAL_URL, FRAMEHEIGHT)
qualifications = Qualifications()
qualifications.add(PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo", PERCENT_APPROVED_REQUIREMENT))
self.hit_parameters = {
'hit_type' : None,
'question' : question,
'title' : TITLE,
'description' : DESCRIPTION,
'keywords' : KEYWORDS,
'max_assignments' : MAX_ASSIGNMENTS,
'lifetime' : datetime.timedelta(hours=HIT_LIFETIME),
'reward' : REWARD,
'duration' : datetime.timedelta(DURATION),
'approval_delay' : None,
'questions' : None,
'qualifications' : qualifications
}
def configure_hit(self, hit_config):
# configure question_url based on the id
experimentPortalURL = hit_config['ad_location']
frameheight = 600
mturkQuestion = ExternalQuestion(experimentPortalURL, frameheight)
# Qualification:
quals = Qualifications()
approve_requirement = hit_config['approve_requirement']
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if hit_config['us_only']:
quals.add(LocaleRequirement("EqualTo", "US"))
# Specify all the HIT parameters
self.paramdict = dict(
hit_type = None,
question = mturkQuestion,
lifetime = hit_config['lifetime'],
max_assignments = hit_config['max_assignments'],
title = hit_config['title'],
description = hit_config['description'],
keywords = hit_config['keywords'],
reward = hit_config['reward'],
duration = hit_config['duration'],
approval_delay = None,
questions = None,
qualifications = quals
)
def send_task(self):
if self.osm_from is not None and self.osm_to is not None and self.reference is not None:
title = 'Find the corresponding product'
description = ('Select the most appropriate answer')
keywords = 'images, selecting, products, matching, match, selection'
self.save_task()
question = ExternalQuestion(
'http://www.dalliz.com/mturk/key/%s' % (self.key), 1106)
qualifications = Qualifications()
qualifications.add(
PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo',
95))
#--------------- CREATE THE HIT -------------------
a = self.mtc.create_hit(
question=question,
title=title,
description=description,
keywords=keywords,
reward=0.01,
max_assignments=10,
approval_delay=datetime.timedelta(seconds=3600 * 24 *
30), # auto-approve timeout
duration=datetime.timedelta(seconds=3600 * 24 * 30))
def test_locale_qualification_in_notin(self):
self.set_http_response(
status_code=200,
body=MOCK_SERVER_RESPONSE)
q = ExternalQuestion(
external_url="http://samplesite",
frame_height=800)
keywords = ['boto', 'test', 'doctest']
title = "Boto External Question Test"
annotation = 'An annotation from boto external question test'
qualifications = Qualifications()
test_requirement1 = LocaleRequirement(
comparator='In',
locale=[('US', 'CA')])
test_requirement2 = LocaleRequirement(
comparator='NotIn',
locale=[('US', 'WA'), 'CA'])
qualifications.add(test_requirement1)
qualifications.add(test_requirement2)
create_hit_rs = self.service_connection.create_hit(
question=q,
lifetime=60*65,
max_assignments=2,
title=title,
keywords=keywords,
reward=0.05,
duration=60*6,
approval_delay=60*60,
annotation=annotation,
qualifications=qualifications)
self.assert_request_parameters({
'QualificationRequirement.1.Comparator': 'In',
'QualificationRequirement.1.LocaleValue.1.Country': 'US',
'QualificationRequirement.1.LocaleValue.1.Subdivision': 'CA',
'QualificationRequirement.1.QualificationTypeId':
'00000000000000000071',
'QualificationRequirement.2.Comparator': 'NotIn',
'QualificationRequirement.2.LocaleValue.1.Country': 'US',
'QualificationRequirement.2.LocaleValue.1.Subdivision': 'WA',
'QualificationRequirement.2.LocaleValue.2.Country': 'CA',
'QualificationRequirement.2.QualificationTypeId':
'00000000000000000071'},
ignore_params_values=['AWSAccessKeyId',
'SignatureVersion',
'Timestamp',
'Title',
'Question',
'AssignmentDurationInSeconds',
'RequesterAnnotation',
'Version',
'LifetimeInSeconds',
'AutoApprovalDelayInSeconds',
'Reward.1.Amount',
'Description',
'MaxAssignments',
'Reward.1.CurrencyCode',
'Keywords',
'Operation'])
self.assertEquals(create_hit_rs.status, True)
def build_qualifications():
"""
Creates a single qualification that workers have a > 95% acceptance rate.
:return: boto qualification obj.
"""
qualifications = Qualifications()
req1 = PercentAssignmentsApprovedRequirement(comparator="GreaterThan", integer_value="95")
qualifications.add(req1)
return qualifications
def test():
q = ExternalQuestion(external_url="http://websort.net/s/F3481C", frame_height=800)
conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com')
keywords=['boto', 'test', 'doctest']
qualifications = Qualifications()
qualifications.add(PercentAssignmentsApprovedRequirement(comparator="GreaterThan", integer_value="95"))
create_hit_rs = conn.create_hit(question=q, lifetime=60*65, max_assignments=2, title="Boto External Question Test", keywords=keywords, reward = 0.05, duration=60*6, approval_delay=60*60, annotation='An annotation from boto external question test', qualifications=qualifications)
assert(create_hit_rs.status == True)
print create_hit_rs.HITTypeId
def test():
q = ExternalQuestion(external_url="http://websort.net/s/F3481C", frame_height=800)
conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com')
keywords=['boto', 'test', 'doctest']
qualifications = Qualifications()
qualifications.add(PercentAssignmentsApprovedRequirement(comparator="GreaterThan", integer_value="95"))
create_hit_rs = conn.create_hit(question=q, lifetime=60*65,max_assignments=2,title="Boto External Question Test", keywords=keywords,reward = 0.05, duration=60*6,approval_delay=60*60, annotation='An annotation from boto external question test', qualifications=qualifications)
assert(create_hit_rs.status == True)
print create_hit_rs.HITTypeId
def register_hit(self, configuration=None):
"""
Register a hit on Mechanical Turk according to `hit_params`. This will provide you with a HITTypeId.
Parameters
----------
configuration : dict
Returns
-------
str
The HITTypeId which is how you refer to your newly registered hit with Amazon
"""
if configuration is None:
configuration = app.config
qualifications = Qualifications()
if self.debug:
qualifications.add(NumberHitsApprovedRequirement('GreaterThanOrEqualTo', 0))
qualifications.add(PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 0))
else:
qualifications.add(NumberHitsApprovedRequirement('GreaterThanOrEqualTo',
configuration['MTURK_NUMBER_HITS_APPROVED_REQUIREMENT']))
qualifications.add(PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo',
configuration[
'MTURK_PERCENT_ASSIGNMENTS_APPROVED_REQUIREMENT']))
hit_type = self.connection.register_hit_type(configuration['MTURK_TITLE'],
configuration['MTURK_DESCRIPTION'],
Price(configuration['MTURK_REWARD']),
configuration['MTURK_ASSIGNMENT_DURATION_IN_SECONDS'],
configuration['MTURK_KEYWORDS'],
configuration['MTURK_AUTO_APPROVAL_DELAY_IN_SECONDS'],
qualifications)
return hit_type[0].HITTypeId
def test_locale_qualification_in_notin(self):
self.set_http_response(status_code=200, body=MOCK_SERVER_RESPONSE)
q = ExternalQuestion(external_url="http://samplesite",
frame_height=800)
keywords = ['boto', 'test', 'doctest']
title = "Boto External Question Test"
annotation = 'An annotation from boto external question test'
qualifications = Qualifications()
test_requirement1 = LocaleRequirement(comparator='In',
locale=[('US', 'CA')])
test_requirement2 = LocaleRequirement(comparator='NotIn',
locale=[('US', 'WA'), 'CA'])
qualifications.add(test_requirement1)
qualifications.add(test_requirement2)
create_hit_rs = self.service_connection.create_hit(
question=q,
lifetime=60 * 65,
max_assignments=2,
title=title,
keywords=keywords,
reward=0.05,
duration=60 * 6,
approval_delay=60 * 60,
annotation=annotation,
qualifications=qualifications)
self.assert_request_parameters(
{
'QualificationRequirement.1.Comparator':
'In',
'QualificationRequirement.1.LocaleValue.1.Country':
'US',
'QualificationRequirement.1.LocaleValue.1.Subdivision':
'CA',
'QualificationRequirement.1.QualificationTypeId':
'00000000000000000071',
'QualificationRequirement.2.Comparator':
'NotIn',
'QualificationRequirement.2.LocaleValue.1.Country':
'US',
'QualificationRequirement.2.LocaleValue.1.Subdivision':
'WA',
'QualificationRequirement.2.LocaleValue.2.Country':
'CA',
'QualificationRequirement.2.QualificationTypeId':
'00000000000000000071'
},
ignore_params_values=[
'AWSAccessKeyId', 'SignatureVersion', 'Timestamp', 'Title',
'Question', 'AssignmentDurationInSeconds',
'RequesterAnnotation', 'Version', 'LifetimeInSeconds',
'AutoApprovalDelayInSeconds', 'Reward.1.Amount', 'Description',
'MaxAssignments', 'Reward.1.CurrencyCode', 'Keywords',
'Operation'
])
self.assertEquals(create_hit_rs.status, True)
def test_locale_qualification_notin_in(self):
self.set_http_response(status_code=200, body=MOCK_SERVER_RESPONSE)
q = ExternalQuestion(external_url="http://samplesite", frame_height=800)
keywords = ["boto", "test", "doctest"]
title = "Boto External Question Test"
annotation = "An annotation from boto external question test"
qualifications = Qualifications()
test_requirement1 = LocaleRequirement(comparator="NotIn", locale=[("US", "WA"), "CA"])
test_requirement2 = LocaleRequirement(comparator="In", locale=[("US", "CA")])
qualifications.add(test_requirement1)
qualifications.add(test_requirement2)
create_hit_rs = self.service_connection.create_hit(
question=q,
lifetime=60 * 65,
max_assignments=2,
title=title,
keywords=keywords,
reward=0.05,
duration=60 * 6,
approval_delay=60 * 60,
annotation=annotation,
qualifications=qualifications,
)
self.assert_request_parameters(
{
"QualificationRequirement.1.Comparator": "NotIn",
"QualificationRequirement.1.LocaleValue.1.Country": "US",
"QualificationRequirement.1.LocaleValue.1.Subdivision": "WA",
"QualificationRequirement.1.LocaleValue.2.Country": "CA",
"QualificationRequirement.1.QualificationTypeId": "00000000000000000071",
"QualificationRequirement.2.Comparator": "In",
"QualificationRequirement.2.LocaleValue.1.Country": "US",
"QualificationRequirement.2.LocaleValue.1.Subdivision": "CA",
"QualificationRequirement.2.QualificationTypeId": "00000000000000000071",
},
ignore_params_values=[
"AWSAccessKeyId",
"SignatureVersion",
"Timestamp",
"Title",
"Question",
"AssignmentDurationInSeconds",
"RequesterAnnotation",
"Version",
"LifetimeInSeconds",
"AutoApprovalDelayInSeconds",
"Reward.1.Amount",
"Description",
"MaxAssignments",
"Reward.1.CurrencyCode",
"Keywords",
"Operation",
],
)
self.assertEquals(create_hit_rs.status, True)
def test_qualification_qualtypewithscore_in(self):
self.set_http_response(
status_code=200,
body=MOCK_SERVER_RESPONSE)
q = ExternalQuestion(
external_url="http://samplesite",
frame_height=800)
keywords = ['boto', 'test', 'doctest']
title = "Boto External Question Test"
annotation = 'An annotation from boto external question test'
qualifications = Qualifications()
test_requirement = Requirement(
qualification_type_id=QUAL_WITH_SCORE_ID,
comparator='In',
integer_value=[100, 90, 80])
qualifications.add(test_requirement)
create_hit_rs = self.service_connection.create_hit(
question=q,
lifetime=60*65,
max_assignments=2,
title=title,
keywords=keywords,
reward=0.05,
duration=60*6,
approval_delay=60*60,
annotation=annotation,
qualifications=qualifications)
self.assert_request_parameters({
'QualificationRequirement.1.Comparator':
'In',
'QualificationRequirement.1.QualificationTypeId':
'333333333333333333333333333333',
'QualificationRequirement.1.IntegerValue.1':
100,
'QualificationRequirement.1.IntegerValue.2':
90,
'QualificationRequirement.1.IntegerValue.3':
80},
ignore_params_values=['AWSAccessKeyId',
'SignatureVersion',
'Timestamp',
'Title',
'Question',
'AssignmentDurationInSeconds',
'RequesterAnnotation',
'Version',
'LifetimeInSeconds',
'AutoApprovalDelayInSeconds',
'Reward.1.Amount',
'Description',
'MaxAssignments',
'Reward.1.CurrencyCode',
'Keywords',
'Operation'])
self.assertEquals(create_hit_rs.status, True)
def create_campaign_hits(campaignid):
"""
Create HITs for a campaign.
"""
campaign = Campaign.query.filter_by(id=campaignid).first()
options = campaign.options
terms = campaign.terms
# Determine the number of quizzes that need to be generated
numquizzes = int(math.ceil(float(len(terms)) / campaign.terms_per_quiz))
# A set of questions will be built for each "page"
for pagenum in range(numquizzes):
conn = create_connection()
question_form = create_question_form(campaign.title, campaign.title,"categorization")
# Add the term questions to the page / quiz
for i in range(campaign.terms_per_quiz):
termnum = i + (pagenum * campaign.terms_per_quiz)
if termnum < len(terms):
term = terms[termnum]
# Build up an answer dictionary for use with the question form
answers = [(cgi.escape(option.option_text),
("%s|%s" % (option.id, cgi.escape(option.option_text))))
for option in options]
# A question should be appended for each term under consideration
questiontext = campaign.question.replace("[term]", cgi.escape(term.term))
identifier = "%s|%s|%s" % (campaign.id, term.id, cgi.escape(term.term))
question_form.append(create_question(identifier,
questiontext,
answers))
# Create a locale requirement that answerers must be in the US
qualifications = Qualifications()
lr = LocaleRequirement("EqualTo","US")
qualifications.add(lr)
print lr.get_as_params()
# Now that the questions have been added, create the HIT
conn.create_hit(questions=question_form,
max_assignments=campaign.times_per_term,
title=campaign.title,
description=campaign.title,
duration=60*30, # allot 30 minutes per quiz
qualifications=qualifications, # require the answerers be in the US
reward=campaign.reward_per_quiz)
# Update the campaign to prevent multiple generation attempts
campaign.job_generated = True
db.session.add(campaign)
db.session.commit()
def getHITType():
# Changing this will add another hit type and might mess up later fetches...
# Only change if you know what you are doing...
_mtc = MTurkConnection(host=_host)
_title = "Select Person-Object Interactions in Images"
_description = "Please click on all the objects (or other people) that the highlighted person is interacting with."
_reward = _mtc.get_price_as_price(0.10)
_duration = 60 * 10
_keywords = "person, people, image, images, object, objects, actions, interactions"
_approval_delay = 60 * 60 * 24 * 5
_qualifications = Qualifications()
_qualifications.add(
PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo',
98,
required_to_preview=True))
_qualifications.add(
NumberHitsApprovedRequirement('GreaterThanOrEqualTo',
100,
required_to_preview=True))
_qualifications.add(
LocaleRequirement('EqualTo', 'US', required_to_preview=True))
return _mtc.register_hit_type(title=_title,
description=_description,
reward=_reward,
duration=_duration,
keywords=_keywords,
approval_delay=_approval_delay,
qual_req=_qualifications)
def getHITType():
# Changing this will add another hit type and might mess up later fetches...
# Only change if you know what you are doing...
_mtc = MTurkConnection(host=_host)
_title = "Guess the Closest Part of a Person!"
_description = "Help us find out which body part of a person is closest to the camera that took the picture."
_keywords = "person, people, image, images, object, objects, depth, comparisons, human3.6m"
_reward = _mtc.get_price_as_price(0.1)
_duration = 60 * 15
_approval_delay = 60 * 60 * 24 * 10
_qualifications = Qualifications()
_qualifications.add(
PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo',
98,
required_to_preview=True))
_qualifications.add(
NumberHitsApprovedRequirement('GreaterThanOrEqualTo',
100,
required_to_preview=True))
_qualifications.add(
LocaleRequirement('EqualTo', 'US', required_to_preview=True))
return _mtc.register_hit_type(title=_title,
description=_description,
reward=_reward,
duration=_duration,
keywords=_keywords,
approval_delay=_approval_delay,
qual_req=_qualifications)
def build_hit_qualifications(self, approve_requirement, restrict_to_usa,
blacklist):
"""Translate restrictions/qualifications to boto Qualifications objects
@blacklist is a list of names for Qualifications workers must
not already hold in order to see and accept the HIT.
"""
quals = Qualifications()
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if restrict_to_usa:
quals.add(LocaleRequirement("EqualTo", "US"))
if blacklist is not None:
for item in blacklist:
qtype = self.get_qualification_type_by_name(item)
if qtype:
quals.add(
Requirement(qtype['id'],
"DoesNotExist",
required_to_preview=True))
return quals
def build_qualifications(locales=None):
"""
Creates a single qualification that workers have a > 95% acceptance rate.
:return: boto qualification obj.
"""
qualifications = Qualifications()
requirements = [
PercentAssignmentsApprovedRequirement(comparator="GreaterThan",
integer_value="95")
]
if locales:
loc_req = LocaleRequirement(comparator='In', locale=locales)
requirements.append(loc_req)
_ = [qualifications.add(req) for req in requirements]
return qualifications
class Production(Config):
DEBUG = False
DEVELOPMENT = False
TESTING = False
MTURK_HOST = 'mechanicalturk.amazonaws.com'
MTURK_EXTERNAL_SUBMIT = 'https://www.mturk.com/mturk/externalSubmit'
QUALIFICATIONS = Qualifications([
NumberHitsApprovedRequirement('GreaterThan', 500),
PercentAssignmentsApprovedRequirement('GreaterThan', 95),
LocaleRequirement('EqualTo', 'US')
])
CELERYBEAT_SCHEDULE = {
'delete_temp_files': {
'task': 'delete_temp_files',
'schedule': timedelta(seconds=86400),
'args': ()
},
'run_gather': {
'task': 'run_gather',
'schedule': timedelta(seconds=60),
'args': ()
},
}
TACKBP_NW_09_CORPUS_URL = 'https://s3-us-west-2.amazonaws.com/tac-kbp-2009/sentences.meta-sentencesonly-no-liu-et-al-naacl2016-test-set-270k'
def post_HIT(task, trial):
amount = 0.10
# Start the verify task
url = "https://cs279-final-project.herokuapp.com/%s?trial=%s" % (task,
trial)
questionform = ExternalQuestion(url, 1200)
create_hit_result = connection.create_hit(
title="Help locate things in Google Street View — one question only!",
description=
"Participate in a short study to find things in Google Street View",
keywords=["find", "locate", "quick"],
#duration is in seconds
duration=60 * 5,
#max_assignments will set the amount of independent copies of the task (turkers can only see one)
max_assignments=TASK_LIMIT,
question=questionform,
reward=Price(amount),
#Determines information returned by method in API, not super important
response_groups=('Minimal', 'HITDetail'),
qualifications=Qualifications(),
)
# The response included several fields that will be helpful later
hit_type_id = create_hit_result[0].HITTypeId
hit_id = create_hit_result[0].HITId
print "Your HIT has been created. You can see it at this link:"
print "https://workersandbox.mturk.com/mturk/preview?groupId={}".format(
hit_type_id)
print "Your HIT ID is: {}".format(hit_id)
return
def get_qualifications(self, project, boomerang_threshold, add_boomerang):
requirements = []
if project.qualification is not None:
requirements += self._mturk_system_qualifications(project.qualification)
boomerang_qual, success = self.create_qualification_type(owner_id=project.owner_id,
project_id=project.group_id,
name='Boomerang Score #{}'.format(project.group_id),
flag=FLAG_Q_BOOMERANG,
description='No description available')
boomerang = None
if boomerang_threshold <= int(settings.BOOMERANG_MIDPOINT * 100):
for i, bucket in enumerate(WAIT_LIST_BUCKETS):
if int(bucket[1] * 100) <= boomerang_threshold:
boomerang_blacklist, success = \
self.create_qualification_type(owner_id=project.owner_id,
name='Boomerang Waitlist #{}-{}'.format(project.group_id, len(
WAIT_LIST_BUCKETS) - i),
flag=FLAG_Q_BOOMERANG,
description='No description available',
deny=True,
project_id=project.group_id,
bucket=bucket)
if success and add_boomerang:
boomerang = BoomerangRequirement(qualification_type_id=boomerang_blacklist.type_id,
comparator=OP_DNE,
integer_value=None)
requirements.append(boomerang)
else:
boomerang = BoomerangRequirement(qualification_type_id=boomerang_qual.type_id, comparator=OP_GTEQ,
integer_value=boomerang_threshold)
if success and add_boomerang:
requirements.append(boomerang)
return Qualifications(requirements), boomerang_qual
def create_session_hit_type(session):
conn = get_mt_connection(session)
keywords = session.task_def.get_keywords()
t = session.task_def
qualifications = Qualifications()
add_session_qualifications(qualifications, session)
#qualifications.add(PercentAssignmentsApprovedRequirement(comparator="GreaterThan", integer_value="80"))
print qualifications
create_hit_rs = conn.register_hit_type(title=t.title,
description=t.description,
keywords=str(t.keywords),
reward=t.reward,
duration=t.duration,
approval_delay=t.approval_delay,
qual_req=qualifications)
print str(create_hit_rs)
if (create_hit_rs.status != True):
raise MTurkException(create_hit_rs)
print "Created HIT Type", create_hit_rs.HITTypeId
hit_type_id = create_hit_rs.HITTypeId
return hit_type_id
def update_hit(settings):
global hit
hit_quals = Qualifications()
old_hit_type_id = hit["type_id"]
settings_quals = settings["qualifications"]
## TODO: master worker, custom quals, utility for creating qualifications?
if (settings_quals):
if settings_quals["location"]:
hit_quals.add(LocaleRequirement("EqualTo", settings_quals["location"]))
if settings_quals["approval_percentage"]:
hit_quals.add(PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
settings_quals["approval_percentage"]))
request_settings = dict(
title = settings["title"],
description = settings["description"],
keywords = settings["keywords"],
reward = settings["reward"],
approval_delay = timedelta(seconds = settings["auto_approval_delay"]),
duration = timedelta(seconds = settings["assignment_duration"]),
qual_req = hit_quals
)
try:
new_hit_type_id = mtc.register_hit_type(**request_settings)[0].HITTypeId
if new_hit_type_id == old_hit_type_id:
prints("Settings haven't changed; not updating")
sys.exit(1)
change_existing_hit_type_result = mtc.change_hit_type_of_hit(hit["id"], new_hit_type_id)
except MTurkRequestError as e:
print("Error\n")
pp(e.__dict__)
hit_modes[mode]["type_id"] = new_hit_type_id
## write new_hit_type_id to hit_modes.json
with open("hit_modes.json", 'w') as f:
json.dump(hit_modes, f, indent=4, separators=(',', ': '))
prints("Updated from type:",
old_hit_type_id,
"to type:",
new_hit_type_id)
def create_hit(hit_options):
""" Create a new HIT on AMT.
`hit_options` is a dictionary that can contain:
* `title`: The title that will show up in AMT's HIT listings
* `description`: The description that will show up in AMT's HIT listings
* `reward`: A float containing the number of cents to pay for each
assignment
* `duration`: The expected amount of time a worker should spend on each
assignment, in minutes
* `num_responses`: The number of responses to get for the HIT
* `frame_height`: The height of the iframe in which workers will see the
assignment
* `use_https`: whether or not to load assignment in AMT's iframe using
HTTPS. Strongly recommended to be True
By default, options are loaded from `settings.AMT_DEFAULT_HIT_OPTIONS`.
"""
options = settings.AMT_DEFAULT_HIT_OPTIONS
options.update(hit_options)
scheme = 'https' if options['use_https'] else 'http'
from interface import AMT_INTERFACE
path = AMT_INTERFACE.get_assignment_url()
url = (scheme + '://' + settings.PUBLIC_IP + ':8000' +
path if settings.HAVE_PUBLIC_IP else scheme + '://' +
settings.AMT_CALLBACK_HOST + path)
question = ExternalQuestion(external_url=url,
frame_height=options['frame_height'])
qualifications = Qualifications(requirements=[
PercentAssignmentsApprovedRequirement(
'GreaterThanOrEqualTo', 85, required_to_preview=True),
])
conn = get_amt_connection(options['sandbox'])
try:
create_response = conn.create_hit(
question=question,
title=options['title'],
description=options['description'],
reward=Price(amount=options['reward']),
duration=timedelta(minutes=options['duration']),
max_assignments=options['num_responses'],
approval_delay=3600,
qualifications=qualifications)
except MTurkRequestError:
logger.debug(traceback.format_exc())
raise AMTException("""
Could not reach Amazon Mechanical Turk.
Check that you are using https mode, and defined a valid assignment.
Details of the exception have been logged to the ampcrowd server.
""")
return create_response[0].HITId
def publish_hit(credentials, isSandbox, ename, num_hits, lvlset, adminToken,
db, mode, no_ifs, individual, with_new_var_powerup, mc=None,
email=None, maxlvls=None, colSwap=False, replay=False):
title = "Play a Math Puzzle Game For Science!"
max_assignments = 1
keywords = "game, math, programming"
description = \
""" NOTE: PLEASE DON'T ACCEPT MORE THAN ONE HIT AT A TIME! First finish your current HIT before looking for others!\n """ + \
"""Help us evaluate our verification game InvGen! Each level is structured as a math puzzle, where you try to come up with correct expressions. Your work directly helps with the verification of programs! This HIT involves playing at least two non-tutorial levels of our game. Played it before? Come play again! You will bypass the tutorial and get new levels! New player? Come try it out! We aim to pay about $10/hr. More specifically: (a) $1.50 for the HIT, which involves playing the game for at least 2 non-tutorial levels (b) $1.50 bonus for doing the tutorial, which you only do the first time (c) $0.75 bonus for each non-tutorial level you pass beyond two."""
mastersQualType = "2ARFPLSP75KLA8M8DH1HTEQVJT3SY6" if isSandbox else \
"2F1QJWKUDD8XADTFD2Q0G6UTO95ALH"
quals = [] if isSandbox else\
[
NumberHitsApprovedRequirement("GreaterThanOrEqualTo", 1000),
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo", 97),
Requirement(mastersQualType, "Exists")
]
if mc is None:
mc = connect(credentials, isSandbox)
balance = mc.get_account_balance()
print("Balance:", balance[0])
exp = Experiment(ename, True)
print("Running under experiment", ename)
for i in range(num_hits):
port = get_unused_port()
srid = exp.create_unique_server_run_id()
p = start_server(port, ename, srid, lvlset, adminToken, db, email,
maxlvls, colSwap, replay)
print("Started server run", srid, "on port", port, "with pid", p.pid)
start_url =\
"https://zoidberg.ucsd.edu:{0}/mturk_landing.html?mode=" + mode
if (no_ifs):
start_url += "&noifs"
if individual:
start_url += "&individual=1"
if with_new_var_powerup:
start_url += "&nvpower=1"
q = ExternalQuestion(start_url.format(port), 1024)
kind = "ExternalQuestion"
r = mc.create_hit(question=q,
lifetime=timedelta(7),
max_assignments=max_assignments,
title=title,
description=description,
keywords=keywords,
reward=HIT_REWARD,
duration=timedelta(0, 45*60),
qualifications=Qualifications(quals))
assert len(r) == 1
print("Created", kind, "HIT", r[0].HITId)
exp.add_session(ServerRun(srid, r[0].HITId, p.pid, port))
return exp
def __init__(self, sandbox):
if sandbox == '0':
mturk_host = settings.MTURK_REST_ENDPOINT
else:
mturk_host = settings.MTURK_REST_SANDBOX_ENDPOINT
self.mturk_connection = MTurkConnection(
aws_access_key_id=settings.AWS_ACCESS_KEY,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
host=mturk_host)
quals = Qualifications()
if settings.MTURK_APPROVED_PERCENT_REQUIREMENT > 0:
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
settings.MTURK_APPROVED_PERCENT_REQUIREMENT))
if settings.MTURK_USONLY_REQUIREMENT:
quals.add( LocaleRequirement("EqualTo", "US"))
if settings.MTURK_MASTERS_REQUIREMENT:
quals.add(Requirement('2ARFPLSP75KLA8M8DH1HTEQVJT3SY6' if sandbox!='0' \
else '2F1QJWKUDD8XADTFD2Q0G6UTO95ALH', "Exists"))
self.paramdict = dict(
hit_type=None,
lifetime=datetime.timedelta(hours=settings.MTURK_HIT_LIFETIME_HOURS),
max_assignments=settings.MTURK_MAX_ASSIGNMENTS,
title = settings.MTURK_GAME_TITLE,
description = settings.MTURK_GAME_DESCRIPTION,
keywords = 'search',
reward = settings.MTURK_HIT_REWARD,
duration = datetime.timedelta(hours=settings.MTURK_HIT_DURATION),
approval_delay = datetime.timedelta(hours=settings.MTURK_APPROVAL_DELAY),
questions = None,
qualifications = quals
)
def create_hit(url=None,
title=None,
description=None,
keywords=None,
reward_amount=None,
max_assignments=None,
duration_in_minutes=None,
lifetime_in_days=None,
approval_delay_in_days=None,
qualification_percent_approved=None,
qualification_hits_approved=None):
url = url or "https://squadtest.herokuapp.com/"
title = title or "Compare 100 sets of 2 Instagram posts to guess which performed better (<10 minutes)"
description = description or "This HIT will take at most 15 minutes (usually much less). If you have a problem with the HIT, message us so we can fix it!"
keywords = keywords or [
"easy", "survey", "study", "bonus", "image", "images", "compare",
"comparisons", "collection", "data", "research", "listings", "simple",
"photo", "answer", "opinion", "question"
]
frame_height = 800
reward_amount = reward_amount or 1
max_assignments = max_assignments or 200
duration_in_minutes = duration_in_minutes or 20
duration = datetime.timedelta(minutes=duration_in_minutes)
lifetime_in_days = lifetime_in_days or 3
lifetime = datetime.timedelta(days=3)
approval_delay_in_days = approval_delay_in_days or 5
approval_delay = datetime.timedelta(days=approval_delay_in_days)
qualification_percent_approved = qualification_percent_approved or 95
q1 = PercentAssignmentsApprovedRequirement('GreaterThan',
qualification_percent_approved)
qualification_hits_approved = qualification_hits_approved or 500
q2 = NumberHitsApprovedRequirement('GreaterThan',
qualification_hits_approved)
qualifications = Qualifications([q1, q2])
questionform = ExternalQuestion(url, frame_height)
result = connection.create_hit(title=title,
description=description,
keywords=keywords,
max_assignments=max_assignments,
question=questionform,
reward=Price(amount=reward_amount),
response_groups=('Minimal', 'HITDetail',
'HITQuestion',
'HITAssignmentSummary'),
lifetime=lifetime,
duration=duration,
approval_delay=approval_delay,
qualifications=qualifications)
print result
def test_qualification_qualtypewithscore_in(self):
self.set_http_response(status_code=200, body=MOCK_SERVER_RESPONSE)
q = ExternalQuestion(external_url="http://samplesite",
frame_height=800)
keywords = ['boto', 'test', 'doctest']
title = "Boto External Question Test"
annotation = 'An annotation from boto external question test'
qualifications = Qualifications()
test_requirement = Requirement(
qualification_type_id=QUAL_WITH_SCORE_ID,
comparator='In',
integer_value=[100, 90, 80])
qualifications.add(test_requirement)
create_hit_rs = self.service_connection.create_hit(
question=q,
lifetime=60 * 65,
max_assignments=2,
title=title,
keywords=keywords,
reward=0.05,
duration=60 * 6,
approval_delay=60 * 60,
annotation=annotation,
qualifications=qualifications)
self.assert_request_parameters(
{
'QualificationRequirement.1.Comparator': 'In',
'QualificationRequirement.1.QualificationTypeId':
'333333333333333333333333333333',
'QualificationRequirement.1.IntegerValue.1': 100,
'QualificationRequirement.1.IntegerValue.2': 90,
'QualificationRequirement.1.IntegerValue.3': 80
},
ignore_params_values=[
'AWSAccessKeyId', 'SignatureVersion', 'Timestamp', 'Title',
'Question', 'AssignmentDurationInSeconds',
'RequesterAnnotation', 'Version', 'LifetimeInSeconds',
'AutoApprovalDelayInSeconds', 'Reward.1.Amount', 'Description',
'MaxAssignments', 'Reward.1.CurrencyCode', 'Keywords',
'Operation'
])
self.assertEquals(create_hit_rs.status, True)
def get_qualifications(self):
requirements = []
approved_hits = NumberHitsApprovedRequirement('GreaterThan',
self.min_hits)
percentage_approved = PercentAssignmentsApprovedRequirement(
'GreaterThanOrEqualTo', 97)
locale = MultiLocaleRequirement('In', self.countries)
requirements.append(locale)
requirements.append(approved_hits)
requirements.append(percentage_approved)
return Qualifications(requirements)
def register_hit(self, configuration=None):
"""
Register a hit on Mechanical Turk according to `hit_params`. This will provide you with a HITTypeId.
Parameters
----------
configuration : dict
Returns
-------
str
The HITTypeId which is how you refer to your newly registered hit with Amazon
"""
if configuration is None:
configuration = app.config
qualifications = Qualifications()
if self.debug:
qualifications.add(
NumberHitsApprovedRequirement('GreaterThanOrEqualTo', 0))
qualifications.add(
PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo',
0))
else:
qualifications.add(
NumberHitsApprovedRequirement(
'GreaterThanOrEqualTo',
configuration['MTURK_NUMBER_HITS_APPROVED_REQUIREMENT']))
qualifications.add(
PercentAssignmentsApprovedRequirement(
'GreaterThanOrEqualTo', configuration[
'MTURK_PERCENT_ASSIGNMENTS_APPROVED_REQUIREMENT']))
hit_type = self.connection.register_hit_type(
configuration['MTURK_TITLE'], configuration['MTURK_DESCRIPTION'],
Price(configuration['MTURK_REWARD']),
configuration['MTURK_ASSIGNMENT_DURATION_IN_SECONDS'],
configuration['MTURK_KEYWORDS'],
configuration['MTURK_AUTO_APPROVAL_DELAY_IN_SECONDS'],
qualifications)
return hit_type[0].HITTypeId
def __init__(self):
#connect to MTurk
self.connect = MTurkConnection(self.AWS_ACCESS_KEY_ID,
self.AWS_SECRET_ACCESS_KEY,
host=self.HOST_SANDBOX)
#Qualification setting
q = self.qualifications = Qualifications()
# if required_to_preview == True unqualified user even can't view the hit.
# q.add( PercentAssignmentsApprovedRequirement( comparator="GreaterThan", integer_value="95" ) )
q.add(AdultRequirement(comparator="EqualTo", integer_value="1"))
def configure_hit(self):
# Configure portal
experimentPortalURL = self.config.get('HIT Configuration', 'question_url')
frameheight = 600
mturkQuestion = ExternalQuestion(experimentPortalURL, frameheight)
# Qualification:
quals = Qualifications()
approve_requirement = self.config.get('HIT Configuration', 'Approve_Requirement')
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if self.config.getboolean('HIT Configuration', 'US_only'):
quals.add(LocaleRequirement("EqualTo", "US"))
# Specify all the HIT parameters
self.paramdict = dict(
hit_type = None,
question = mturkQuestion,
lifetime = datetime.timedelta(hours=self.config.getfloat('HIT Configuration', 'HIT_lifetime')),
max_assignments = self.config.getint('HIT Configuration', 'max_assignments'),
title = self.config.get('HIT Configuration', 'title'),
description = self.config.get('HIT Configuration', 'description'),
keywords = self.config.get('HIT Configuration', 'keywords'),
reward = self.config.getfloat('HIT Configuration', 'reward'),
duration = datetime.timedelta(hours=self.config.getfloat('HIT Configuration', 'duration')),
approval_delay = None,
questions = None,
qualifications = quals
)
def configure_hit(self, hit_config):
# configure question_url based on the id
experimentPortalURL = hit_config['ad_location']
frameheight = 600
mturkQuestion = ExternalQuestion(experimentPortalURL, frameheight)
# Qualification:
quals = Qualifications()
approve_requirement = hit_config['approve_requirement']
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if hit_config['us_only']:
quals.add(LocaleRequirement("EqualTo", "US"))
# Specify all the HIT parameters
self.paramdict = dict(hit_type=None,
question=mturkQuestion,
lifetime=hit_config['lifetime'],
max_assignments=hit_config['max_assignments'],
title=hit_config['title'],
description=hit_config['description'],
keywords=hit_config['keywords'],
reward=hit_config['reward'],
duration=hit_config['duration'],
approval_delay=None,
questions=None,
qualifications=quals)
def configure_hit(self, hit_config):
''' Configure HIT '''
# configure question_url based on the id
experiment_portal_url = hit_config['ad_location']
frame_height = 600
mturk_question = ExternalQuestion(experiment_portal_url, frame_height)
# Qualification:
quals = Qualifications()
approve_requirement = hit_config['approve_requirement']
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if hit_config['us_only']:
quals.add(LocaleRequirement("EqualTo", "US"))
# Create a HIT type for this HIT.
hit_type = self.mtc.register_hit_type(hit_config['title'],
hit_config['description'],
hit_config['reward'],
hit_config['duration'],
keywords=hit_config['keywords'],
approval_delay=None,
qual_req=quals)[0]
# Check the config file to see if notifications are wanted.
config = PsiturkConfig()
config.load_config()
try:
url = config.get('Server Parameters', 'notification_url')
all_event_types = [
"AssignmentAccepted",
"AssignmentAbandoned",
"AssignmentReturned",
"AssignmentSubmitted",
"HITReviewable",
"HITExpired",
]
self.mtc.set_rest_notification(hit_type.HITTypeId,
url,
event_types=all_event_types)
except:
pass
# Specify all the HIT parameters
self.param_dict = dict(hit_type=hit_type.HITTypeId,
question=mturk_question,
lifetime=hit_config['lifetime'],
max_assignments=hit_config['max_assignments'],
questions=None,
response_groups=[
'Minimal', 'HITDetail', 'HITQuestion',
'HITAssignmentSummary'
])
def createQualifications(sandbox, qualificationIDs):
qualifications = Qualifications()
for qualification_id in qualificationIDs[:8]:
qualifications.add(
Requirement(qualification_id,
'DoesNotExist',
required_to_preview=True))
qualifications.add(
PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 90,
True))
qualifications.add(
NumberHitsApprovedRequirement('GreaterThanOrEqualTo', 100, True)) #100
return qualifications
def __init__(self, host, n=10):
mturkparams = dict(
aws_access_key_id = config.get( 'AWS Access', 'aws_access_key_id' ),
aws_secret_access_key = config.get( 'AWS Access', 'aws_secret_access_key' ),
host=host)
self.mtc = MTurkConnection( **mturkparams )
# Configure portal
experimentPortalURL = config.get( 'HIT Configuration', 'question_url' )
frameheight = 600
mturkQuestion = ExternalQuestion( experimentPortalURL, frameheight )
# Qualification:
quals = Qualifications();
approve_requirement = config.getint('HIT Configuration',
'Approve_Requirement')
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if config.getboolean('HIT Configuration', 'US_only'):
quals.add( LocaleRequirement("EqualTo", "US") )
# Specify all the HIT parameters
self.paramdict = dict(
hit_type = None,
question = mturkQuestion,
lifetime = datetime.timedelta(hours=config.getfloat('HIT Configuration', 'HIT_lifetime')),
max_assignments = config.getint('HIT Configuration', 'max_assignments'),
title = config.get('HIT Configuration', 'title'),
description = config.get('HIT Configuration', 'description'),
keywords = config.get('HIT Configuration', 'keywords'),
reward = config.getfloat('HIT Configuration', 'reward'),
duration = datetime.timedelta(
hours=config.getfloat('HIT Configuration', 'duration')),
approval_delay = None,
questions = None,
qualifications = quals
)
def createHIT():
# TODO: fix height here
url = 'https://www.zeerak.net/279akz/getTask/'
external_question = ExternalQuestion(url, 1200)
qualifications = Qualifications()
qualifications.add(LocaleRequirement("EqualTo", "US"))
response = mtc.create_hit(
title='Read a short article and learn something new!',
keywords='read, react, share, news',
description='How would you share this on social media? Just one task!',
duration=1800,
question=external_question,
reward=0.30,
max_assignments=145,
qualifications=qualifications,
# response_groups='', # batches??
)
# The response included several fields that will be helpful later
hit_type_Id = response[0].HITTypeId
hit_Id = response[0].HITId
available_balance = mtc.get_account_balance()
print("-------------------------------------------------------")
print("Your HIT has been created. You can see it at this link:")
print(">>>>>> https://workersandbox.mturk.com/mturk/preview?groupId={}".
format(hit_type_Id))
print("Your HIT Id is: {}".format(hit_Id))
print("And see results here:")
print(">>>>>> {}".format(mturk_environment['manage']))
print("Your account balance is {}".format(available_balance))
print("-------------------------------------------------------")
return
def create_hit(task_type, URL, num_of_assignments):
title = title_set[task_type]
description = description_set[task_type]
keywords = keywords_set[task_type]
reward = price_set[task_type]
print mturk.get_account_balance()
# Qualification
qualifications = Qualifications()
qualifications.add(PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo", approve_requirement))
#qualifications.add(LocaleRequirement("EqualTo", "US"))
#qualifications.add(LocaleRequirement("In", ['US', 'GB', 'IN'], required_to_preview=True))
questionform = ExternalQuestion(URL, frame_height)
# create HIT
create_hit_result = mturk.create_hit(
title=title,
description=description,
keywords=keywords,
question=questionform,
reward=reward,
qualifications=qualifications,
duration=duration,
max_assignments=num_of_assignments,
lifetime=lifetime,
approval_delay=approval_delay
#response_groups = ( 'Minimal', 'HITDetail' ), # I don't know what response groups are
)
HIT = create_hit_result[0]
assert create_hit_result.status
print '[create_hit( %s, $%s ): %s]' % (URL, reward, HIT.HITId)
print HIT.HITId
return HIT.HITId
def createSeveralHits(numberOfSubjects):
# HOST = 'mechanicalturk.sandbox.amazonaws.com'
# HOST = 'mechanicalturk.amazonaws.com'
global connection
global layoutID
url = 'THE LINK TO THE STUDY'
title = "TITLE OF THE STUDY"
description = "DESCRIPTION OF THE STUDY"
keywords = ["KEYWORDS"]
amount = 0.4
# game = ExternalQuestion(url, frame_height)
remaining = numberOfSubjects
create_hit_result = None
timestr = time.strftime("%Y%m%d-%H%M%S")
myfile=open("./HITIDs/"+timestr + '.txt', 'w') # Saves all the created hits under HITIDS. The name of the file is based on the date and time of creation.
####Setting Worker Requirements:
workerRequirements=Qualifications()
req1= PercentAssignmentsApprovedRequirement(comparator = "GreaterThan", integer_value = "96",required_to_preview=True)
workerRequirements.add(req1)
req2= LocaleRequirement(comparator = "EqualTo",locale ="US",required_to_preview=True)
workerRequirements.add(req2)
req3=NumberHitsApprovedRequirement(comparator = "GreaterThan", integer_value = "50",required_to_preview=True)
workerRequirements.add(req3)
while remaining>0:
no_assignments = min(9, remaining)
create_hit_result = connection.create_hit(
title=title,
description=description,
keywords=keywords,
max_assignments=no_assignments,
hit_layout=layoutID,
lifetime=6*60*60,
duration=3*60*60,
approval_delay=3*60*60*24,
reward=Price(amount=amount),
qualifications=workerRequirements)
remaining -= no_assignments
myfile.write(create_hit_result[0].HITId+"\n")
print "No. Assignments: ", no_assignments
print "Results:", create_hit_result
print "hit ID:", create_hit_result[0].HITId
def build_hit_qualifications(self, approve_requirement, restrict_to_usa):
"""Translate restrictions/qualifications to boto Qualifications objects"""
quals = Qualifications()
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if restrict_to_usa:
quals.add(LocaleRequirement("EqualTo", "US"))
return quals
def get_requirement(qualification_type_id):
"""
Function to set the requirements. This is optional.
:return: list
"""
qualifications = Qualifications()
qualifications.add(
PercentAssignmentsApprovedRequirement(comparator="GreaterThan",
integer_value="90"))
qualifications.add(LocaleRequirement("EqualTo", "US"))
# qualifications.add(Requirement(qualification_type_id=qualification_type_id, comparator="EqualTo", integer_value="1"))
return qualifications
def create_hit_type(self, *args):
print "Creating HIT Type..."
#Set up qualifications
my_qual_req = Qualifications()
#my_qual_req.add(PercentAssignmentsApprovedRequirement("GreaterThan", 85))
#my_qual_req.add(NumberHitsApprovedRequirement("GreaterThan", 100))
#my_qual_req.add(LocaleRequirement("EqualTo", "US"))
#Create json object for storage to mongo
J = {
'title':
"Read a blog post or news article and answer questions about it",
'description':
"Read a short blog post or news article and answer questions about it. These are quick HITs, and there are going to be a lot of them. Bonuses for accurate answers!",
'reward': 0.06,
'duration': 3600,
'keywords': "blog, news, article, quick, easy, bonus, civility",
'approval_delay': 2592000,
'qual_req': [req.__dict__ for req in my_qual_req.requirements],
}
#Register HIT type on mongo
HT_id = self.mt_conn.register_hit_type(
title=J["title"],
description=J["description"],
reward=Price(J["reward"]),
duration=J["duration"],
keywords=J["keywords"],
approval_delay=J["approval_delay"],
qual_req=my_qual_req,
)
#Retrieve the HITTypeId
J["hittypeid"] = HT_id[0].HITTypeId
J["created_at"] = datetime.datetime.now()
# print json.dumps(J, cls=helpers.MongoEncoder, indent=2)
print "\tHIT Type created with ID", J["hittypeid"]
#Save the object to mongo
db_id = self.db_conn['cvm_mturk_hittypeid'].insert(J)
print "\tHIT Type stored to database with ID ", db_id
return 1
def get_qualifications(prod):
qualifications = Qualifications()
if args.prod:
qualifications.add(
PercentAssignmentsApprovedRequirement(comparator="GreaterThan",
integer_value=str(95)))
qualifications.add(
NumberHitsApprovedRequirement(comparator="GreaterThan",
integer_value=str(500)))
# these location constraints are potentially useful
# qualifications.add(LocaleRequirement(comparator="NotEqualTo", locale="your country code", required_to_preview=True))
# qualifications.add(LocaleRequirement(comparator="EqualTo", locale="your country code", required_to_preview=True))
return qualifications
def activate_hit(session, hit):
if session.standalone_mode:
return (True, "%s" % hit.ext_hitid)
taskurl = settings.HOST_NAME_FOR_MTURK + "mt/get_task/" + str(
session.code) + "/?extid=" + hit.ext_hitid
q = ExternalQuestion(external_url=taskurl, frame_height=800)
conn = get_mt_connection(session)
keywords = session.task_def.get_keywords()
t = session.task_def
if not session.hit_type:
qualifications = Qualifications()
add_session_qualifications(qualifications, session)
create_hit_rs = conn.create_hit(question=q,
lifetime=t.lifetime,
max_assignments=t.max_assignments,
title=t.title,
keywords=str(t.keywords),
reward=t.reward,
duration=t.duration,
approval_delay=t.approval_delay,
annotation="IGNORE",
qualifications=qualifications)
if create_hit_rs.status != True:
return (False, "Error talking to AWS: %s (%s)" %
(create_hit_rs.Message, create_hit_rs.Code))
try:
session.hit_type = create_hit_rs.HITTypeId
except Exception, e:
print str(e)
print str(create_hit_rs)
return (
False,
"Exception found while creating HIT: %s (AWS error): %s (%s)" %
(str(e), create_hit_rs.Message, create_hit_rs.Code))
session.save()
def createQualifications(sandbox):
qualifications = Qualifications()
if sandbox:
MastersQualID = '2F1KVCNHMVHV8E9PBUB2A4J79LU20F'
else:
MastersQualID = '2NDP2L92HECWY8NS8H3CK0CP5L9GHO'
#qualifications.add(Requirement(MastersQualID,'DoesNotExist',required_to_preview=True))
qualifications.add(
PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 90,
True))
qualifications.add(
NumberHitsApprovedRequirement('GreaterThanOrEqualTo', 100, True))
return qualifications
def generate_hits(subset, begin_index, args, data_ids, images_metainfo):
from boto.mturk.connection import MTurkConnection
from boto.mturk.question import QuestionContent, Question, QuestionForm, Overview, AnswerSpecification, SelectionAnswer, FormattedContent, FreeTextAnswer
from boto.mturk.qualification import PercentAssignmentsApprovedRequirement, Qualifications
ACCESS_ID = amazon_config.ACCESS_ID
SECRET_KEY = amazon_config.SECRET_KEY
HOST = 'mechanicalturk.amazonaws.com'
mtc = MTurkConnection(aws_access_key_id=ACCESS_ID,
aws_secret_access_key=SECRET_KEY,
host=HOST)
title = 'Give your opinion of interestingness level about images'
description = ('Watch images and give us your opinion of interestingness level about the images')
keywords = 'image, interestingness, interesting, rating, opinions'
ratings =[('Very boring','-2'),
('Boring','-1'),
('Neutral','0'),
('Interesting','1'),
('Very interesting, I would like to share it with my friends.','2')]
#--------------- BUILD OVERVIEW -------------------
overview = Overview()
overview.append_field('Title', 'Give your opinion about interestingness level on those images')
#--------------- BUILD QUESTIONs -------------------
questions = []
utils.write_file(subset, args.o + '.index_' + str(begin_index) + '.txt')
index = 0
for image_url in subset:
image_id = data_ids[index]
image = images_metainfo[image_id]
interestingness = 0
if ('repin_count' in image):
interestingness = int(image['repin_count']) + int(image['like_count'])
#else:
# interestingness = int(image['interestingness'])
index = index + 1
qc = QuestionContent()
context = ''
if (interestingness > 0):
context = ' (shared by ' + str(interestingness) + ' people)'
qc.append_field('Title','How interesting the image' + context + ' to you?')
qc.append(FormattedContent('<img src="' + image_url + '" alt="image" />'))
fta = SelectionAnswer(min=1, max=1,style='dropdown',
selections=ratings,
type='text',
other=False)
q = Question(identifier='interestingness',
content=qc,
answer_spec=AnswerSpecification(fta),
is_required=True)
questions.append(q)
#--------------- BUILD THE QUESTION FORM -------------------
question_form = QuestionForm()
question_form.append(overview)
for question in questions:
question_form.append(question)
# BUILD QUALIFICATION
qualifications = Qualifications()
req = PercentAssignmentsApprovedRequirement(comparator = "GreaterThan", integer_value = "95")
qualifications.add(req)
#--------------- CREATE THE HIT -------------------
mtc.create_hit(questions = question_form,
qualifications = qualifications,
max_assignments = 10,
title = title,
description = description,
keywords = keywords,
duration = 60 * 30,
reward = 0.2)
def generate_hits(mtc_type, subset, begin_index, args):
from boto.mturk.connection import MTurkConnection
from boto.mturk.question import QuestionContent, Question, QuestionForm, Overview, AnswerSpecification, SelectionAnswer, FormattedContent, FreeTextAnswer
from boto.mturk.qualification import PercentAssignmentsApprovedRequirement, Qualifications, Requirement
mtc = mtk_utils.get_mtc(mtc_type)
title = 'Give your opinion of aesthetics level about images'
description = ('View images and give us your opinion of aesthetics level about the images')
keywords = 'image, aesthetic, aesthetics, rating, opinions'
ratings =[('Very ugly','-2'),
('Ugly','-1'),
('Neutral','0'),
('Beautiful','1'),
('Very beautiful, I would like to take such beautiful photo too.','2')]
#--------------- BUILD OVERVIEW -------------------
overview = Overview()
overview.append_field('Title', 'Give your opinion about aesthetics level on those images')
#--------------- BUILD QUESTIONs -------------------
questions = []
if (args.m != 'qua'):
utils.write_file(subset, args.o + '.index_' + str(begin_index) + '.txt')
if (args.m == 'qua_init' and begin_index > 0):
return
for image_url in subset:
qc = QuestionContent()
qc.append_field('Title','How beautiful the image to you?')
qc.append(FormattedContent('<img src="' + image_url + '" alt="image" />'))
fta = SelectionAnswer(min=1, max=1,style='dropdown',
selections=ratings,
type='text',
other=False)
q = Question(identifier='aesthetics',
content=qc,
answer_spec=AnswerSpecification(fta),
is_required=True)
questions.append(q)
#--------------- BUILD THE QUESTION FORM -------------------
question_form = QuestionForm()
question_form.append(overview)
for question in questions:
question_form.append(question)
# BUILD QUALIFICATION
qualifications = Qualifications()
req = PercentAssignmentsApprovedRequirement(comparator = "GreaterThan", integer_value = "95")
qualifications.add(req)
if (args.m == 'qua'):
if (args.q != None):
qua_req = Requirement(qualification_type_id = args.q, comparator = 'EqualTo', integer_value = '1')
qualifications.add(qua_req)
else:
print("Please give qualification type id in 'qua' mode.")
sys.exit(0)
#--------------- CREATE THE HIT -------------------
hit = mtc.create_hit(questions = question_form,
qualifications = qualifications,
max_assignments = 10 * 2,
title = title,
description = description,
keywords = keywords,
duration = 60 * 30 * 2,
reward = 0.2 * 2)
if (args.m == 'qua_init'):
print("Create qualification type for HIT id: " + hit[0].HITId)
quatype = mtc.create_qualification_type(name = hit[0].HITId, description = "Temporary qualification for HIT " + hit[0].HITId, status = 'Active')
print("Qualification type id: " + quatype[0].QualificationTypeId)
ACCESS_ID ="AKIAIRZ5JZ2KFCLKU4MQ"
SECRET_KEY = "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"
#HOST = 'mechanicalturk.amazonaws.com'
HOST = 'mechanicalturk.sandbox.amazonaws.com'
mtc = MTurkConnection(aws_access_key_id=ACCESS_ID,
aws_secret_access_key=SECRET_KEY,
host=HOST)
title = 'Transcribe Palauan Dictionary entries'
description = ('Transcribe a column from Palauan-English dictionary' )
keywords = 'transcribe, data entry, google form, dictionary, palauan language'
#--------------- QUALIFICATIONS -------------------
quals = Qualifications()
quals.add(PercentAssignmentsApprovedRequirement(comparator="GreaterThan",integer_value=98))
quals.add(NumberHitsApprovedRequirement("GreaterThan",1000))
#--------------- BUILD QUESTION 1 -------------------
qc1 = QuestionContent()
qc1.append_field('Title','The Text from the Column')
fta1 = FreeTextAnswer(num_lines=40)
q1 = Question(identifier='first',
content=qc1,
answer_spec=AnswerSpecification(fta1),
is_required=True)
SANDBOX_HOST = 'mechanicalturk.sandbox.amazonaws.com'
mtc = MTurkConnection(host=SANDBOX_HOST)
if not runInSandbox:
mtc = MTurkConnection()
urlOfEvalScript += "?destination=production"
title = 'Help us evaluate creative stories'
description = 'In this HIT you will be shown a short story and asked to evaluate it.'
keywords = 'creativity,stories'
# Set qualifications
qualifications = Qualifications()
if not runInSandbox:
qualifications.add(PercentAssignmentsApprovedRequirement(comparator="GreaterThan", integer_value="95"))
qualifications.add(NumberHitsApprovedRequirement(comparator="GreaterThan", integer_value="500"))
qualifications.add(LocaleRequirement('EqualTo', 'US'))
#--------------- BUILD QUESTION 1 -------------------
q1 = ExternalQuestion(urlOfEvalScript, 600)
#--------------- CREATE THE HIT -------------------
mtc.create_hit(question=q1,
max_assignments=totalNumberOfAssignments,
title=title,
def create_hit(href, hit):
if hit.type == "MULTIPLE_URLS": # the max_workers was not set in this form
hit.max_workers = (
hit.size1
+ hit.size2
+ hit.size3
+ hit.size4
+ hit.size5
+ hit.size6
+ hit.size7
+ hit.size8
+ hit.size9
+ hit.size10
)
hostURL = SANDBOX_HOST if hit.sandbox else HOST
qualifications = Qualifications()
if hit.hit_approval > 0:
qualifications.add(NumberHitsApprovedRequirement("GreaterThan", hit.hit_approval, False))
if hit.hit_approval_rate > 0:
qualifications.add(PercentAssignmentsApprovedRequirement("GreaterThan", hit.hit_approval_rate, False))
if hit.accepted_hit_rate > 0:
qualifications.add(PercentAssignmentsSubmittedRequirement("GreaterThan", hit.accepted_hit_rate, False))
if hit.returned_hit_rate > 0:
qualifications.add(PercentAssignmentsReturnedRequirement("GreaterThan", hit.returned_hit_rate, False))
if hit.abandoned_hit_rate > 0:
qualifications.add(PercentAssignmentsAbandonedRequirement("LessThan", hit.abandoned_hit_rate, False))
if hit.rejected_hit_rate > 0:
qualifications.add(PercentAssignmentsRejectedRequirement("LessThan", hit.rejected_hit_rate, False))
if hit.locale_qualification is not None and hit.locale_qualification != "all":
qualifications.add(LocaleRequirement("EqualTo", hit.locale_qualification, False))
connection = MTurkConnection(
aws_access_key_id=hit.aws_access_key, aws_secret_access_key=hit.aws_secret_key, host=hostURL, is_secure=True
)
return Response(
connection.create_hit(
question=ExternalQuestion(href, hit.frame_height),
lifetime=hit.lifetime,
max_assignments=hit.max_workers,
title=hit.title,
description=hit.description,
keywords=["turktime"], # TODO
reward=hit.reward,
duration=hit.duration,
approval_delay=hit.approval_delay,
qualifications=qualifications,
response_groups=["Minimal", "HITDetail", "HITQuestion", "HITAssignmentSummary"],
)
)
config = Config()
AWS_ID = config.get('Credentials', 'aws_access_key_id', None)
SECRET_ID = config.get('Credentials', 'aws_secret_access_key_id', None)
HOST = 'mechanicalturk.amazonaws.com'
mt = MTurkConnection(
aws_access_key_id=AWS_ID,
aws_secret_access_key=SECRET_ID,
host=HOST
)
# HIT Configuration - global title, description, keywords, qualifications
TITLE = 'Provide Related Terms'
DESC = 'Given a word or phrase, provide another (different) word that relates to the given one.'
KEYWORDS = 'opinions, relations, idea, brainstorm, crowdstorm'
QUAL = Qualifications()
QUAL = QUAL.add(PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 75))
REWARD = 0.01
MAX_ASSN = BRANCHES
# HIT Overview
overview = Overview()
overview.append_field('Title', 'Tell us things or phrases that relate to a given phrase or picture')
# Build Question(s)
qc = QuestionContent()
qc.append_field('Title', 'Enter a word or phrase that relates to:')
qc.append_field('Text', PHRASE)
answer = FreeTextAnswer()
def createHIT(parameter, max_assignments=1):
quals = Qualifications();
quals.add(PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo", "97"))
quals.add(NumberHitsApprovedRequirement("GreaterThanOrEqualTo", "100"))
if not(useSandbox):
#quals.add(Requirement("2VNUKQOYGNIW2Y6UN6R177SVGBOZHD", "GreaterThanOrEqualTo", "24"))
#quals.add(Requirement("26C2D21O3W5XIBXBOI92RYEPLVQ9FH", "GreaterThanOrEqualTo", "24"))
quals.add(Requirement("22V6DG67D1X3WBT6Z2FK9JE1MAQ9GG", "GreaterThanOrEqualTo", "25"))
#f1 = 'hello1.txt'
#f2 = 'hello2.txt'
params = LayoutParameters()
params.add(LayoutParameter('image1', parameter))
#params.add(LayoutParameter('file2',f2))
#print "creating HIT"
retryDelay = 400
print "mtc create hit"
hitWasCreated = False
for repeat in range(0, 20):
print "repeat", repeat
try:
#if 1:
print "creating hit"
resultSet = mtc.create_hit(hit_layout=ZProcessLayoutID,
layout_params=params,
reward=0.01,
title="Does the dot stay inside the cell",
description="Does the dot stay inside the cell",
keywords="image, tag",
duration=timedelta(minutes=2),
lifetime=timedelta(minutes=60*48),
qualifications=quals,
max_assignments=max_assignments)
hitWasCreated = True
break
except Exception:
traceback.print_exc()
#if repeat == 19: raise Exception("messed up")
print "mechanical turk failed to create HIT, trying again in %d seconds" % retryDelay
time.sleep(retryDelay)
except:
traceback.print_exc()
#if repeat == 19: raise Exception("messed up")
print "mechanical turk failed to create HIT, trying again in %d seconds" % retryDelay
time.sleep(retryDelay)
else:
traceback.print_exc()
#if repeat == 19: raise Exception("messed up")
print "mechanical turk failed to create HIT, trying again in %d seconds" % retryDelay
time.sleep(retryDelay)
if not hitWasCreated:
# do this to generate the error message
resultSet = mtc.create_hit(hit_layout=ZProcessLayoutID,
layout_params=params,
reward=0.01,
title="Does the dot stay inside the cell",
description="Does the dot stay inside the cell",
keywords="image, tag",
duration=timedelta(minutes=1),
lifetime=timedelta(minutes=40),
qualifications=quals,
max_assignments=max_assignments)
sys.exit(1)
print "mtc create hit finished"
return resultSet
def generate_hits(mtc_type, subset, begin_index, args, data_ids, images_metainfo):
from boto.mturk.connection import MTurkConnection
from boto.mturk.question import QuestionContent, Question, QuestionForm, Overview, AnswerSpecification, SelectionAnswer, FormattedContent, FreeTextAnswer
from boto.mturk.qualification import PercentAssignmentsApprovedRequirement, Qualifications, Requirement
mtc = mtk_utils.get_mtc(mtc_type)
title = 'Tell us if you like those images or not'
description = ('View following images and tell us if you like them or not.')
keywords = 'image, like, interesting, rating, opinions'
ratings =[('Very hate it','-2'),
('Hate it','-1'),
('Neutral','0'),
('Like it','1'),
('Very like it.','2')]
#--------------- BUILD OVERVIEW -------------------
overview = Overview()
overview.append_field('Title', 'Tell us if you like those images or not.')
#--------------- BUILD QUESTIONs -------------------
questions = []
subset_with_pinterest = []
index = 0
for image_url in subset:
image_id = data_ids[index]
image = images_metainfo[image_id]
interestingness = 0
index = index + 1
if ('repin_count' in image):
interestingness = int(image['repin_count']) + int(image['like_count'])
subset_with_pinterest.append(image_url)
else:
continue
# interestingness = int(image['interestingness'])
qc = QuestionContent()
context = ''
if (interestingness > 0):
context = ' (shared by ' + str(interestingness) + ' people)'
qc.append_field('Title', str(interestingness) + ' people said they like following image, would you like it too?')
qc.append(FormattedContent('<img src="' + image_url + '" alt="image" />'))
fta = SelectionAnswer(min=1, max=1,style='dropdown',
selections=ratings,
type='text',
other=False)
q = Question(identifier='interestingness',
content=qc,
answer_spec=AnswerSpecification(fta),
is_required=True)
questions.append(q)
if (len(questions) == 0):
return
if (args.m != 'qua'):
utils.write_file(subset_with_pinterest, args.o + '.index_' + str(begin_index) + '.txt')
if (args.m == 'qua_init' and begin_index > 0):
return
#--------------- BUILD THE QUESTION FORM -------------------
question_form = QuestionForm()
question_form.append(overview)
for question in questions:
question_form.append(question)
# BUILD QUALIFICATION
qualifications = Qualifications()
req = PercentAssignmentsApprovedRequirement(comparator = "GreaterThan", integer_value = "95")
qualifications.add(req)
if (args.m == 'qua'):
if (args.q != None):
qua_req = Requirement(qualification_type_id = args.q, comparator = 'EqualTo', integer_value = '1')
qualifications.add(qua_req)
else:
print("Please give qualification type id in 'qua' mode.")
sys.exit(0)
#--------------- CREATE THE HIT -------------------
hit = mtc.create_hit(questions = question_form,
qualifications = qualifications,
max_assignments = 10 * 2,
title = title,
description = description,
keywords = keywords,
duration = 60 * 30 * 4,
reward = 0.2 * 2)
if (args.m == 'qua_init'):
print("Create qualification type for HIT id: " + hit[0].HITId)
quatype = mtc.create_qualification_type(name = hit[0].HITId, description = "Temporary qualification for HIT " + hit[0].HITId, status = 'Active')
print("Qualification type id: " + quatype[0].QualificationTypeId)
def configure_hit(self, hit_config):
''' Configure HIT '''
# configure question_url based on the id
experiment_portal_url = hit_config['ad_location']
frame_height = 600
mturk_question = ExternalQuestion(experiment_portal_url, frame_height)
# Qualification:
quals = Qualifications()
approve_requirement = hit_config['approve_requirement']
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if hit_config['us_only']:
quals.add(LocaleRequirement("EqualTo", "US"))
# Create a HIT type for this HIT.
hit_type = self.mtc.register_hit_type(
hit_config['title'],
hit_config['description'],
hit_config['reward'],
hit_config['duration'],
keywords=hit_config['keywords'],
approval_delay=None,
qual_req=None)[0]
# Check the config file to see if notifications are wanted.
config = PsiturkConfig()
config.load_config()
try:
url = config.get('Server Parameters', 'notification_url')
all_event_types = [
"AssignmentAccepted",
"AssignmentAbandoned",
"AssignmentReturned",
"AssignmentSubmitted",
"HITReviewable",
"HITExpired",
]
self.mtc.set_rest_notification(
hit_type.HITTypeId,
url,
event_types=all_event_types)
except:
pass
# Specify all the HIT parameters
self.param_dict = dict(
hit_type=hit_type.HITTypeId,
question=mturk_question,
lifetime=hit_config['lifetime'],
max_assignments=hit_config['max_assignments'],
title=hit_config['title'],
description=hit_config['description'],
keywords=hit_config['keywords'],
reward=hit_config['reward'],
duration=hit_config['duration'],
approval_delay=None,
questions=None,
qualifications=quals,
response_groups=[
'Minimal',
'HITDetail',
'HITQuestion',
'HITAssignmentSummary'
])
def send_hit(self):
# First check for qualifications
qualifications = Qualifications()
if self.qualification_adult == True:
qualifications.add(AdultRequirement("EqualTo", 1))
else:
qualifications.add(AdultRequirement("EqualTo",0))
if self.qualification_custom not in [None,""]:
qualifications.add(
Requirement(self.qualification_custom, self.qualification_custom_operator,
self.qualification_custom_value, required_to_preview = True)
)
if self.qualification_number_hits_approved != None:
qual_number_hits = NumberHitsApprovedRequirement("GreaterThan",self.qualification_number_hits_approved)
qualifications.add(qual_number_hits)
if self.qualification_percent_assignments_approved != None:
qual_perc_approved = PercentAssignmentsApprovedRequirement("GreaterThan",self.qualification_percent_assignments_approved)
qualifications.add(qual_perc_approved)
if self.qualification_locale != 'None':
qualifications.add(LocaleRequirement("EqualTo",self.qualification_locale))
# Domain name must be https
url = "%s/turk/%s" %(DOMAIN_NAME,self.id)
frame_height = 900
questionform = ExternalQuestion(url, frame_height)
if len(qualifications.requirements)>0:
result = self.connection.create_hit(title=self.title,
description=self.description,
keywords=self.keywords,
duration=datetime.timedelta(self.assignment_duration_in_hours/24.0),
lifetime=datetime.timedelta(self.lifetime_in_hours/24.0),
max_assignments=self.max_assignments,
question=questionform,
qualifications=qualifications,
reward=Price(amount=self.reward),
response_groups=('Minimal', 'HITDetail'))[0]
else:
result = self.connection.create_hit(title=self.title,
description=self.description,
keywords=self.keywords,
duration=datetime.timedelta(self.assignment_duration_in_hours/24.0),
lifetime=datetime.timedelta(self.lifetime_in_hours/24.0),
max_assignments=self.max_assignments,
question=questionform,
reward=Price(amount=self.reward),
response_groups=('Minimal', 'HITDetail'))[0]
# Update our hit object with the aws HIT
self.mturk_id = result.HITId
# When we generate the hit, we won't have any assignments to update
self.update(mturk_hit=result)
qualification_type_id=qual_type,
required_to_preview=required_to_preview,
)
# <codecell>
custom_qual_type = make_qual_type(
comparator="GreaterThanOrEqualTo",
integer_value=1,
qual_type=custom_id,
required_to_preview=False,
)
# <codecell>
qualifications = Qualifications()
qualifications.add(custom_qual_type)
# <codecell>
from collections import defaultdict
ONE_HOUR = 3600 # seconds
ONE_DAY = ONE_HOUR * 24
ONE_WEEK = ONE_HOUR * 24 * 7
hits = defaultdict(lambda : defaultdict())
# <codecell>
for url in urls:
def create_requirement(conn, workers, qualid, performance_thresh):
"""Returns an MTurk Qualification object for a list of workers which
can then be passed to a HIT object.
"""
# qualification type must already exist, for now.
# qual_id = '28EKH1Q6SVQD54NMWRXLEOBVCK22L4'
for worker in workers:
try:
conn.assign_qualification(qualid, worker, value=100, send_notification=False)
except Exception, e:
print 'Worker qualification already exists.'
req = Requirement(qualification_type_id=qualid, comparator='GreaterThan',
integer_value=performance_thresh)
qual = Qualifications()
qual.add(req)
return qual
from boto.mturk.connection import MTurkConnection
from mturkutils.base import parse_credentials_file
access_key_id, secretkey = parse_credentials_file(section_name='MTurkCredentials')
conn = MTurkConnection(aws_access_key_id=access_key_id,
aws_secret_access_key=secretkey)
workers = ['A3K3IZ0S0YOLO1',
'A2FV7F41LYJT25',
'A3AA5G6HENO6VJ',
'A26T3M57NK46L5',
'A3ND4KJK19EZ42',
'A1B9AYZBWJ0Z7Y',
'A3HIIRTGJUJZ1D',
def create_hit(settings):
global hit
## make sure there isn't already a hit
if (hit is not None):
sys.exit("Error: it looks like you already created the hit in %s mode (HIT ID stored in hit_modes.json)" % mode)
hit_quals = Qualifications()
settings_quals = settings["qualifications"]
## TODO: master worker, custom quals, utility for creating qualifications?
if (settings_quals):
if settings_quals["location"]:
hit_quals.add(LocaleRequirement("EqualTo", settings_quals["location"]))
if settings_quals["approval_percentage"]:
hit_quals.add(PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
settings_quals["approval_percentage"]))
prints(
"Your settings are:",
"",
dict_str(settings_raw)
)
if "http:" in settings["url"]:
sys.exit("Error: inside settings.json, your url is set to use 'http:'. It needs to use 'https:'")
## todo: some static analysis
if dialogue_mode=="verbose":
prints(
"",
"Are these settings okay? (yes/no)")
confirm_settings = raw_input("> ")
else:
confirm_settings = "yes"
if "n" in confirm_settings:
sys.exit()
if dialogue_mode=="verbose":
prints(
"",
"How many assignments do you want to start with?",
"(you can always add more later using cosub add)")
max_assignments = None
while max_assignments is None:
try:
max_assignments = int(raw_input("> "))
except ValueError:
prints("Couldn't understand answer. Try entering a positive integer (e.g., 20)")
else:
prints("You will start with " + str(defaultNSs) + " assignments.")
max_assignments = defaultNSs
if mode == "production":
reward = settings["reward"]
cost = max_assignments * float(reward) * 1.1
prints(
"The cost will be $%s ([%s assignments * $%s per assignment] + 10 percent fee)" % (cost, max_assignments, reward)
)
if dialogue_mode=="verbose":
prints("Is this okay? (yes/no)")
confirm_cost = raw_input("> ")
if "n" in confirm_cost:
sys.exit()
else:
print("(This won't cost anything because you're in sandbox mode)")
## TODO: implement bounds checking for assignments
if dialogue_mode=="verbose":
prints(
"",
"How long do you want to collect data for?",
"You can give an answer in seconds, minutes, hours, days, or weeks.",
"(and you can always add more time using cosub add)")
lifetime_seconds = None
while lifetime_seconds is None:
lifetime = raw_input("> ")
lifetime_seconds = timeparse(lifetime)
if not lifetime_seconds:
prints("Couldn't understand answer; try an answer in one of these formats:",
" 2 weeks",
" 3 days",
" 12 hours",
" 30 minutes")
else:
prints("You will collect data for " + defaultDuration + ".")
lifetime_seconds = timeparse(defaultDuration)
## TODO: implement bounds checking for time (30 seconds - 1 year)
prints("","Creating HIT...","")
request_settings = dict(
title = settings["title"],
description = settings["description"],
keywords = settings["keywords"],
question = question.ExternalQuestion(settings["url"], settings["frame_height"]),
max_assignments = max_assignments,
reward = settings["reward"],
approval_delay = timedelta(seconds = settings["auto_approval_delay"]),
duration = timedelta(seconds = settings["assignment_duration"]),
lifetime = timedelta(seconds = lifetime_seconds),
qualifications = hit_quals
)
try:
create_result = mtc.create_hit(**request_settings)[0]
except MTurkRequestError as e:
print("Error\n")
pp(e.__dict__)
sys.exit(1)
hit = {
"id": create_result.HITId,
# hit_group_id = hit.HITGroupId,
"type_id": create_result.HITTypeId
}
hit_modes[mode] = hit
print("Successfully created HIT")
## write hit and HITTypeId into even-odd.json
with open("hit_modes.json", 'w') as new_settings_file:
json.dump(hit_modes, new_settings_file, indent=4, separators=(',', ': '))
print("Wrote HIT ID and HIT Type ID to hit_modes.json")
prints(
"- The number of initial assignments is set to %s" % request_settings["max_assignments"],
"- The initial HIT lifetime is set to %s" % humane_timedelta(request_settings["lifetime"]))
prints(
"",
"Manage HIT: ",
HOST_requester + "/mturk/manageHIT?HITId=" + hit["id"])
prints(
"",
"View HIT: ",
HOST_worker + "/mturk/preview?groupId=" + hit["type_id"],
"")
logger.write({'Action': 'Create', 'Data': settings_raw })
q3 = Question(identifier="evaluation_feedback",
content=qc3,
answer_spec=AnswerSpecification(fta3))
#--------------- BUILD THE QUESTION FORM -------------------
question_form = QuestionForm()
question_form.append(overview)
question_form.append(q1)
question_form.append(q2)
question_form.append(q3)
#--------------- CREATE THE HIT -------------------
qualification = Qualifications()
qualification.add(LocaleRequirement('EqualTo','US'))
hitDetails = mtc.create_hit(questions=question_form,
max_assignments=5,
title=title,
description=description,
keywords=keywords,
duration = 60*60,
reward=0.1,
qualifications=qualification,
response_groups = ['Minimal'],
)
created_hits.append([count, hitDetails[0].HITId])
count += 1
def make_hit(image_url):
title = 'Label image with its location'
description = 'Answer questions about an image to label its location.'
keywords = 'image categorization, locations, scene recognition'
in_out = [('indoors','0'), ('outdoors','1')]
nat_manmade = [('man-made','0'), ('natural','1')]
functions = [('transportation/urban','0'),
('restaurant','1'),
('recreation','2'),
('domestic','3'),
('work/education','4'),
('other/unclear','5')]
landscapes = [('body of water/beach','0'),
('field','1'),
('mountain','2'),
('forest/jungle','3'),
('other/unclear','4')]
#--------------- BUILD OVERVIEW -------------------
overview = Overview()
overview.append_field('Title', title)
with open(INSTRUCTIONS_HTML) as html:
instructions = html.read()
overview.append(FormattedContent(instructions))
image = Binary('image', None, image_url, 'image')
overview.append(image)
#--------------- BUILD QUESTION 1 -------------------
qc1 = QuestionContent()
qc1.append_field('Text',
'Is the location shown in the image indoors or outdoors?')
fta1 = SelectionAnswer(min=1, max=1,style='checkbox',
selections=in_out,
type='text',
other=False)
q1 = Question(identifier='Question 1',
content=qc1,
answer_spec=AnswerSpecification(fta1),
is_required=True)
#--------------- BUILD QUESTION 2 -------------------
qc2 = QuestionContent()
qc2.append_field('Text',
'Is the location shown in the image man-made or ' +
'natural? Examples of man-made locations include ' +
'buildings and parks while examples of natural ' +
'locations include mountains and rivers.')
fta2 = SelectionAnswer(min=1, max=1,style='checkbox',
selections=nat_manmade,
type='text',
other=False)
q2 = Question(identifier='Question 2',
content=qc2,
answer_spec=AnswerSpecification(fta2),
is_required=True)
#--------------- BUILD QUESTION 3 -------------------
qc3 = QuestionContent()
qc3.append_field('Text',
'If the location in the image is man-made, what is the ' +
'general function or type of the location? If the ' +
'location is natural (not man-made), don\'t select ' +
'anything here.')
fta3 = SelectionAnswer(min=0, max=1,style='checkbox',
selections=functions,
type='text',
other=False)
q3 = Question(identifier='Question 3',
content=qc3,
answer_spec=AnswerSpecification(fta3),
is_required=False)
#--------------- BUILD QUESTION 4 -------------------
qc4 = QuestionContent()
qc4.append_field('Text',
'If the location in the picture is natural, what ' +
'kind of natural location is it? If the location ' +
'man-made (not natural), don\'t select anything here.')
fta4 = SelectionAnswer(min=0, max=1,style='checkbox',
selections=landscapes,
type='text',
other=False)
q4 = Question(identifier='Question 4',
content=qc4,
answer_spec=AnswerSpecification(fta4),
is_required=False)
#--------------- BUILD THE QUESTION FORM -------------------
question_form = QuestionForm()
question_form.append(overview)
question_form.append(q1)
question_form.append(q2)
question_form.append(q3)
question_form.append(q4)
#-------------- QUALIFICATIONS -------------------
percent = PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 95)
number = NumberHitsApprovedRequirement('GreaterThanOrEqualTo', 200)
quals = Qualifications()
quals.add(percent)
quals.add(number)
#--------------- CREATE THE HIT -------------------
mtc.create_hit(questions=question_form,
max_assignments=1,
title=title,
description=description,
keywords=keywords,
qualifications=quals,
annotation=image_url,
duration=60*10,
reward=0.03)
