class Production(Config):
DEBUG = False
DEVELOPMENT = False
TESTING = False
MTURK_HOST = 'mechanicalturk.amazonaws.com'
MTURK_EXTERNAL_SUBMIT = 'https://www.mturk.com/mturk/externalSubmit'
QUALIFICATIONS = Qualifications([
NumberHitsApprovedRequirement('GreaterThan', 500),
PercentAssignmentsApprovedRequirement('GreaterThan', 95),
LocaleRequirement('EqualTo', 'US')
])
CELERYBEAT_SCHEDULE = {
'delete_temp_files': {
'task': 'delete_temp_files',
'schedule': timedelta(seconds=86400),
'args': ()
},
'run_gather': {
'task': 'run_gather',
'schedule': timedelta(seconds=60),
'args': ()
},
}
TACKBP_NW_09_CORPUS_URL = 'https://s3-us-west-2.amazonaws.com/tac-kbp-2009/sentences.meta-sentencesonly-no-liu-et-al-naacl2016-test-set-270k'
def getHITType():
# Changing this will add another hit type and might mess up later fetches...
# Only change if you know what you are doing...
_mtc = MTurkConnection(host=_host)
_title = "Select Person-Object Interactions in Images"
_description = "Please click on all the objects (or other people) that the highlighted person is interacting with."
_reward = _mtc.get_price_as_price(0.10)
_duration = 60 * 10
_keywords = "person, people, image, images, object, objects, actions, interactions"
_approval_delay = 60 * 60 * 24 * 5
_qualifications = Qualifications()
_qualifications.add(
PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo',
98,
required_to_preview=True))
_qualifications.add(
NumberHitsApprovedRequirement('GreaterThanOrEqualTo',
100,
required_to_preview=True))
_qualifications.add(
LocaleRequirement('EqualTo', 'US', required_to_preview=True))
return _mtc.register_hit_type(title=_title,
description=_description,
reward=_reward,
duration=_duration,
keywords=_keywords,
approval_delay=_approval_delay,
qual_req=_qualifications)
def _mturk_system_qualifications(qualification):
requirements = []
for item in qualification.items.all():
if item.expression['attribute'] not in [
'location', 'approval_rate', 'total_tasks'
]:
continue
requirement = None
if item.expression['attribute'] == 'location':
op = OP_IN if item.expression['operator'] == 'in' else OP_NOT_IN
requirement = MultiLocaleRequirement(op, [
val.strip() for val in item.expression['value']
if val is not None and val != ''
])
elif item.expression['attribute'] == 'approval_rate':
op = OP_GT if item.expression['operator'] == 'gt' else OP_LT
requirement = PercentAssignmentsApprovedRequirement(
op, item.expression['value'])
elif item.expression['attribute'] == 'total_tasks':
op = OP_GT if item.expression['operator'] == 'gt' else OP_LT
requirement = NumberHitsApprovedRequirement(
op, item.expression['value'])
requirements.append(requirement)
return requirements
def getHITType():
# Changing this will add another hit type and might mess up later fetches...
# Only change if you know what you are doing...
_mtc = MTurkConnection(host=_host)
_title = "Guess the Closest Part of a Person!"
_description = "Help us find out which body part of a person is closest to the camera that took the picture."
_keywords = "person, people, image, images, object, objects, depth, comparisons, human3.6m"
_reward = _mtc.get_price_as_price(0.1)
_duration = 60 * 15
_approval_delay = 60 * 60 * 24 * 10
_qualifications = Qualifications()
_qualifications.add(
PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo',
98,
required_to_preview=True))
_qualifications.add(
NumberHitsApprovedRequirement('GreaterThanOrEqualTo',
100,
required_to_preview=True))
_qualifications.add(
LocaleRequirement('EqualTo', 'US', required_to_preview=True))
return _mtc.register_hit_type(title=_title,
description=_description,
reward=_reward,
duration=_duration,
keywords=_keywords,
approval_delay=_approval_delay,
qual_req=_qualifications)
def publish_hit(credentials, isSandbox, ename, num_hits, lvlset, adminToken,
db, mode, no_ifs, individual, with_new_var_powerup, mc=None,
email=None, maxlvls=None, colSwap=False, replay=False):
title = "Play a Math Puzzle Game For Science!"
max_assignments = 1
keywords = "game, math, programming"
description = \
""" NOTE: PLEASE DON'T ACCEPT MORE THAN ONE HIT AT A TIME! First finish your current HIT before looking for others!\n """ + \
"""Help us evaluate our verification game InvGen! Each level is structured as a math puzzle, where you try to come up with correct expressions. Your work directly helps with the verification of programs! This HIT involves playing at least two non-tutorial levels of our game. Played it before? Come play again! You will bypass the tutorial and get new levels! New player? Come try it out! We aim to pay about $10/hr. More specifically: (a) $1.50 for the HIT, which involves playing the game for at least 2 non-tutorial levels (b) $1.50 bonus for doing the tutorial, which you only do the first time (c) $0.75 bonus for each non-tutorial level you pass beyond two."""
mastersQualType = "2ARFPLSP75KLA8M8DH1HTEQVJT3SY6" if isSandbox else \
"2F1QJWKUDD8XADTFD2Q0G6UTO95ALH"
quals = [] if isSandbox else\
[
NumberHitsApprovedRequirement("GreaterThanOrEqualTo", 1000),
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo", 97),
Requirement(mastersQualType, "Exists")
]
if mc is None:
mc = connect(credentials, isSandbox)
balance = mc.get_account_balance()
print("Balance:", balance[0])
exp = Experiment(ename, True)
print("Running under experiment", ename)
for i in range(num_hits):
port = get_unused_port()
srid = exp.create_unique_server_run_id()
p = start_server(port, ename, srid, lvlset, adminToken, db, email,
maxlvls, colSwap, replay)
print("Started server run", srid, "on port", port, "with pid", p.pid)
start_url =\
"https://zoidberg.ucsd.edu:{0}/mturk_landing.html?mode=" + mode
if (no_ifs):
start_url += "&noifs"
if individual:
start_url += "&individual=1"
if with_new_var_powerup:
start_url += "&nvpower=1"
q = ExternalQuestion(start_url.format(port), 1024)
kind = "ExternalQuestion"
r = mc.create_hit(question=q,
lifetime=timedelta(7),
max_assignments=max_assignments,
title=title,
description=description,
keywords=keywords,
reward=HIT_REWARD,
duration=timedelta(0, 45*60),
qualifications=Qualifications(quals))
assert len(r) == 1
print("Created", kind, "HIT", r[0].HITId)
exp.add_session(ServerRun(srid, r[0].HITId, p.pid, port))
return exp
def create_hit(url=None,
title=None,
description=None,
keywords=None,
reward_amount=None,
max_assignments=None,
duration_in_minutes=None,
lifetime_in_days=None,
approval_delay_in_days=None,
qualification_percent_approved=None,
qualification_hits_approved=None):
url = url or "https://squadtest.herokuapp.com/"
title = title or "Compare 100 sets of 2 Instagram posts to guess which performed better (<10 minutes)"
description = description or "This HIT will take at most 15 minutes (usually much less). If you have a problem with the HIT, message us so we can fix it!"
keywords = keywords or [
"easy", "survey", "study", "bonus", "image", "images", "compare",
"comparisons", "collection", "data", "research", "listings", "simple",
"photo", "answer", "opinion", "question"
]
frame_height = 800
reward_amount = reward_amount or 1
max_assignments = max_assignments or 200
duration_in_minutes = duration_in_minutes or 20
duration = datetime.timedelta(minutes=duration_in_minutes)
lifetime_in_days = lifetime_in_days or 3
lifetime = datetime.timedelta(days=3)
approval_delay_in_days = approval_delay_in_days or 5
approval_delay = datetime.timedelta(days=approval_delay_in_days)
qualification_percent_approved = qualification_percent_approved or 95
q1 = PercentAssignmentsApprovedRequirement('GreaterThan',
qualification_percent_approved)
qualification_hits_approved = qualification_hits_approved or 500
q2 = NumberHitsApprovedRequirement('GreaterThan',
qualification_hits_approved)
qualifications = Qualifications([q1, q2])
questionform = ExternalQuestion(url, frame_height)
result = connection.create_hit(title=title,
description=description,
keywords=keywords,
max_assignments=max_assignments,
question=questionform,
reward=Price(amount=reward_amount),
response_groups=('Minimal', 'HITDetail',
'HITQuestion',
'HITAssignmentSummary'),
lifetime=lifetime,
duration=duration,
approval_delay=approval_delay,
qualifications=qualifications)
print result
def get_qualifications(self):
requirements = []
approved_hits = NumberHitsApprovedRequirement('GreaterThan',
self.min_hits)
percentage_approved = PercentAssignmentsApprovedRequirement(
'GreaterThanOrEqualTo', 97)
locale = MultiLocaleRequirement('In', self.countries)
requirements.append(locale)
requirements.append(approved_hits)
requirements.append(percentage_approved)
return Qualifications(requirements)
def register_hit(self, configuration=None):
"""
Register a hit on Mechanical Turk according to `hit_params`. This will provide you with a HITTypeId.
Parameters
----------
configuration : dict
Returns
-------
str
The HITTypeId which is how you refer to your newly registered hit with Amazon
"""
if configuration is None:
configuration = app.config
qualifications = Qualifications()
if self.debug:
qualifications.add(
NumberHitsApprovedRequirement('GreaterThanOrEqualTo', 0))
qualifications.add(
PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo',
0))
else:
qualifications.add(
NumberHitsApprovedRequirement(
'GreaterThanOrEqualTo',
configuration['MTURK_NUMBER_HITS_APPROVED_REQUIREMENT']))
qualifications.add(
PercentAssignmentsApprovedRequirement(
'GreaterThanOrEqualTo', configuration[
'MTURK_PERCENT_ASSIGNMENTS_APPROVED_REQUIREMENT']))
hit_type = self.connection.register_hit_type(
configuration['MTURK_TITLE'], configuration['MTURK_DESCRIPTION'],
Price(configuration['MTURK_REWARD']),
configuration['MTURK_ASSIGNMENT_DURATION_IN_SECONDS'],
configuration['MTURK_KEYWORDS'],
configuration['MTURK_AUTO_APPROVAL_DELAY_IN_SECONDS'],
qualifications)
return hit_type[0].HITTypeId
def get_qualifications(prod):
qualifications = Qualifications()
if args.prod:
qualifications.add(
PercentAssignmentsApprovedRequirement(comparator="GreaterThan",
integer_value=str(95)))
qualifications.add(
NumberHitsApprovedRequirement(comparator="GreaterThan",
integer_value=str(500)))
# these location constraints are potentially useful
# qualifications.add(LocaleRequirement(comparator="NotEqualTo", locale="your country code", required_to_preview=True))
# qualifications.add(LocaleRequirement(comparator="EqualTo", locale="your country code", required_to_preview=True))
return qualifications
def createQualifications(sandbox, qualificationIDs):
qualifications = Qualifications()
for qualification_id in qualificationIDs[:8]:
qualifications.add(
Requirement(qualification_id,
'DoesNotExist',
required_to_preview=True))
qualifications.add(
PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 90,
True))
qualifications.add(
NumberHitsApprovedRequirement('GreaterThanOrEqualTo', 100, True)) #100
return qualifications
def createQualifications(sandbox):
qualifications = Qualifications()
if sandbox:
MastersQualID = '2F1KVCNHMVHV8E9PBUB2A4J79LU20F'
else:
MastersQualID = '2NDP2L92HECWY8NS8H3CK0CP5L9GHO'
#qualifications.add(Requirement(MastersQualID,'DoesNotExist',required_to_preview=True))
qualifications.add(
PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 90,
True))
qualifications.add(
NumberHitsApprovedRequirement('GreaterThanOrEqualTo', 100, True))
return qualifications
def createSeveralHits(numberOfSubjects):
# HOST = 'mechanicalturk.sandbox.amazonaws.com'
# HOST = 'mechanicalturk.amazonaws.com'
global connection
global layoutID
url = 'THE LINK TO THE STUDY'
title = "TITLE OF THE STUDY"
description = "DESCRIPTION OF THE STUDY"
keywords = ["KEYWORDS"]
amount = 0.4
# game = ExternalQuestion(url, frame_height)
remaining = numberOfSubjects
create_hit_result = None
timestr = time.strftime("%Y%m%d-%H%M%S")
myfile=open("./HITIDs/"+timestr + '.txt', 'w') # Saves all the created hits under HITIDS. The name of the file is based on the date and time of creation.
####Setting Worker Requirements:
workerRequirements=Qualifications()
req1= PercentAssignmentsApprovedRequirement(comparator = "GreaterThan", integer_value = "96",required_to_preview=True)
workerRequirements.add(req1)
req2= LocaleRequirement(comparator = "EqualTo",locale ="US",required_to_preview=True)
workerRequirements.add(req2)
req3=NumberHitsApprovedRequirement(comparator = "GreaterThan", integer_value = "50",required_to_preview=True)
workerRequirements.add(req3)
while remaining>0:
no_assignments = min(9, remaining)
create_hit_result = connection.create_hit(
title=title,
description=description,
keywords=keywords,
max_assignments=no_assignments,
hit_layout=layoutID,
lifetime=6*60*60,
duration=3*60*60,
approval_delay=3*60*60*24,
reward=Price(amount=amount),
qualifications=workerRequirements)
remaining -= no_assignments
myfile.write(create_hit_result[0].HITId+"\n")
print "No. Assignments: ", no_assignments
print "Results:", create_hit_result
print "hit ID:", create_hit_result[0].HITId
def qualification_to_boto(*args):
""" Convert a qualification to the format required by boto """
if len(args) == 2:
name = args[0]
value = args[1]
elif len(args) == 1:
name = args[0].name
value = args[0].value
else:
raise ValueError("Invalid arguments")
from mturk.models import MtQualification
try:
q = MtQualification.objects.get(slug=name)
return Requirement(qualification_type_id=q.id,
comparator="EqualTo",
integer_value=value)
except MtQualification.DoesNotExist:
pass
if name == 'num_approved':
return NumberHitsApprovedRequirement(comparator='GreaterThanOrEqualTo',
integer_value=value)
elif name == 'perc_abandoned':
return PercentAssignmentsAbandonedRequirement(
comparator='LessThanOrEqualTo', integer_value=value)
elif name == 'perc_approved':
return PercentAssignmentsApprovedRequirement(
comparator='GreaterThanOrEqualTo', integer_value=value)
elif name == 'perc_rejected':
return PercentAssignmentsRejectedRequirement(
comparator='LessThanOrEqualTo', integer_value=value)
elif name == 'perc_returned':
return PercentAssignmentsReturnedRequirement(
comparator='LessThanOrEqualTo', integer_value=value)
elif name == 'perc_submitted':
return PercentAssignmentsSubmittedRequirement(
comparator='GreaterThanOrEqualTo', integer_value=value)
else:
raise ValueError("Unknown name: %s" % name)
'tests.demo',
]
mturk_hit_settings = {
'keywords': ['easy', 'bonus', 'choice', 'study'],
'title': 'Title for your experiment',
'description': 'Description for your experiment',
'frame_height': 500,
'preview_template': 'global/MTurkPreview.html',
'minutes_allotted_per_assignment': 60,
'expiration_hours': 7*24, # 7 days
# to prevent retakes
'grant_qualification_id': 'YOUR_QUALIFICATION_ID_HERE',
'qualification_requirements': [
LocaleRequirement("EqualTo", "US"),
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo", 50),
NumberHitsApprovedRequirement("GreaterThanOrEqualTo", 5),
# Requirement('YOUR_QUALIFICATION_ID_HERE', 'DoesNotExist'),
]
}
SESSION_CONFIG_DEFAULTS = {
'real_world_currency_per_point': 0.01,
'participation_fee': 10.00,
'doc': "",
'mturk_hit_settings': mturk_hit_settings,
}
SESSION_CONFIGS = [
{
AMAZON_HOST = "https://www.mturk.com/mturk/externalSubmit"
connection = MTurkConnection(aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
host=AMAZON_HOST)
#frame_height in pixels
frame_height = 800
#Here, I create two sample qualifications
qualifications = Qualifications()
qualifications.add(
PercentAssignmentsApprovedRequirement(comparator="GreaterThan",
integer_value="90"))
qualifications.add(
NumberHitsApprovedRequirement(comparator="GreaterThan",
integer_value="100"))
#This url will be the url of your application, with appropriate GET parameters
url = "google.com"
questionform = ExternalQuestion(url, frame_height)
create_hit_result = connection.create_hit(
title="Insert the title of your HIT",
description="Insert your description here",
keywords=["add", "some", "keywords"],
#duration is in seconds
duration=60 * 60,
#max_assignments will set the amount of independent copies of the task (turkers can only see one)
max_assignments=15,
question=questionform,
reward=Price(amount=amount),
#Determines information returned by method in API, not super important
def submitComparisons(population):
#print "mturkLayoutID", population.amusic.conf['mturkLayoutID']
rawListOfPairs = []
if not population.amusic.conf['mturkLayoutID']:
print "LayoutID not set"
return
l = [
i[0] for i in population.amusic.engine.execute(
'SELECT title FROM song WHERE population="%s";' %
(population.title)).fetchall()
]
print "list of songs", l
tempList = list(l)
subset = []
for count in range(sizeOfSubset):
item = tempList.pop(random.randint(0, len(tempList) - 1))
subset.append(item)
print "subset", subset
for d, i in enumerate(l):
print 'Song %d/%d (%s)' % (d + 1, len(l), i)
s = Song(population, i)
s.fromDB()
s.synth(i + '.mp3' if i.find('.') == -1 else i[:i.find('.')] + '.mp3',
upload=True,
generateMP3=True)
print "Creating HITs..."
print "host:", population.amusic.mtc.host
s = "http://%s.s3.amazonaws.com/tracks/" % (
population.amusic.conf['bucket'])
n = 1
#for d,i in enumerate(l):
for i in l:
for j in subset:
rawListOfPairs.append([i, j])
#listOfPairs = filterListOfPairs(rawListOfPairs)
listOfPairs = rawListOfPairs
# submit items to mechanical turk
for i, j in listOfPairs:
f1 = i + '.mp3' if i.find('.') == -1 else i[:i.find('.')] + '.mp3'
f2 = j + '.mp3' if j.find('.') == -1 else j[:j.find('.')] + '.mp3'
#print "\tHIT %d/%d (%s,%s)" % (n,len(l)*(len(l)-1)/2,f1,f2)
print "\tHIT %d (%s, %s)" % (n, f1, f2)
params = LayoutParameters()
params.add(LayoutParameter('file1', s + f1))
params.add(LayoutParameter('file2', s + f2))
#print population.amusic.conf['hitRewardPerAssignment']
#sys.exit()
#quals = quals = Qualifications();
quals = Qualifications()
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
"97"))
quals.add(NumberHitsApprovedRequirement("GreaterThanOrEqualTo", "100"))
population.amusic.mtc.create_hit(
hit_layout=population.amusic.conf['mturkLayoutID'],
layout_params=params,
reward=0.01,
title=population.amusic.conf['hitTitle'],
description=population.amusic.conf['hitDescription'],
keywords=population.amusic.conf['hitKeywords'],
duration=timedelta(minutes=3),
lifetime=timedelta(minutes=300),
qualifications=quals,
max_assignments=numAssignmentsPerHIT)
n += 1
print "raw list of pairs has %d items" % len(rawListOfPairs)
print "cleaned list of pairs has %d items" % len(listOfPairs)
return listOfPairs
def order(session):
"""Order new MTurk HITs."""
parser = reqparse.RequestParser()
parser.add_argument('title', type=str, required=True)
parser.add_argument('description', type=str, required=True)
parser.add_argument('task', type=str, required=True)
parser.add_argument('keywords', type=str, required=True)
parser.add_argument('reward', type=float, required=True)
parser.add_argument('duration', type=int, required=True)
parser.add_argument('quota', type=int, required=True)
parser.add_argument('limit', type=int, required=True)
parser.add_argument('batch', type=int, required=True)
#parser.add_argument('pin', None, type=int)
#parser.add_argument('approval_delay', type=str)
parser.add_argument('sandbox', type=str, required=True)
args = parser.parse_args()
if not args.sandbox:
if args.pin is None:
return 'pin required', 400
if not authenticate(APP.config['TWO_FACTOR_SECRET'], args.pin):
return 'authentication failed', 400
connection = get_mturk_connection(args.sandbox)
duration = datetime.timedelta(minutes=args.duration)
hittype = session.query(database.HITType) \
.filter_by(
title=args.title,
description=args.description,
keywords=args.keywords,
reward=args.reward,
duration=duration,
approval_delay=None,
sandbox=args.sandbox
).first()
if hittype is None:
if args.sandbox:
qual_req = None
else:
num_approved = NumberHitsApprovedRequirement(
'GreaterThanOrEqualTo', 1000)
percent_approved = PercentAssignmentsApprovedRequirement(
'GreaterThanOrEqualTo', 97)
qual_req = Qualifications([
num_approved,
percent_approved,
])
hit_type = connection.register_hit_type(
title=args.title,
description=args.description,
reward=args.reward,
duration=60 * args.duration,
keywords=args.keywords,
approval_delay=None,
qual_req=qual_req,
)
hittype = database.HITType(
hittypeid=hit_type[0].HITTypeId,
title=args.title,
description=args.description,
keywords=args.keywords,
reward=args.reward,
duration=duration,
approval_delay=None,
sandbox=args.sandbox,
ignore=False,
)
session.add(hittype)
hittype.ignore = False
db_type = {
'clicker': database.ClickSession,
'bbox': database.BoundingBoxSession,
'approve_pairs': database.ApprovePairSession,
'labeler': database.Annotation,
'occlusion': database.OcclusionRanking,
'daynight': database.DayNight,
}[args.task]
hit_factory = {
'clicker': create_photo_hit,
'bbox': create_vehicle_hit_num(1),
'approve_pairs': create_approve_pair_hit,
'labeler': create_photo_hit,
'occlusion': create_vehicle_hit_num(24),
'daynight': create_photo_hit,
}[args.task]
hit_factory(
session,
connection,
hittype,
db_type,
args.quota,
args.batch,
args.limit,
args.task,
)
session.flush()
# pylint: disable-msg=E1101,E1103
hittype_id = hittype.id
# pylint: enable-msg=E1101,E1103
session.commit()
return jsonify(status='ok', id=hittype_id)
SANDBOX = True
HOST = {
True: "mechanicalturk.sandbox.amazonaws.com",
False: "mechanicalturk.amazonaws.com"
}[SANDBOX]
NUM_ITERATIONS = 10
EXPERIMENT_URL = """https://sophiaray.github.io/quadmods/abstract.html?shapeCond=Rh,S&condition=label-order"""
mtc = MTurkConnection(aws_access_key_id=AK,
aws_secret_access_key=SK,
host=HOST)
quals = Qualifications()
quals.add(PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 95))
quals.add(NumberHitsApprovedRequirement('GreaterThanOrEqualTo', 1))
quals.add(LocaleRequirement('EqualTo', 'US'))
new_hit = mtc.create_hit(
hit_type=None,
question=ExternalQuestion(EXPERIMENT_URL, 600),
lifetime=2 * 60 *
60, # Amount of time HIT will be available to accept unless 'max_assignments' are accepted before
max_assignments=NUM_ITERATIONS,
title='$3 for 15 min. | Concept learning | University of Louisville',
description=
'Participate in a simple psychological experiment on concept learning. The complete duration should be approximately 15 minutes (reward is estimated according to $12/hr).',
keywords='concepts, learning',
reward=3.0,
duration=45 *
60, # Maximum amount of time turkers are allowed to spend on HIT
def configure_hit(self, hit_config):
''' Configure HIT '''
# configure question_url based on the id
experiment_portal_url = hit_config['ad_location']
frame_height = 600
mturk_question = ExternalQuestion(experiment_portal_url, frame_height)
# Qualification:
quals = Qualifications()
approve_requirement = hit_config['approve_requirement']
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
number_hits_approved = hit_config['number_hits_approved']
quals.add(
NumberHitsApprovedRequirement("GreaterThanOrEqualTo",
number_hits_approved))
require_master_workers = hit_config['require_master_workers']
if require_master_workers:
quals.add(MasterRequirement(sandbox=self.is_sandbox))
if hit_config['us_only']:
quals.add(LocaleRequirement("EqualTo", "US"))
# Create a HIT type for this HIT.
hit_type = self.mtc.register_hit_type(
hit_config['title'],
hit_config['description'],
hit_config['reward'],
hit_config['duration'],
keywords=hit_config['keywords'],
approval_delay=None,
qual_req=quals)[0]
# Check the config file to see if notifications are wanted.
config = PsiturkConfig()
config.load_config()
try:
url = config.get('Server Parameters', 'notification_url')
all_event_types = [
"AssignmentAccepted",
"AssignmentAbandoned",
"AssignmentReturned",
"AssignmentSubmitted",
"HITReviewable",
"HITExpired",
]
self.mtc.set_rest_notification(
hit_type.HITTypeId,
url,
event_types=all_event_types)
except:
pass
# Specify all the HIT parameters
self.param_dict = dict(
hit_type=hit_type.HITTypeId,
question=mturk_question,
lifetime=hit_config['lifetime'],
max_assignments=hit_config['max_assignments'],
questions=None,
response_groups=[
'Minimal',
'HITDetail',
'HITQuestion',
'HITAssignmentSummary'
])
def send_hit(self):
# First check for qualifications
qualifications = Qualifications()
if self.qualification_adult:
qualifications.add(AdultRequirement("EqualTo", 1))
else:
qualifications.add(AdultRequirement("EqualTo", 0))
if self.qualification_custom not in [None, ""]:
qualifications.add(
Requirement(self.qualification_custom,
self.qualification_custom_operator,
self.qualification_custom_value,
required_to_preview=True))
if self.qualification_number_hits_approved is not None:
qual_number_hits = NumberHitsApprovedRequirement(
"GreaterThan", self.qualification_number_hits_approved)
qualifications.add(qual_number_hits)
if self.qualification_percent_assignments_approved is not None:
qual_perc_approved = PercentAssignmentsApprovedRequirement(
"GreaterThan", self.qualification_percent_assignments_approved)
qualifications.add(qual_perc_approved)
if self.qualification_locale != 'None':
qualifications.add(
LocaleRequirement("EqualTo", self.qualification_locale))
# Domain name must be https
url = "%s/turk/%s" % (DOMAIN_NAME, self.id)
frame_height = 900
questionform = ExternalQuestion(url, frame_height)
if len(qualifications.requirements) > 0:
result = self.connection.create_hit(
title=self.title,
description=self.description,
keywords=self.keywords,
duration=datetime.timedelta(self.assignment_duration_in_hours /
24.0),
lifetime=datetime.timedelta(self.lifetime_in_hours / 24.0),
max_assignments=self.max_assignments,
question=questionform,
qualifications=qualifications,
reward=Price(amount=self.reward),
response_groups=('Minimal', 'HITDetail'))[0]
else:
result = self.connection.create_hit(
title=self.title,
description=self.description,
keywords=self.keywords,
duration=datetime.timedelta(self.assignment_duration_in_hours /
24.0),
lifetime=datetime.timedelta(self.lifetime_in_hours / 24.0),
max_assignments=self.max_assignments,
question=questionform,
reward=Price(amount=self.reward),
response_groups=('Minimal', 'HITDetail'))[0]
# Update our hit object with the aws HIT
self.mturk_id = result.HITId
# When we generate the hit, we won't have any assignments to update
self.update(mturk_hit=result)
def make_hit(image_url):
title = 'Label image with its location'
description = 'Answer questions about an image to label its location.'
keywords = 'image categorization, locations, scene recognition'
in_out = [('indoors', '0'), ('outdoors', '1')]
nat_manmade = [('man-made', '0'), ('natural', '1')]
functions = [('transportation/urban', '0'), ('restaurant', '1'),
('recreation', '2'), ('domestic', '3'),
('work/education', '4'), ('other/unclear', '5')]
landscapes = [('body of water/beach', '0'), ('field', '1'),
('mountain', '2'), ('forest/jungle', '3'),
('other/unclear', '4')]
#--------------- BUILD OVERVIEW -------------------
overview = Overview()
overview.append_field('Title', title)
with open(INSTRUCTIONS_HTML) as html:
instructions = html.read()
overview.append(FormattedContent(instructions))
image = Binary('image', None, image_url, 'image')
overview.append(image)
#--------------- BUILD QUESTION 1 -------------------
qc1 = QuestionContent()
qc1.append_field(
'Text', 'Is the location shown in the image indoors or outdoors?')
fta1 = SelectionAnswer(min=1,
max=1,
style='checkbox',
selections=in_out,
type='text',
other=False)
q1 = Question(identifier='Question 1',
content=qc1,
answer_spec=AnswerSpecification(fta1),
is_required=True)
#--------------- BUILD QUESTION 2 -------------------
qc2 = QuestionContent()
qc2.append_field(
'Text', 'Is the location shown in the image man-made or ' +
'natural? Examples of man-made locations include ' +
'buildings and parks while examples of natural ' +
'locations include mountains and rivers.')
fta2 = SelectionAnswer(min=1,
max=1,
style='checkbox',
selections=nat_manmade,
type='text',
other=False)
q2 = Question(identifier='Question 2',
content=qc2,
answer_spec=AnswerSpecification(fta2),
is_required=True)
#--------------- BUILD QUESTION 3 -------------------
qc3 = QuestionContent()
qc3.append_field(
'Text', 'If the location in the image is man-made, what is the ' +
'general function or type of the location? If the ' +
'location is natural (not man-made), don\'t select ' +
'anything here.')
fta3 = SelectionAnswer(min=0,
max=1,
style='checkbox',
selections=functions,
type='text',
other=False)
q3 = Question(identifier='Question 3',
content=qc3,
answer_spec=AnswerSpecification(fta3),
is_required=False)
#--------------- BUILD QUESTION 4 -------------------
qc4 = QuestionContent()
qc4.append_field(
'Text', 'If the location in the picture is natural, what ' +
'kind of natural location is it? If the location ' +
'man-made (not natural), don\'t select anything here.')
fta4 = SelectionAnswer(min=0,
max=1,
style='checkbox',
selections=landscapes,
type='text',
other=False)
q4 = Question(identifier='Question 4',
content=qc4,
answer_spec=AnswerSpecification(fta4),
is_required=False)
#--------------- BUILD THE QUESTION FORM -------------------
question_form = QuestionForm()
question_form.append(overview)
question_form.append(q1)
question_form.append(q2)
question_form.append(q3)
question_form.append(q4)
#-------------- QUALIFICATIONS -------------------
percent = PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 95)
number = NumberHitsApprovedRequirement('GreaterThanOrEqualTo', 200)
quals = Qualifications()
quals.add(percent)
quals.add(number)
#--------------- CREATE THE HIT -------------------
mtc.create_hit(questions=question_form,
max_assignments=1,
title=title,
description=description,
keywords=keywords,
qualifications=quals,
annotation=image_url,
duration=60 * 10,
reward=0.03)
host = 'mechanicalturk.amazonaws.com'
else:
host = 'mechanicalturk.sandbox.amazonaws.com'
mtc = MTurkConnection(host=host,
aws_access_key_id=os.environ["AWS_KEY"],
aws_secret_access_key=os.environ["AWS_SECRET"])
# if DEV.intersection(argset):
# ext_url = 'https://wearedynamo-dev.herokuapp.com/registration_codes/new'
# else:
ext_url = 'https://s3-us-west-2.amazonaws.com/cs399-research/testhit.html'
question = ExternalQuestion(ext_url, 400)
requirement = NumberHitsApprovedRequirement('GreaterThan', 5, True)
qualifications = Qualifications()
if REQUIREMENT.intersection(argset):
#if '-r' in sys.argv or '--requirement' in sys.argv:
qualifications.add(requirement)
payment = 0 if FREE.intersection(argset) else 0.03
#"--free" in sys.argv else 0.10
try:
#if "-e" in sys.argv or "--extend" in sys.argv
if NEWHIT.intersection(argset):
mtc.create_hit(question=question,
title=TITLE,
description=DESCRIPTION,
def create_hit(href, hit):
if hit.type == 'MULTIPLE_URLS': # the max_workers was not set in this form
hit.max_workers = hit.size1 + hit.size2 + hit.size3 + hit.size4 + hit.size5 + \
hit.size6 + hit.size7 + hit.size8 + hit.size9 + hit.size10
hostURL = SANDBOX_HOST if hit.sandbox else HOST
qualifications = Qualifications()
if hit.hit_approval > 0:
qualifications.add(
NumberHitsApprovedRequirement("GreaterThan", hit.hit_approval,
False))
if hit.hit_approval_rate > 0:
qualifications.add(
PercentAssignmentsApprovedRequirement("GreaterThan",
hit.hit_approval_rate,
False))
if hit.accepted_hit_rate > 0:
qualifications.add(
PercentAssignmentsSubmittedRequirement("GreaterThan",
hit.accepted_hit_rate,
False))
if hit.returned_hit_rate > 0:
qualifications.add(
PercentAssignmentsReturnedRequirement("GreaterThan",
hit.returned_hit_rate,
False))
if hit.abandoned_hit_rate > 0:
qualifications.add(
PercentAssignmentsAbandonedRequirement("LessThan",
hit.abandoned_hit_rate,
False))
if hit.rejected_hit_rate > 0:
qualifications.add(
PercentAssignmentsRejectedRequirement("LessThan",
hit.rejected_hit_rate,
False))
if hit.locale_qualification is not None and hit.locale_qualification != 'all':
qualifications.add(
LocaleRequirement("EqualTo", hit.locale_qualification, False))
connection = MTurkConnection(aws_access_key_id=hit.aws_access_key,
aws_secret_access_key=hit.aws_secret_key,
host=hostURL,
is_secure=True)
return Response(
connection.create_hit(
question=ExternalQuestion(href, hit.frame_height),
lifetime=hit.lifetime,
max_assignments=hit.max_workers,
title=hit.title,
description=hit.description,
keywords=['turktime'] # TODO
,
reward=hit.reward,
duration=hit.duration,
approval_delay=hit.approval_delay,
qualifications=qualifications,
response_groups=[
'Minimal', 'HITDetail', 'HITQuestion', 'HITAssignmentSummary'
]))