def create_hits(batch, num_assignments, duration, reward, examples):
"""
Creates HITs and posts on Mechanical Turk for a batch of pairs of strings to be matched
Args:
batch (list) : List of pairs of strings to be matched
num_assignments (int) : Number of assignments for each HIT
duration (float) : Time for which a HIT is to be posted
reward (float) : Reward for each correct answer
examples (tuple) : Input examples for which HTML is to be generated
"""
question_form = create_question(batch, examples)
mtc = MTurkConnection(aws_access_key_id = aws_parameters['access_key'],
aws_secret_access_key = aws_parameters['secret_key'],
debug = 1,
host = DEV_HOST)
mtc.create_hit(questions = question_form,
max_assignments = num_assignments,
title = title,
description = description,
keywords = keywords,
duration = duration,
reward = reward)
def prepareBatch(self):
if not self.amusic.conf['mturkLayoutID']:
print "LayoutID not set"
return
l = [i[0] for i in self.amusic.engine.execute('SELECT title FROM song WHERE population="%s";'%(self.title)).fetchall()]
for d,i in enumerate(l):
print 'Song %d/%d (%s)'%(d+1,len(l),i)
s = Song(self,i)
s.fromDB()
s.synth(i+'.mp3' if i.find('.')==-1 else i[:i.find('.')]+'.mp3',upload=True)
print "Creating HITs..."
hostName = 'mechanicalturk.amazonaws.com'
#hostName = 'mechanicalturk.sandbox.amazonaws.com'
mtc = MTurkConnection(self.amusic.ACCESS_ID,self.amusic.SECRET_KEY,host=hostName)#,debug=5)
print "host:", hostName
s = "http://%s.s3.amazonaws.com/tracks/" % (self.amusic.conf['bucket'])
n=1
for d,i in enumerate(l):
for j in l[d+1:]:
f1 = i+'.mp3' if i.find('.')==-1 else i[:i.find('.')]+'.mp3'
f2 = j+'.mp3' if j.find('.')==-1 else j[:j.find('.')]+'.mp3'
print "\tHIT %d/%d (%s,%s)" % (n,len(l)*(len(l)-1)/2,f1,f2)
params = LayoutParameters()
params.add(LayoutParameter('file1',s+f1))
params.add(LayoutParameter('file2',s+f2))
mtc.create_hit(hit_layout=self.amusic.conf['mturkLayoutID'],layout_params=params,reward=self.amusic.conf['hitRewardPerAssignment'],title=self.amusic.conf['hitTitle'],description=self.amusic.conf['hitDescription'],keywords=self.amusic.conf['hitKeywords'])
n+=1
class MTurk(object):
def __init__(self):
self.mtc = MTurkConnection(aws_access_key_id=ACCESS_ID,
aws_secret_access_key=SECRET_KEY,
host=HOST)
def balance(self):
return self.mtc.get_account_balance()
def create_question(self, id, title):
content = QuestionContent()
content.append_field('Title', title)
text = FreeTextAnswer()
return Question(identifier=id, content=content, answer_spec=AnswerSpecification(text))
def create_hit(self, title, description=''):
question_form = QuestionForm()
question_form.append(self.create_question(id='1', title='Comments'))
question_form.append(self.create_question(id='2', title='More Comments'))
self.mtc.create_hit(questions=question_form, max_assignments=1, title=title, description=description,
duration=60*5, reward=0.01)
def external(self):
q = ExternalQuestion(external_url="http://mturk-hit-wizard.herokuapp.com/view/2", frame_height=800)
#conn = MTurkConnection(host=HOST)
keywords=['boto', 'test', 'doctest']
create_hit_rs = self.mtc.create_hit(question=q, lifetime=60*65,max_assignments=2,title="Boto External Question Test", keywords=keywords,reward = 0.05, duration=60*6,approval_delay=60*60, annotation='An annotation from boto external question test', response_groups=['Minimal','HITDetail','HITQuestion','HITAssignmentSummary',])
assert(create_hit_rs.status == True)
def submitHITs(self, mtc=None, questionForms=[], max_assignments=5,
title='Rank the most important sentences',
description='Rank the following sentences by importance',
keywords='summary, survey',
duration=60*3,
reward=0.3):
""" Creates and submits a list of HITTS with the exact same
title, descriptions, durations and prices from a list of questions.
"""
if mtc is None:
mtc = MTurkConnection(aws_access_key_id=ACCESS_ID,
aws_secret_access_key=SECRET_KEY,
host=HOST)
for questionForm in questionForms:
mtc.create_hit(questions=questionForm[1],
max_assignments=max_assignments,
title=title,
description=description,
keywords=keywords,
duration=60*60*12,
approval_delay=60*60*12,
annotation=questionForm[0],
reward=0.03)
pass
def submitHITs(self,
mtc=None,
questionForms=[],
max_assignments=5,
title='Rank the most important sentences',
description='Rank the following sentences by importance',
keywords='summary, survey',
duration=60 * 3,
reward=0.3):
""" Creates and submits a list of HITTS with the exact same
title, descriptions, durations and prices from a list of questions.
"""
if mtc is None:
mtc = MTurkConnection(aws_access_key_id=ACCESS_ID,
aws_secret_access_key=SECRET_KEY,
host=HOST)
for questionForm in questionForms:
mtc.create_hit(questions=questionForm[1],
max_assignments=max_assignments,
title=title,
description=description,
keywords=keywords,
duration=60 * 60 * 12,
approval_delay=60 * 60 * 12,
annotation=questionForm[0],
reward=0.03)
pass
def generate_hit(self, num_assignments, hit_duration, hit_reward):
"""
Purpose: Generate and publish the HIT
Parameters: num_assignments is the number of avaliable assignments for hit,
hit_duration is the duration of the hit in seconds (60*5 for 5 minutes),
hit_reward is the reward given per hit in dollars (0.05 is 5 cents)
"""
# CONNECT TO MTURK
mtc = MTurkConnection(aws_access_key_id = self.access_id,
aws_secret_access_key = self.secret_key,
host = self.host)
# BUILD OVERVIEW
overview = Overview()
overview.append_field('Title', 'The following one or more sentences constitute an incomplete story.')
story = ""
for sentence in self.story_sentences:
story += sentence + " "
overview.append(FormattedContent(story))
# BUILD QUESTION 1: Copy the first sentence of the story
qc1 = QuestionContent()
qc1.append_field('Title','Copy verbatim the first sentence of the provided incomplete story. Please keep all capitalization and punctuation as given. Your sumbission will automatically be rejected if any character is incorrect.')
fta1 = FreeTextAnswer()
q1 = Question(identifier='verify_sentence', content = qc1, answer_spec = AnswerSpecification(fta1), is_required = True)
# BUILD QUESTION 2: Vote on the best sentence to continue the story
sentence_options = []
for i, sentence in enumerate (self.vote_sentences):
selection = (sentence, str(i))
sentence_options.append(selection)
qc2 = QuestionContent()
qc2.append_field('Title','Choose the best sentence to continue the story.')
fta2 = SelectionAnswer(min=1, max=1,style='radiobutton',
selections=sentence_options,
type='text',
other=False)
q2 = Question(identifier='vote_sentence', content = qc2, answer_spec = AnswerSpecification(fta2), is_required = True)
# BUILD THE QUESTION FORM
question_form = QuestionForm()
question_form.append(overview)
question_form.append(q1)
question_form.append(q2)
# CREATE THE HIT
mtc.create_hit(questions = question_form,
max_assignments = num_assignments,
title = self.title,
description = self.description,
keywords = self.keywords,
duration = hit_duration,
reward = hit_reward)
def post_HIT1(ACCESS_ID,SECRET_KEY,HOST,url_to_task):
mtc = MTurkConnection(aws_access_key_id=ACCESS_ID,
aws_secret_access_key=SECRET_KEY,
host=HOST)
title = 'Dev deploying simulation test Report From SERVER'
description = ('Report on events in a simulation')
keywords = 'website, rating, opinions'
instructions=('<p>You will take part in a web-based experiment where you will watch a simple simulation and provide reports on events</p>'
'<p>Instructions:</p>'
'<p>1. Click the link below, which will open the webpage in a new window in your browser</p>'
'<p>2. Follow the instructions on the website</p>'
'<p>3. Once you have completed your work, you will receive a Reward Code</p>'
'<p>4. Return to the mechanical turk webpage and enter your code in the Reward Code text box</p>'
'<p>5. Your work will then be checked, after which you will receive your payment</p>'
'<br/>CLICK "ACCEPT HIT" BEFORE FOLLOWING LINK'
'<br/>YOU WILL NOT BE PAID WITHOUT ACCEPTING THE HIT')
#--------------- BUILD OVERVIEW -------------------
overview = Overview()
overview.append_field('Title', description)
overview.append(FormattedContent(instructions))
overview.append(FormattedContent('<p>Click "Accept HIT" then click this link <a target="_blank"'
' href="'+url_to_task+'">'
' Link to task</a></p>'))
#--------------- BUILD QUESTION 1 -------------------
qc1 = QuestionContent()
qc1.append_field('Title','Enter reward code here:')
fta1 = FreeTextAnswer(num_lines=1)
q1 = Question(identifier='reward_code',
content=qc1,
answer_spec=AnswerSpecification(fta1),
is_required=True)
#--------------- BUILD THE QUESTION FORM -------------------
question_form = QuestionForm()
question_form.append(overview)
question_form.append(q1)
#--------------- CREATE THE HIT -------------------
mtc.create_hit(questions=question_form,
max_assignments=1,
title=title,
description=description,
keywords=keywords,
duration = 60*5,
reward=0.05)
def generate_hit(self, num_assignments, hit_duration, hit_reward):
"""
Purpose: Generate and publish the HIT
Parameters: num_assignments is the number of avaliable assignments for hit,
hit_duration is the duration of the hit in seconds (60*5 for 5 minutes),
hit_reward is the reward given per hit in dollars (0.05 is 5 cents)
"""
# CONNECT TO MTURK
mtc = MTurkConnection(aws_access_key_id = self.access_id,
aws_secret_access_key = self.secret_key,
host = self.host)
# BUILD OVERVIEW
overview = Overview()
overview.append_field('Title', 'The sentence below constitues the beginning of a story.')
overview.append(FormattedContent(self.starter_sentence))
# BUILD QUESTION 1: Copy given sentence
qc1 = QuestionContent()
qc1.append_field('Title','Copy verbatim the provided sentence. Please keep all capitalization and punctuation as given. Your sumbission will automatically be rejected if any character is incorrect.')
fta1 = FreeTextAnswer()
q1 = Question(identifier='verify_sentence', content = qc1, answer_spec = AnswerSpecification(fta1), is_required = True)
# BUILD QUESTION 2: Create new sentence
qc2 = QuestionContent()
qc2.append_field('Title','Type a single sentence to continue the story begun by the given sentence, and please ensure the sentence ends with a period.')
fta2 = FreeTextAnswer()
q2 = Question(identifier='create_sentence', content = qc2, answer_spec = AnswerSpecification(fta2), is_required = True)
# BUILD THE QUESTION FORM
question_form = QuestionForm()
question_form.append(overview)
question_form.append(q1)
question_form.append(q2)
# CREATE THE HIT
mtc.create_hit(questions = question_form,
max_assignments = num_assignments,
title = self.title,
description = self.description,
keywords = self.keywords,
duration = hit_duration,
reward = hit_reward)
class AMTConnection(object):
def __init__(self):
if SANDBOX:
host = 'mechanicalturk.sandbox.amazonaws.com'
else:
host = 'mechanicalturk.amazonaws.com'
login_params = {
'aws_access_key_id': ACCESS_KEY_ID,
'aws_secret_access_key': SECRET_ACCESS_KEY,
'host': host
}
self.connection = MTurkConnection(**login_params)
question = ExternalQuestion(EXTERNAL_URL, FRAMEHEIGHT)
qualifications = Qualifications()
qualifications.add(PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo", PERCENT_APPROVED_REQUIREMENT))
self.hit_parameters = {
'hit_type' : None,
'question' : question,
'title' : TITLE,
'description' : DESCRIPTION,
'keywords' : KEYWORDS,
'max_assignments' : MAX_ASSIGNMENTS,
'lifetime' : datetime.timedelta(hours=HIT_LIFETIME),
'reward' : REWARD,
'duration' : datetime.timedelta(DURATION),
'approval_delay' : None,
'questions' : None,
'qualifications' : qualifications
}
def create_hit(self):
return self.connection.create_hit(**self.hit_parameters)[0]
def create_hit(problem, hit_type, params={}):
"""Utility method for creating a new HIT on AMT"""
hit = Hit(hit_id='?', hit_type=hit_type, problem=problem, params=json.dumps(params),
title=hit_type.title%params, description=hit_type.description%params, body=hit_type.body%params)
hit.save()
# post a HIT on Mechanical Turk using boto
q = ExternalQuestion(external_url=settings.URL_ROOT + hit.get_absolute_url(), frame_height=800)
conn = MTurkConnection(aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
host=settings.AWS_HOST)
# remove commas from the keywords if they exist
keywords=[k.replace(',', '') for k in hit_type.keywords.split()]
create_hit_rs = conn.create_hit(question=q, lifetime=hit_type.lifetime, max_assignments=hit_type.max_assignments,
keywords=keywords, reward=hit_type.payment, duration=hit_type.duration, approval_delay=hit_type.approval_delay,
title=hit.title, description=hit.description, annotation=`hit_type`)
#code.interact(local=locals())
assert(create_hit_rs.status == True)
# set the new HIT ID to be the hit_id for the new row.
hit.hit_id = create_hit_rs[0].HITId
hit.save()
return hit
def createHits(question, answers, params):
if SANDBOX:
mturk_url = 'mechanicalturk.sandbox.amazonaws.com'
preview_url = 'https://workersandbox.mturk.com/mturk/preview?groupId='
else:
mturk_url = 'mechanicalturk.amazonaws.com'
preview_url = 'https://mturk.com/mturk/preview?groupId='
#Create Hit Form Structure
overview = Overview()
overview.append_field('Title', 'We want to know the crowds opinion!')
overview.append(FormattedContent('<a href="http://programthecrowd.com/">Visit us here</a>'))
questionContent = QuestionContent()
questionContent.append_field('Title', question);
answerChoices = SelectionAnswer(min=1, max=1, style='checkbox', selections=answers, type='text', other=False)
q = Question(identifier='Help', content=questionContent, answer_spec=AnswerSpecification(answerChoices), is_required=True)
questionForm = QuestionForm();
questionForm.append(overview)
questionForm.append(q)
hitIdList = []
global conn
# key = params['aws_access_key']
# secret = params['aws_secret_key']
conn = MTurkConnection(aws_access_key_id='AKIAJBTEJI2RGTJH7OBA', aws_secret_access_key='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', host=mturk_url)
#For Loop to create and post hits
for i in range(0, NUMBER_OF_HITS):
create_hit_rs = conn.create_hit(questions=questionForm, lifetime=LIFETIME, max_assignments=NUMBER_OF_ASSIGNMENTS, title=TITLE, keywords=KEYWORDS, reward=REWARD, duration=DURATION, approval_delay=APPROVAL_DELAY, annotation=DESCRIPTION)
#print(preview_url + create_hit_rs[0].HITTypeId)
#print("HIT ID: " + create_hit_rs[0].HITId)
hitIdList.append(create_hit_rs[0].HITId);
return hitIdList
def SubmitHIT(self, sandbox = 'false'):
""""
Constructs a HIT from the HITGenerator's attributes, registers it with Amazon, and returns the HITId as a unicode string.
If the sandbox flag is set to true then the hit will be registered with the Sandbox, otherwise it is registered to AWS
directly. All of the necessary data must have been submitted during the HITGenerator's initiation.
"""
if sandbox is 'true':
self.host = 'mechanicalturk.sandbox.amazonaws.com'
conn = MTurkConnection(host = self.host, aws_access_key_id = self.AWS_KEY, aws_secret_access_key = self.AWS_SECRET)
answer_specification = AnswerSpecification(SelectionAnswer(style = self.answer_style, selections = self.answer_options))
questions = []
for i in self.question_list:
questions.append(Question(identifier=i[1], content = QuestionContent(i[0]), answer_spec = answer_specification))
question_form = QuestionForm(questions)
self.hit_response = conn.create_hit(question = question_form,
lifetime = self.lifetime,
max_assignments = self.assignment_count,
title = self.title,
description = self.description,
keywords = self.keywords,
reward = self.reward,
duration = self.duration,
approval_delay = self.approval_delay,
annotation = self.annotation)
# Returns the HITId as a unicode string
self.HITId = self.hit_response.HITId
return self.HITId
def main():
# parse arguments
parser = argparse.ArgumentParser()
parser.add_argument('keys_file')
parser.add_argument('--production', default=False, action='store_true')
args = parser.parse_args()
# read keys
access_key_id, secret_key = None, None
with open(args.keys_file, 'r') as keys:
for line in keys.xreadlines():
items = [xx.strip() for xx in line.split('=')]
if items[0] == 'AWSAccessKeyId':
access_key_id = items[1]
elif items[0] == 'AWSSecretKey':
secret_key = items[1]
if not access_key_id or not secret_key:
raise RuntimeError('Invalid keys file format.')
# set up URLs
if args.production:
mturk_url = 'mechanicalturk.amazonaws.com'
preview_url = 'https://www.mturk.com/mturk/preview?groupId='
else:
print 'SANDBOX'
mturk_url = 'mechanicalturk.sandbox.amazonaws.com'
preview_url = 'https://workersandbox.mturk.com/mturk/preview?groupId='
# connect
connection = MTurkConnection(aws_access_key_id=access_key_id,
aws_secret_access_key=secret_key,
host=mturk_url)
# make the HIT
question = ExternalQuestion(
external_url=
'https://kirbpowell.github.io/crowdsourcing-assignment2/', # URL to serve HIT
frame_height=600) # height of frame
reward = Price(amount=0.25) # reward for HIT completion
create_hit_result = connection.create_hit(
title='Count Recognizable People in Pictures',
description='Count how many *recognizable* people are in some pictures',
keywords=[
'count', 'people', 'pictures', 'simple', 'easy', 'quick', 'label',
'classification'
],
max_assignments=10, # number of assignments
lifetime=datetime.timedelta(days=2), # time HIT is available
duration=datetime.timedelta(minutes=5), # time to complete
approval_delay=datetime.timedelta(days=7), # time til HIT approved
question=question,
reward=reward,
response_groups=('Minimal', 'HITDetail'))
print('Preview: ' + preview_url + create_hit_result[0].HITTypeId)
print('HIT Id: ' + create_hit_result[0].HITId)
def handle(self, *args, **options):
# create a connection
mturk = MTurkConnection(
getattr(settings, 'MTURK_AWS_KEY', settings.MEDIASYNC['AWS_KEY']),
getattr(settings, 'MTURK_AWS_SECRET', settings.MEDIASYNC['AWS_SECRET']),
host = 'mechanicalturk.sandbox.amazonaws.com' if options['sandbox'] else 'mechanicalturk.amazonaws.com'
)
# if --delete, delete all the old ones first.
if options['delete_first']:
for hit in mturk.get_all_hits():
mturk.disable_hit(hit.HITId)
if options['exclude']:
exclude_reader = csv.DictReader(open(options['exclude'], 'r'))
exclude = set()
for row in exclude_reader:
exclude.add(row['td_id'])
# iterate over items and create them one by one
cursor = connection.cursor()
cursor.execute(
"""
select entity_id, type from matchbox_wikipediainfo, matchbox_entity where entity_id not in (select entity_id from matchbox_sunlightinfo where bio is not null) and bio != '' and bio is not null and entity_id = matchbox_entity.id %s order by entity_id limit %s;
""" % ("and type = '%s'" % options['type'] if options['type'] else '', '%s'), # hack to put the interpolation string back in for PG to catch it
[options['count']])
for row in cursor:
if options['exclude']:
if str(row[0]).replace('-', '') in exclude:
continue
if options['practice']:
print row[0]
continue
try:
hit = mturk.create_hit(
question = FakeQuestionForm(get_hit_xml(row[0])),
max_assignments = 3,
annotation = row[0],
title = "Wikipedia match validation",
description = "We have matched a set of entities in a database to descriptions pulled from Wikipedia via an automated process. Confirm that the match is correct.",
reward = 0.06,
duration = datetime.timedelta(minutes=30),
lifetime = datetime.timedelta(days=7),
keywords = ['wikipedia', 'matching'],
approval_delay = datetime.timedelta(days=3),
qualifications = Qualifications([PercentAssignmentsApprovedRequirement("GreaterThan", 90)])
)
print hit[0].HITId
except Exception as e:
sys.stderr.write("Failed to create hit %s\n" % row[0])
sys.stderr.write(getattr(e, 'body', ''))
sys.stderr.write('\n')
except:
pass
def test():
q = ExternalQuestion(external_url="http://websort.net/s/F3481C", frame_height=800)
conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com')
keywords=['boto', 'test', 'doctest']
qualifications = Qualifications()
qualifications.add(PercentAssignmentsApprovedRequirement(comparator="GreaterThan", integer_value="95"))
create_hit_rs = conn.create_hit(question=q, lifetime=60*65,max_assignments=2,title="Boto External Question Test", keywords=keywords,reward = 0.05, duration=60*6,approval_delay=60*60, annotation='An annotation from boto external question test', qualifications=qualifications)
assert(create_hit_rs.status == True)
print create_hit_rs.HITTypeId
def test():
q = ExternalQuestion(external_url="http://websort.net/s/F3481C", frame_height=800)
conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com')
keywords=['boto', 'test', 'doctest']
qualifications = Qualifications()
qualifications.add(PercentAssignmentsApprovedRequirement(comparator="GreaterThan", integer_value="95"))
create_hit_rs = conn.create_hit(question=q, lifetime=60*65, max_assignments=2, title="Boto External Question Test", keywords=keywords, reward = 0.05, duration=60*6, approval_delay=60*60, annotation='An annotation from boto external question test', qualifications=qualifications)
assert(create_hit_rs.status == True)
print create_hit_rs.HITTypeId
def PostHits_loop():
# To post a HIT, first connect to Turk using our access codes:
# The ACCESS_ID and SECRET_KEY are loaded before this function is called
# (e.g., from alvarezlab import * );
mtc = MTurkConnection(aws_access_key_id=ACCESS_ID,
aws_secret_access_key=SECRET_KEY,
host=HOST)
for x in range(startHSet, endHSet + 1):
HSet = x
urlForHIT = "https://scorsese.wjh.harvard.edu/turk/experiments/cfm/Search10/index_cmtrial_Search10.html?HSetNum=%d" % HSet
# Now lets setup a structure for our external HIT. We need the URL we want to be
# shown within the Turk window and also how tall we want the Turk iframe to be:
q = ExternalQuestion(external_url=urlForHIT, frame_height=frameHeight)
# And any qualifications we want people to have:
qualifications = mtqu.Qualifications()
qualifications.add(
mtqu.PercentAssignmentsApprovedRequirement(
'GreaterThanOrEqualTo', percentAssignmentsApprovedRequirement))
qualifications.add(mtqu.LocaleRequirement("EqualTo",
localeRequirement))
if (qualificationID != "NONE"):
qualifications.add(
mtqu.Requirement(qualificationID, "EqualTo", 1,
notifyWorkerOfQualification))
# Post:
theHIT = mtc.create_hit(
question=q,
lifetime=minToSec(minutesBeforeHitExpires),
max_assignments=numAssignmentsToPost,
title=titleForWorkers,
description=descriptionForWorkers,
keywords=keywordsForWorkers,
qualifications=qualifications,
reward=pay,
duration=minToSec(minutesForUsersToFinish),
approval_delay=minToSec(minutesBeforeAutoApproved),
annotation=projectNameForRequesters)
# get more info about the hit
hit = mtc.get_hit(theHIT[0].HITId)
# Print out the HIT's ID if all went well:
# pprint(vars(hit[0]))
# print "Experiment info:"
# print HOST
print "preview hit in HSet ", HSet, ": "
# print urlForHIT, "\n"
# print "HITId"
# print theHIT[0].HITId, "\n"
print PREVIEW + "?groupId=" + hit[0].HITGroupId, "\n"
def submit_extract_keywords_hit(note):
"""Create a Mechanical Turk HIT that asks a worker to
choose keywords and definitions from the given note."""
try:
MTURK_HOST = os.environ['MTURK_HOST']
except:
logger.warn('Could not find Mechanical Turk secrets, not running submit_extract_keywords_hit')
return
connection = MTurkConnection(settings.AWS_ACCESS_KEY_ID, settings.AWS_SECRET_ACCESS_KEY,
host=MTURK_HOST)
if note.course.school:
title = KEYWORDS_HIT_TITLE_TEMPLATE.format(course=note.course.name, school=note.course.school.name)
else:
title = KEYWORDS_HIT_TITLE_TEMPLATE.format(course=note.course.name, school=note.course.department.school.name)
overview = Overview()
overview.append(FormattedContent(KEYWORDS_HIT_OVERVIEW_TEMPLATE.format(domain=Site.objects.get_current(),
link=note.get_absolute_url())))
keyword_fta = FreeTextAnswer()
keyword_fta.num_lines = 1
definition_fta = FreeTextAnswer()
definition_fta.num_lines = 3
question_form = QuestionForm()
question_form.append(overview)
for i in range(min(len(KEYWORDS_HIT_KEYWORD_FIELDS), len(KEYWORDS_HIT_DEFINITION_FIELDS))):
keyword_content = QuestionContent()
keyword_content.append_field('Title', KEYWORDS_HIT_KEYWORD_FIELDS[i][1])
keyword_question = Question(identifier=KEYWORDS_HIT_KEYWORD_FIELDS[i][0],
content=keyword_content,
answer_spec=AnswerSpecification(keyword_fta),
is_required=True if i <= 10 else False)
question_form.append(keyword_question)
definition_content = QuestionContent()
definition_content.append_field('Title', KEYWORDS_HIT_DEFINITION_FIELDS[i][1])
definition_question = Question(identifier=KEYWORDS_HIT_DEFINITION_FIELDS[i][0],
content=definition_content,
answer_spec=AnswerSpecification(definition_fta),
is_required=False)
question_form.append(definition_question)
hit = connection.create_hit(questions=question_form, max_assignments=1,
title=title, description=KEYWORDS_HIT_DESCRIPTION,
keywords=KEYWORDS_HIT_KEYWORDS, duration=KEYWORDS_HIT_DURATION,
reward=KEYWORDS_HIT_REWARD, qualifications=KEYWORDS_HIT_QUALIFICATION,
annotation=str(note.id))[0]
HIT.objects.create(HITId=hit.HITId, note=note, processed=False)
def create_hits(self, category_id, task_id, num_hits):
print "Connecting to Turk host at"
print app.config['MTURK_HOST']
sys.stdout.flush()
mturk = MTurkConnection(app.config['AWS_ACCESS_KEY_ID'],
app.config['AWS_SECRET_ACCESS_KEY'],
host=app.config['MTURK_HOST'])
print "Uploading %d hits to turk" % num_hits
hits = []
qualifications = app.config['QUALIFICATIONS']
for hit_num in range(num_hits):
category = app.config['EXAMPLE_CATEGORIES'][category_id]
hit_html = category['hit_html']
hit_html = hit_html.replace('${task_id}', task_id)
hit_html = hit_html.replace('${requester_id}',
app.config['CROWDJS_REQUESTER_ID'])
hit_html = hit_html.replace(
'${task_data_url}', app.config['CROWDJS_GET_TASK_DATA_URL'])
hit_html = hit_html.replace(
'${submit_answer_url}',
app.config['CROWDJS_SUBMIT_ANSWER_URL'])
hit_html = hit_html.replace('${compute_taboo_url}',
app.config['SUBMIT_TABOO_URL'])
hit_html = hit_html.replace('${return_hit_url}',
app.config['CROWDJS_RETURN_HIT_URL'])
hit_html = hit_html.replace('${assign_url}',
app.config['CROWDJS_ASSIGN_URL'])
hit_html = hit_html.replace('${taboo_threshold}',
str(app.config['TABOO_THRESHOLD']))
hit_html = hit_html.replace(
'${mturk_external_submit}',
str(app.config['MTURK_EXTERNAL_SUBMIT']))
question = HTMLQuestion(hit_html, 800)
hit = mturk.create_hit(
title=category['task_name'],
description=category['task_description'],
question=question,
reward=Price(category['price']),
duration=datetime.timedelta(minutes=10),
lifetime=datetime.timedelta(days=7),
keywords=
'information extraction, events, natural language processing',
max_assignments=1,
approval_delay=3600,
qualifications=qualifications)[0]
hits.append(hit.HITId)
return hits
def createHits(question, answers, params):
if SANDBOX:
mturk_url = 'mechanicalturk.sandbox.amazonaws.com'
preview_url = 'https://workersandbox.mturk.com/mturk/preview?groupId='
else:
mturk_url = 'mechanicalturk.amazonaws.com'
preview_url = 'https://mturk.com/mturk/preview?groupId='
#Create Hit Form Structure
overview = Overview()
overview.append_field('Title', 'We want to know the crowds opinion!')
overview.append(
FormattedContent(
'<a href="http://programthecrowd.com/">Visit us here</a>'))
questionContent = QuestionContent()
questionContent.append_field('Title', question)
answerChoices = SelectionAnswer(min=1,
max=1,
style='checkbox',
selections=answers,
type='text',
other=False)
q = Question(identifier='Help',
content=questionContent,
answer_spec=AnswerSpecification(answerChoices),
is_required=True)
questionForm = QuestionForm()
questionForm.append(overview)
questionForm.append(q)
hitIdList = []
global conn
# key = params['aws_access_key']
# secret = params['aws_secret_key']
conn = MTurkConnection(
aws_access_key_id='AKIAJBTEJI2RGTJH7OBA',
aws_secret_access_key='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',
host=mturk_url)
#For Loop to create and post hits
for i in range(0, NUMBER_OF_HITS):
create_hit_rs = conn.create_hit(questions=questionForm,
lifetime=LIFETIME,
max_assignments=NUMBER_OF_ASSIGNMENTS,
title=TITLE,
keywords=KEYWORDS,
reward=REWARD,
duration=DURATION,
approval_delay=APPROVAL_DELAY,
annotation=DESCRIPTION)
#print(preview_url + create_hit_rs[0].HITTypeId)
#print("HIT ID: " + create_hit_rs[0].HITId)
hitIdList.append(create_hit_rs[0].HITId)
return hitIdList
def make_poll(title, question, description, keywords, poll_price, ratings):
"""Take submitted request and answers.
Resubmit for polling to ensure validity."""
ACCESS_ID = mtkey.ACCESS_KEY
SECRET_KEY = mtkey.SECRET_KEY
HOST = 'mechanicalturk.amazonaws.com'
# link to HITs: https://requester.mturk.com/mturk/manageHITs
# HOST = 'mechanicalturk.sandbox.amazonaws.com'
# link to HITs: https://requestersandbox.mturk.com/mturk/manageHITs
mtc = MTurkConnection(aws_access_key_id=ACCESS_ID,
aws_secret_access_key=SECRET_KEY,
host=HOST)
#--------------- BUILD OVERVIEW -------------------
overview = Overview()
overview.append_field('Title', title)
#--------------- BUILD POLL -------------------
qc2 = QuestionContent()
qc2.append_field('Title', question)
fta2 = SelectionAnswer(min=1, max=4, style='checkbox',
selections=ratings,
type='text',
other=False)
q2 = Question(identifier='selection',
content=qc2,
answer_spec=AnswerSpecification(fta2),
is_required=True)
#--------------- BUILD THE POLL FORM -------------------
question_form = QuestionForm()
question_form.append(overview)
question_form.append(q2)
#--------------- CREATE THE HIT -------------------
return mtc.create_hit(questions=question_form,
max_assignments=8,
title=title,
description=description,
keywords=keywords,
duration=60*5,
reward=poll_price)
def create_hit(href, hit):
if hit.type == "MULTIPLE_URLS": # the max_workers was not set in this form
hit.max_workers = (
hit.size1
+ hit.size2
+ hit.size3
+ hit.size4
+ hit.size5
+ hit.size6
+ hit.size7
+ hit.size8
+ hit.size9
+ hit.size10
)
hostURL = SANDBOX_HOST if hit.sandbox else HOST
qualifications = Qualifications()
if hit.hit_approval > 0:
qualifications.add(NumberHitsApprovedRequirement("GreaterThan", hit.hit_approval, False))
if hit.hit_approval_rate > 0:
qualifications.add(PercentAssignmentsApprovedRequirement("GreaterThan", hit.hit_approval_rate, False))
if hit.accepted_hit_rate > 0:
qualifications.add(PercentAssignmentsSubmittedRequirement("GreaterThan", hit.accepted_hit_rate, False))
if hit.returned_hit_rate > 0:
qualifications.add(PercentAssignmentsReturnedRequirement("GreaterThan", hit.returned_hit_rate, False))
if hit.abandoned_hit_rate > 0:
qualifications.add(PercentAssignmentsAbandonedRequirement("LessThan", hit.abandoned_hit_rate, False))
if hit.rejected_hit_rate > 0:
qualifications.add(PercentAssignmentsRejectedRequirement("LessThan", hit.rejected_hit_rate, False))
if hit.locale_qualification is not None and hit.locale_qualification != "all":
qualifications.add(LocaleRequirement("EqualTo", hit.locale_qualification, False))
connection = MTurkConnection(
aws_access_key_id=hit.aws_access_key, aws_secret_access_key=hit.aws_secret_key, host=hostURL, is_secure=True
)
return Response(
connection.create_hit(
question=ExternalQuestion(href, hit.frame_height),
lifetime=hit.lifetime,
max_assignments=hit.max_workers,
title=hit.title,
description=hit.description,
keywords=["turktime"], # TODO
reward=hit.reward,
duration=hit.duration,
approval_delay=hit.approval_delay,
qualifications=qualifications,
response_groups=["Minimal", "HITDetail", "HITQuestion", "HITAssignmentSummary"],
)
)
def create_hits(): if SANDBOX: mturk_url = 'mechanicalturk.sandbox.amazonaws.com' preview_url = 'https://workersandbox.mturk.com/mturk/preview?groupId=' else: mturk_url = 'mechanicalturk.amazonaws.com' preview_url = 'https://mturk.com/mturk/preview?groupId=' q = ExternalQuestion(external_url="https://census.stanford.edu/client/demo/maintask.html", frame_height=800) conn = MTurkConnection(aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY, host=mturk_url) keywords=['census'] for i in range(0, NUMBER_OF_HITS): create_hit_rs = conn.create_hit(question=q, lifetime=LIFETIME,max_assignments=NUMBER_OF_ASSIGNMENTS,title=TITLE, keywords=keywords,reward = REWARD, duration=60*6,approval_delay=60*60, annotation=EXPLANATION) print(preview_url + create_hit_rs[0].HITTypeId)
def post(self):
access_key = self.request.get(argument_name='accessKey',
default_value=None)
secret_key = self.request.get(argument_name='secretKey',
default_value=None)
answers_limit = self.request.get(argument_name='answersLimit',
default_value=None)
image = self.request.get(argument_name='image', default_value=None)
text = self.request.get(argument_name='text', default_value=None)
if access_key == None or secret_key == None or answers_limit == None or text == None:
self.error(400)
return
if image == None:
entity = Question(text=text)
else:
raw_image = images.Image(image_data=image)
raw_image.resize(height=440, width=440)
hit_image = raw_image.execute_transforms(
output_encoding=images.PNG)
entity = Question(image=hit_image, text=text)
entity.put()
connection = MTurkConnection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
host='mechanicalturk.amazonaws.com')
question = ExternalQuestion(
'http://www.turgleapi.com/api/hit?questionId=' +
str(entity.key().id()), 880)
result_set = connection.create_hit(
question=question,
lifetime=timedelta(minutes=90),
max_assignments=answers_limit,
title='Answer A Simple Question',
description='Can you answer a simple question for me?',
keywords='question, simple',
reward=0.01,
duration=timedelta(minutes=9),
approval_delay=timedelta(days=3))
try:
for hit in result_set:
entity.hit_id = hit.HITId
entity.put()
except:
self.error(500)
return
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(
simplejson.dumps({'questionId': entity.key().id()}, indent=4))
def create_hits():
if SANDBOX:
mturk_url = 'mechanicalturk.sandbox.amazonaws.com'
preview_url = 'https://workersandbox.mturk.com/mturk/preview?groupId='
else:
mturk_url = 'mechanicalturk.amazonaws.com'
preview_url = 'https://mturk.com/mturk/preview?groupId='
q = ExternalQuestion(external_url=HIT_URL, frame_height=800)
conn = MTurkConnection(aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY, host=mturk_url)
for i in range(0, NUMBER_OF_HITS):
create_hit_rs = conn.create_hit(question=q, lifetime=LIFETIME, max_assignments=NUMBER_OF_ASSIGNMENTS, title=TITLE, keywords=KEYWORDS, reward=REWARD, duration=DURATION, approval_delay=APPROVAL_DELAY, annotation=DESCRIPTION)
print(preview_url + create_hit_rs[0].HITTypeId)
print("HIT ID: " + create_hit_rs[0].HITId)
def create_hits():
if SANDBOX:
mturk_url = 'mechanicalturk.sandbox.amazonaws.com'
preview_url = 'https://workersandbox.mturk.com/mturk/preview?groupId='
else:
mturk_url = 'mechanicalturk.amazonaws.com'
preview_url = 'https://mturk.com/mturk/preview?groupId='
q = ExternalQuestion(external_url=HIT_URL, frame_height=800)
conn = MTurkConnection(aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY, host=mturk_url)
for i in range(0, NUMBER_OF_HITS):
create_hit_rs = conn.create_hit(question=q, lifetime=LIFETIME, max_assignments=NUMBER_OF_ASSIGNMENTS, title=TITLE, keywords=KEYWORDS, reward=REWARD, duration=DURATION, approval_delay=APPROVAL_DELAY, annotation=DESCRIPTION)
print(preview_url + create_hit_rs[0].HITTypeId)
print("HIT ID: " + create_hit_rs[0].HITId)
def run(ht, credentials, host='mechanicalturk.sandbox.amazonaws.com'):
conn = MTurkConnection(*credentials, host=host)
global tr_events
tr_events = np.array(tr_events)
for e in tr_events[[9]]:
NS = NUM_GROUPS/GROUP_SIZE
for ns in range(NS)[-2:]:
e = e.replace(' ', '_').lower()
event_url = "http://web.mit.edu/yamins/www/pq/mturk_pq_%s_%d.html" % (e, ns)
q = ExternalQuestion(external_url=event_url, frame_height=800)
create_hit_rs = conn.create_hit(hit_type=ht,
question=q,
max_assignments=15,
annotation='%s_%d' % (e, ns))
assert(create_hit_rs.status == True)
class TurkConnect:
def __init__(self, host, n=10):
mturkparams = dict(
aws_access_key_id=config.get('AWS Access', 'aws_access_key_id'),
aws_secret_access_key=config.get('AWS Access',
'aws_secret_access_key'),
host=host)
self.mtc = MTurkConnection(**mturkparams)
# Configure portal
experimentPortalURL = config.get('HIT Configuration', 'question_url')
frameheight = 600
mturkQuestion = ExternalQuestion(experimentPortalURL, frameheight)
# Qualification:
quals = Qualifications()
approve_requirement = config.getint('HIT Configuration',
'Approve_Requirement')
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if config.getboolean('HIT Configuration', 'US_only'):
quals.add(LocaleRequirement("EqualTo", "US"))
# Specify all the HIT parameters
self.paramdict = dict(
hit_type=None,
question=mturkQuestion,
lifetime=datetime.timedelta(
hours=config.getfloat('HIT Configuration', 'HIT_lifetime')),
max_assignments=config.getint('HIT Configuration',
'max_assignments'),
title=config.get('HIT Configuration', 'title'),
description=config.get('HIT Configuration', 'description'),
keywords=config.get('HIT Configuration', 'keywords'),
reward=config.getfloat('HIT Configuration', 'reward'),
duration=datetime.timedelta(
hours=config.getfloat('HIT Configuration', 'duration')),
approval_delay=None,
questions=None,
qualifications=quals)
def checkbalance(self):
return self.mtc.get_account_balance() # Tests the connection
def createhit(self):
myhit = self.mtc.create_hit(**self.paramdict)[0]
hitid = myhit.HITId
def PostHits():
mtc = MTurkConnection(aws_access_key_id=ACCESS_ID,
aws_secret_access_key=SECRET_KEY,
host=HOST)
q = ExternalQuestion(
external_url="https://paulscotti.github.io/mturk/ContLTMBlocked90",
frame_height=675)
keywords = [
'memory', 'psychology', 'game', 'attention', 'experiment', 'research'
]
title = 'Memorize the colors of objects! (Psychology Experiment, 1.5 hours)'
experimentName = 'Cont_LTM_90'
description = 'Research study involving color memory.'
pay = 9.00
qualifications = mtqu.Qualifications()
qualifications.add(
mtqu.PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 98))
qualifications.add(
mtqu.NumberHitsApprovedRequirement('GreaterThanOrEqualTo', 1000))
qualifications.add(mtqu.LocaleRequirement("EqualTo", "US"))
qualifications.add(
mtqu.Requirement(
'38XLDN1M8DBWG1FPHU43ZCVTZ4T3DT', 'DoesNotExist', '',
'DiscoverPreviewAndAccept')) # No prior workers of ours
qualifications.add(
mtqu.Requirement('2F1QJWKUDD8XADTFD2Q0G6UTO95ALH', 'Exists', '',
'DiscoverPreviewAndAccept')) # Masters only
theHIT = mtc.create_hit(
question=q,
lifetime=2 * 60 * 60, # 2 hours
max_assignments=1, #needs to be less than 10 else additional fees
title=title,
description=description,
keywords=keywords,
qualifications=qualifications,
reward=pay,
duration=180 *
60, #3 hours (HIT won't be accepted if they go over time)
approval_delay=1 * 60 * 60, # 1 hours
annotation=experimentName)
assert (theHIT.status == True)
print theHIT
print theHIT[0].HITId
def make_question(title, question, description, keywords, price, num_ppl_to_ask):
"""Make a question to send to MTurk in the correct formatting."""
ACCESS_ID = mtkey.ACCESS_KEY
SECRET_KEY = mtkey.SECRET_KEY
HOST = 'mechanicalturk.amazonaws.com'
#link to HITs: https://requester.mturk.com/mturk/manageHITs
#HOST = 'mechanicalturk.sandbox.amazonaws.com'
#link to HITs: https://requestersandbox.mturk.com/mturk/manageHITs
mtc = MTurkConnection(aws_access_key_id=ACCESS_ID,
aws_secret_access_key=SECRET_KEY,
host=HOST)
#--------------- BUILD OVERVIEW -------------------
overview = Overview()
overview.append_field('Title', title)
#--------------- BUILD QUESTION -------------------
qc2 = QuestionContent()
qc2.append_field('Title', question)
fta2 = FreeTextAnswer()
q2 = Question(identifier="comments",
content=qc2,
answer_spec=AnswerSpecification(fta2))
#--------------- BUILD THE QUESTION FORM -------------------
question_form = QuestionForm()
question_form.append(overview)
question_form.append(q2)
#--------------- CREATE THE HIT -------------------
return mtc.create_hit(title=title,
description=description,
questions=question_form,
keywords=keywords,
max_assignments=num_ppl_to_ask,
duration=60*5,
reward=price)
def post_hits(hit_info, n_sub_hits):
mtc = MTurkConnection(aws_access_key_id=access_id,
aws_secret_access_key=secret_key,
host=host)
q = ExternalQuestion(external_url=hit_info['external_url'],
frame_height=hit_info['frame_height'])
qualifications = mtqu.Qualifications()
qualifications.add(
mtqu.PercentAssignmentsApprovedRequirement(
'GreaterThanOrEqualTo', hit_info['approval_rating_cutoff']))
qualifications.add(mtqu.LocaleRequirement("EqualTo", "US"))
print('url:', hit_info['external_url'], n_sub_hits)
the_HIT = mtc.create_hit(question=q,
lifetime=hit_info['lifetime_of_experiment'],
max_assignments=n_sub_hits,
title=hit_info['title'],
description=hit_info['description'],
keywords=hit_info['keywords'],
qualifications=qualifications,
reward=hit_info['payment_for_experiment'],
duration=hit_info['duration_of_experiment'],
approval_delay=hit_info['approval_delay'],
annotation=hit_info['experiment_name'])
assert (the_HIT.status == True)
hit_info['hit_id'] = the_HIT[0].HITId
hit_url = "{}{}".format(base_url, the_HIT[0].HITTypeId)
hit_info['hit_url'] = hit_url
record_name = 'HIT_submission_records_%s.npy' % (context)
if record_name not in os.listdir(os.getcwd()):
turk_info = {}
else:
turk_info = np.load(record_name).item()
key_name = 'submission_%d' % len(turk_info.keys())
turk_info[key_name] = hit_info
np.save(record_name, turk_info)
print('HIT_ID:', the_HIT[0].HITId, 'key_name', key_name,
"\nwhich you can see here:", hit_url)
class TurkConnect:
def __init__(self, host, n=10):
mturkparams = dict(
aws_access_key_id = config.get( 'AWS Access', 'aws_access_key_id' ),
aws_secret_access_key = config.get( 'AWS Access', 'aws_secret_access_key' ),
host=host)
self.mtc = MTurkConnection( **mturkparams )
# Configure portal
experimentPortalURL = config.get( 'HIT Configuration', 'question_url' )
frameheight = 600
mturkQuestion = ExternalQuestion( experimentPortalURL, frameheight )
# Qualification:
quals = Qualifications();
approve_requirement = config.getint('HIT Configuration',
'Approve_Requirement')
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if config.getboolean('HIT Configuration', 'US_only'):
quals.add( LocaleRequirement("EqualTo", "US") )
# Specify all the HIT parameters
self.paramdict = dict(
hit_type = None,
question = mturkQuestion,
lifetime = datetime.timedelta(hours=config.getfloat('HIT Configuration', 'HIT_lifetime')),
max_assignments = config.getint('HIT Configuration', 'max_assignments'),
title = config.get('HIT Configuration', 'title'),
description = config.get('HIT Configuration', 'description'),
keywords = config.get('HIT Configuration', 'keywords'),
reward = config.getfloat('HIT Configuration', 'reward'),
duration = datetime.timedelta(
hours=config.getfloat('HIT Configuration', 'duration')),
approval_delay = None,
questions = None,
qualifications = quals
)
def checkbalance(self):
return self.mtc.get_account_balance() # Tests the connection
def createhit(self):
myhit = self.mtc.create_hit( **self.paramdict )[0]
hitid = myhit.HITId
def createHITs( savePath = '/home/ubuntu/amt_guis/cocoa_depth/hits/coco/', hit_name = '' ):
setIds = range( STARTING_HIT, STARTING_HIT + NUMBER_HITS ) #[1,3,4,6,8,9]
#setIds.extend( range( STARTING_HIT, STARTING_HIT + NUMBER_HITS ) )
mtc = MTurkConnection( host = _host )
hits = []
hitType = getHITType()[0]
hitLifeTime = 60 * 60 * 24 * 7
count = 0
for setId in setIds:
external_url = HOST_DOMAIN + '/cocoa_depth/coco/' + str( setId )
print external_url
q = ExternalQuestion( external_url=external_url, frame_height=1000 )
hit = mtc.create_hit(hit_type=hitType.HITTypeId,
question=q,
max_assignments = NUMBER_HIT_ASSIGNMENTS,
lifetime=hitLifeTime)
hits.append( hit[0] )
count += 1
if count >= MAX_HITS:
# pass is just a place holder
pass
if savePath == '':
if 'MTURK_STORAGE_PATH' in os.environ:
savePath = os.environ['MTURK_STORAGE_PATH']
else:
savePath == './'
if hit_name == '':
hit_name = 'cocoa_test_' + str(NUMBER_HITS) + '_DepthHITS'
time_stamp = time.strftime( "%Y-%m-%d_%H-%M-%S" )
filename = os.path.join( savePath, hit_name + '_' + time_stamp + ".pkl")
print "Storing created hit data at %s" % (filename)
with open(filename, 'wb') as f:
pickle.dump( hits, f )
def createHit(self,text):
mtc = MTurkConnection(aws_access_key_id=self.ACCESS_ID,
aws_secret_access_key=self.SECRET_KEY,
host=self.HOST)
overview = Overview()
overview.append_field('Title','Rate this Tweet! (WARNING: This HIT may contain adult content. Worker discretion is advised.)')
qc = QuestionContent()
qc.append_field('Title','Please read the following: ')
qc.append_field('Text', "\"" + text + "\"" +'\n')
qc.append_field('Text','After reading the above tweet, please choose the mood which matches best with the content.')
selectionAns = SelectionAnswer(min = 1, max = 1, style='radiobutton',
selections = self.moodList,
type="text",
other = False)
q = Question(identifier='mood',
content = qc,
answer_spec = AnswerSpecification(selectionAns),
is_required=True)
qc2 = QuestionContent()
qc2.append_field('Text','Choose an intesity for the mood chosen.\n (1 - lowest | 10 - highest)')
selectionAns2 = SelectionAnswer(min = 1, max = 1, style='radiobutton', #dropdown
selections = self.moodIntensity,
type="text",
other = False)
q2 = Question(identifier='intensity',
content = qc2,
answer_spec = AnswerSpecification(selectionAns2),
is_required=True)
question_form = QuestionForm()
question_form.append(overview)
question_form.append(q)
question_form.append(q2)
my_hit= mtc.create_hit(questions=question_form,
max_assignments=1,
title='Rate this Tweet! (WARNING: This HIT may contain adult content. Worker discretion is advised.)',
description='Easy! Read a single tweet and rate choose a mood and intensity',
keywords='rate, tweet',
duration = 60*5, #60 seconds * 5
reward = 0.01)
return my_hit[0].HITTypeId
class mTurk:
def __init__(self):
self.ACCESS_ID = os.environ["ACCESS_KEY_ID"]
self.SECRET_KEY = os.environ["SECRET_ACCESS_KEY"]
self.HOST = "mechanicalturk.sandbox.amazonaws.com"
self.title = "Please respond as a therapist to this question"
self.description = "Read this diary entry and give a thoughtful advice to this person"
self.keywords = "diary,therapist,friend,advice"
self.connectMTurk()
def connectMTurk(self):
self.mtc = MTurkConnection(
aws_access_key_id=self.ACCESS_ID, aws_secret_access_key=self.SECRET_KEY, host=self.HOST
)
# print(self.mtc.get_account_balance())
def buildOverview(self):
self.overview = Overview()
self.overview.append_field("Title", "Deard Response")
self.overview.append(FormattedContent("<h2>DearD User Post</h2>"))
def buildQuestion(self, diaryEntry):
self.qc = QuestionContent()
self.qc.append_field("Title", diaryEntry)
self.fta = FreeTextAnswer()
self.q1 = Question(identifier="comments", content=self.qc, answer_spec=AnswerSpecification(self.fta))
def buildQuestionForm(self):
self.question_form = QuestionForm()
self.question_form.append(self.overview)
self.question_form.append(self.q1)
def createHit(self, diaryEntry):
self.buildOverview()
self.buildQuestion(diaryEntry)
self.buildQuestionForm()
id = self.mtc.create_hit(
questions=self.question_form,
max_assignments=1,
title=self.title,
description=self.description,
duration=60 * 5,
reward=0.50,
)
return id[0].HITId
def create_disambig_hit():
connection = MTurkConnection(aws_access_key_id=settings.MTURK_ACCESS_KEY, aws_secret_access_key=settings.MTURK_SECRET_KEY,
host=MTURK_HOST)
question = ExternalQuestion('https://kitt.cl.uzh.ch/kitt/mantracrowd-static/disambiguation.html', 500)
result = connection.create_hit(
question=question,
title='Disambiguation survey',
description='Answer questions about word meanings',
reward=Price(0.30),
keywords='language, English, text, linguistics',
max_assignments=3,
# 3 days
lifetime=3*24*60*60,
annotation="Disambiguation survey"
)
return result[0]
class AMTClient:
def __init__(self, sandbox=True):
if sandbox:
self.host = "mechanicalturk.sandbox.amazonaws.com"
self.external_submit_endpoint = "https://workersandbox.mturk.com/mturk/externalSubmit"
else:
self.host = "mechanicalturk.amazonaws.com"
self.external_submit_endpoint = "https://www.mturk.com/mturk/externalSubmit"
self.base_url = "https://labelmelite.csail.mit.edu"
self.connection = MTurkConnection(
aws_access_key_id=amt_config.AWS_ACCESS_KEY_ID,
aws_secret_access_key=amt_config.AWS_SECRET_ACCESS_KEY,
host=self.host)
def create_hit(self, job_id, bundle_id, hitType="yesno"):
params_to_encode = {
"job_id": job_id,
"bundle_id": bundle_id,
"host": self.external_submit_endpoint
}
encoded_url = encode_get_parameters(
self.base_url + "/amt_{}".format(hitType), params_to_encode)
# print(encoded_url)
if hitType == "yesno":
props = hit_properties.YesNoHitProperties
elif hitType == "edit":
props = hit_properties.EditHitProperties
else:
raise Exception("Hit type not implemented")
create_hit_result = self.connection.create_hit(
title=props["title"],
description=props["description"],
keywords=props["keywords"],
duration=props["duration"],
max_assignments=props["max_assignments"],
question=ExternalQuestion(encoded_url, props["frame_height"]),
reward=Price(amount=props["reward"]),
# Determines information returned by certain API methods.
response_groups=('Minimal', 'HITDetail'),
qualifications=props["qualifications"])
def post(self):
access_key = self.request.get(argument_name='accessKey', default_value=None)
secret_key = self.request.get(argument_name='secretKey', default_value=None)
answers_limit = self.request.get(argument_name='answersLimit', default_value=None)
image = self.request.get(argument_name='image', default_value=None)
text = self.request.get(argument_name='text', default_value=None)
if access_key == None or secret_key == None or answers_limit == None or text == None:
self.error(400)
return
if image == None:
entity = Question(text=text)
else:
raw_image = images.Image(image_data=image)
raw_image.resize(height=440, width=440)
hit_image = raw_image.execute_transforms(output_encoding=images.PNG)
entity = Question(image=hit_image, text=text)
entity.put()
connection = MTurkConnection(aws_access_key_id=access_key,
aws_secret_access_key=secret_key,
host='mechanicalturk.amazonaws.com')
question = ExternalQuestion('http://www.turgleapi.com/api/hit?questionId=' + str(entity.key().id()),
880)
result_set = connection.create_hit(question=question,
lifetime=timedelta(minutes=90),
max_assignments=answers_limit,
title='Answer A Simple Question',
description='Can you answer a simple question for me?',
keywords='question, simple',
reward=0.01,
duration=timedelta(minutes=9),
approval_delay=timedelta(days=3))
try:
for hit in result_set:
entity.hit_id = hit.HITId
entity.put()
except:
self.error(500)
return
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(simplejson.dumps({'questionId': entity.key().id()}, indent=4))
def SubmitHIT(self, sandbox = 'false'):
"""
Constructs a HIT from the HITGenerator's attributes, registers it with Amazon, and returns the HITId as a unicode string.
If the sandbox flag is set to true then the hit will be registered with the Sandbox, otherwise it is registered to AWS
directly. All of the necessary data must have been submitted during the HITGenerator's initiation.
"""
if sandbox is 'true':
self.host = 'mechanicalturk.sandbox.amazonaws.com'
conn = MTurkConnection(host = self.host, aws_access_key_id = self.AWS_KEY, aws_secret_access_key = self.AWS_SECRET)
answer_specification = AnswerSpecification(SelectionAnswer(style = self.answer_style, selections = self.answer_options))
overview = Overview()
overview.append('Title', 'Translate these sentences')
overview.append('FormattedContent', overview_content)
qc = QuestionContent()
the_text = "Some arabic Words."
qc.append('FormattedContent', u'<table><tr><td></td><td align="right" width="538">%s</td></tr></table>' % the_text)
# construct an answer field
fta = FreeTextAnswer()
ansp = AnswerSpecification(fta)
ql = []
for q in self.question_list:
ql.append(Question(identifier=q[1],
content=q[0],
answer_spec=ansp))
#q = Question(identifier=str(uuid.uuid4()),
# content=qc,
# answer_spec=ansp)
# build question form with question list
qf = QuestionForm(ql, overview=overview)
self.hit_response = conn.create_hit(question = qf,
lifetime = self.lifetime,
max_assignments = self.assignment_count,
title = self.title,
description = self.description,
keywords = self.keywords,
reward = self.reward,
)
# Returns the HITId as a unicode string
# self.HITId = self.hit_response.HITId
# return self.HITId
return self.hit_response
def createHITs():
hostDomain = 'https://aws-ec2-cit-mronchi.org'
setIds = [1, 3, 4, 6, 8, 9]
setIds.extend(range(_STARTING_HIT, _STARTING_HIT + _NUMBER_HITS))
mtc = MTurkConnection(host=_host)
hits = []
hitType = getHITType()[0]
max_assignments = _NUMBER_HIT_ASSIGNMENTS
lifetime = 60 * 60 * 24 * 5
max_hits = 2694
count = 0
for setId in setIds:
external_url = hostDomain + '/mturk_interactions/' + str(setId)
print external_url
q = ExternalQuestion(external_url=external_url, frame_height=1000)
hit = mtc.create_hit(hit_type=hitType.HITTypeId,
question=q,
max_assignments=max_assignments,
lifetime=lifetime)
hits.append(hit[0])
count += 1
if count >= max_hits:
pass
if 'MTURK_STORAGE_PATH' in os.environ:
time_stamp = time.strftime("%Y-%m-%d_%H-%M-%S")
hit_name = 'cocoa_5000_' + str(_NUMBER_HITS) + '_InteractionHITS'
filename = os.path.join(os.environ['MTURK_STORAGE_PATH'],
hit_name + '_' + time_stamp + ".pkl")
print "Storing created hit data at %s" % (filename, )
with open(filename, 'wb') as f:
pickle.dump(hits, f)
else:
print "WARNING: MTURK_STORAGE_PATH not set in env. Unable to save hit data."
def PostHits(pay, lifetime, max_assignments, exp):
mtc = MTurkConnection(aws_access_key_id=aws_access_key_id,aws_secret_access_key=aws_secret_access_key,host=HOST)
q = ExternalQuestion(external_url = "https://davebraun.org/dissertation/experiments/production/" + exp +"/", frame_height=675)
keywords = ['attention', 'psychology', 'experiment', 'research']
title = 'A Decision Making Experiment'
experimentName = 'Decision Making Experiment' ## this is NOT what it ends up getting called on my server
description = 'This HIT will take about 30 mins to complete. All HITS in this batch are the same, and you will only be able to perform one of the HITS in this batch.'
qualifications = mtqu.Qualifications()
qualifications.add(mtqu.PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 90))
qualifications.add(mtqu.LocaleRequirement("EqualTo", "US"))
#qualifications.add(mtqu.Requirement("2Z046OQ1SNQQREGXAFSQPCNR1605PN"))
theHIT = mtc.create_hit(question=q,
lifetime=lifetime,
max_assignments=max_assignments,
title=title,
description=description,
keywords=keywords,
qualifications=qualifications,
reward=pay,
duration=120 * 60, # 120 minutes
approval_delay=2* 24 * 60 * 60, # the number of seconds after an assignment is submitted is it automatically approved unless explicitly rejected
## the norm is to try to keep this under 7 days, many requesters approve in less than 3 days
annotation=experimentName)
assert(theHIT.status == True)
print theHIT
hit_type_id = theHIT[0].HITId
print hit_type_id + '\n'
#print "https://workersandbox.mturk.com/mturk/preview?groupId={}".format(hit_type_id)
'''
f = open('hit_id.txt', 'w')
f.write(hit_type_id)
f.close()
'''
return hit_type_id
def startHit(self,Url,index):
#--------------- CREATE THE HIT -------------------
mtc = MTurkConnection(aws_access_key_id=ACCESS_ID, aws_secret_access_key=SECRET_KEY, host=HOST)
title = 'Write subtitles for a short movie clip ' + str(index)
description_notag = ('Watch a short clip and write down the dialogue between characters.'
'If there is no dialogue, but there are prominent sounds like music or a door closing,'
'then describe it briefly in parentheses, like (door closing)'
'If there is no prominent sound in the clip, just type a space in the box.'
'Please watch the clip more than once if you need to, we want the most accurate subtitles!')
description_tag = ('<b>Watch a short clip and write down the dialogue between characters.<br/><br/><br/>'
'If there is no dialogue, but there are prominent sounds like music or a door closing,'
'then describe it briefly in parentheses, like (door closing)<br/><br/><br/>'
'If there is no prominent sound in the clip, just type a space in the box.<br/><br/><br/>'
'Please watch the clip more than once if you need to, we want the most accurate subtitles!</b>')
keywords = 'video, transcript, translate'
salary = self.ui.payTextBox.toPlainText()
#--------------- BUILD OVERVIEW -------------------
overview = Overview()
overview.append_field('Title', 'Transcribe a short video clip')
#--------------- BUILD MOVIE CLIP -------------------
qc1 = QuestionContent() # I will modify this to embed some codes indicating time frame
qc1.append_field('FormattedContent','<![CDATA[<br/>'+description_tag+'<br/><br/><br/><iframe width="600" height="400" src="'+ Url.replace('&','&') + '">No video supported</iframe>]]>')
fta1 = FreeTextAnswer()
q1 = Question(identifier='Answer',content=qc1,answer_spec=AnswerSpecification(fta1),is_required=True)
#looking for start time frame
start = Url.find('start=') +6
end = Url.find('&end',start)
time_frame = [('Time',Url[start:end])]
qc2 = QuestionContent()
qc2.append_field('Title',' ')
fta2 = SelectionAnswer(min=1,max=1, style='dropdown',selections=time_frame,type='text',other=False)
q2 = Question(identifier='Time code',content=qc2,answer_spec=AnswerSpecification(fta2),is_required=False)
#--------------- BUILD THE QUESTION FORM ------------------#
question_form = QuestionForm()
question_form.append(overview)
question_form.append(q1)
question_form.append(q2)
my_hit = mtc.create_hit(questions=question_form, lifetime=60*60*12, max_assignments=1, title=title, description=description_notag, keywords=keywords, duration = 10*60,reward=salary)
print "Hit type id 1: %s" % my_hit[0].HITTypeId
def create_hit(hit_type, problem, params={}):
# create a Hit object
hit = Hit(hit_id='?',
hit_type=hit_type,
problem=problem,
params=json.dumps(params),
title=hit_type.title % params,
description=hit_type.description % params,
body=hit_type.body % params)
hit.save()
# post a HIT on Mechanical Turk using boto
q = ExternalQuestion(external_url=settings.URL_ROOT +
hit.get_absolute_url(),
frame_height=800)
conn = MTurkConnection(
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
host=settings.AWS_HOST)
# remove commas from the keywords if they exist
keywords = [k.replace(',', '') for k in hit_type.keywords.split()]
create_hit_rs = conn.create_hit(question=q,
lifetime=hit_type.lifetime,
max_assignments=hit_type.max_assignments,
keywords=keywords,
reward=hit_type.payment,
duration=hit_type.duration,
approval_delay=hit_type.approval_delay,
title=hit.title,
description=hit.description,
annotation=` hit_type `)
assert (create_hit_rs.status == True)
# set the new HIT ID to be the hit_id for the new row.
hit.hit_id = create_hit_rs.HITId
hit.save()
return hit
def test():
q = ExternalQuestion(external_url="http://websort.net/s/F3481C",
frame_height=800)
conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com')
keywords = ['boto', 'test', 'doctest']
create_hit_rs = conn.create_hit(
question=q,
lifetime=60 * 65,
max_assignments=2,
title="Boto External Question Test",
keywords=keywords,
reward=0.05,
duration=60 * 6,
approval_delay=60 * 60,
annotation='An annotation from boto external question test',
response_groups=[
'Minimal',
'HITDetail',
'HITQuestion',
'HITAssignmentSummary',
])
assert (create_hit_rs.status == True)
def ManualReward():
mtc = MTurkConnection(aws_access_key_id=ACCESS_ID,
aws_secret_access_key=SECRET_KEY,
host=HOST)
q = ExternalQuestion(
external_url="https://paulscotti.github.io/mturk/RewardPage",
frame_height=675)
keywords = ['compensation', 'experiment', 'research']
title = 'Compensation for Cont_LTM color experiment'
experimentName = 'Cont_LTM_Reward'
description = 'Compensating a previous worker of the Cont_LTM study.'
pay = 10.27
#for adding new people
#mtc.assign_qualification('35GMP5037Q9KDB285EZ1DGC38JE39C','A1N1EF0MIRSEZZ');
qualifications = mtqu.Qualifications()
qualifications.add(
mtqu.Requirement('35GMP5037Q9KDB285EZ1DGC38JE39C', 'Exists', '',
'DiscoverPreviewAndAccept'))
rewardHIT = mtc.create_hit(
question=q,
lifetime=12 * 60 * 60, #12 hours
max_assignments=1, #needs to be less than 10 else additional fees
title=title,
description=description,
keywords=keywords,
qualifications=qualifications,
reward=pay,
duration=10 *
60, #10 minutes (HIT won't be accepted if they go over time)
approval_delay=12 * 60 * 60, # 12 hours
annotation=experimentName)
assert (rewardHIT.status == True)
print rewardHIT
print rewardHIT[0].HITId
def PostHits():
mtc = MTurkConnection(aws_access_key_id=ACCESS_ID,
aws_secret_access_key=SECRET_KEY,
host=HOST)
# q = ExternalQuestion(external_url = "https://timbrady.org/turk/monthnumber/", frame_height=675)
q = ExternalQuestion(
external_url=
"https://github.com/mice-annotator/mice-annotator.github.io",
frame_height=675)
keywords = ['mice', 'bounding box']
title = 'I love mice so much!'
experimentName = "mice_bounding_box"
description = 'Draw bounding box for mice in the frame.'
pay = 0.02
qualifications = mtqu.Qualifications()
qualifications.add(
mtqu.PercentAssignmentsApprovedRequirement('GreaterThanOrEqualTo', 90))
qualifications.add(mtqu.LocaleRequirement("EqualTo", "US"))
#qualifications.add(mtqu.Requirement("2Z046OQ1SNQQREGXAFSQPCNR1605PN"))
theHIT = mtc.create_hit(
question=q,
lifetime=10 * 60 * 60, # 10 hours
max_assignments=3,
title=title,
description=description,
keywords=keywords,
qualifications=qualifications,
reward=pay,
duration=120 * 60, # 120 minutes
approval_delay=5 * 60 * 60, # 5 hours
annotation=experimentName)
assert (theHIT.status == True)
print(theHIT)
print(theHIT[0].HITId)
def main():
# parse arguments
parser = argparse.ArgumentParser()
parser.add_argument('keys_file')
parser.add_argument('--production', default=False, action='store_true')
args = parser.parse_args()
access_key_id, secret_key, host = mturk.get_keys_and_host(args.keys_file, args.production)
connection = MTurkConnection(aws_access_key_id=access_key_id,
aws_secret_access_key=secret_key,
host=host)
question = ExternalQuestion(
external_url='https://mapmill-pilot.ccs.neu.edu:3000/', # URL to serve HIT
frame_height=800 # height of frame
)
reward=Price(
amount=0.10 # reward for HIT completion
)
create_hit_result = connection.create_hit(
title='Disaster Area Map Image Labeling',
description='Label and categorize aerial images of disaster areas',
keywords=['image', 'label', 'labeling', 'categorize', 'map'],
max_assignments=10,
lifetime=datetime.timedelta(days=5), # time HIT is available
duration=datetime.timedelta(minutes=30), # time worker has to complete HIT once accepted
question=question,
reward=reward,
response_groups=('Minimal', 'HITDetail')
)
def PublishTasks(hitNum, maxAssignments):
# sandbox in which to simulate: mechanicalturk.sandbox.amazonaws.com
# real environment: mechanicalturk.amazonaws.com
mtc = MTurkConnection(host='mechanicalturk.amazonaws.com')
# print mtc.APIVersion
# print mtc.get_account_balance()
# print mtc.get_reviewable_hits()
# print mtc.get_all_hits()
#--------------- BUILD OVERVIEW -------------------
# jbragg: Modified maximum reward description.
#title = '(Maximum reward possible: $70) Identify the relation between two entities in English sentences'
title = 'Identify the relation between two entities in English sentences'
#description = 'You will be given English sentences in which your task is to identify the relation between two designated entities. Your reward will depend on how many questions you have answered. The maximum reward you can earn is $70.'
description = 'You will be given English sentences in which your task is to identify the relation between two designated entities. Your reward will depend on how many questions you have answered. The maximum reward you can earn is approximately $5.'
keywords = 'English sentences, relation identification'
ratings = [('Very Bad', '-2'), ('Bad', '-1'), ('Not bad', '0'),
('Good', '1'), ('Very Good', '1')]
#--------------- BUILD OVERVIEW -------------------
overview = Overview()
overview_title = 'Exercise link (please copy the link and paste it in your browser if it cannot be opened directly.)'
link = '<a target="_blank"' ' href="http://128.208.3.167:3000/mturk">' ' http://128.208.3.167:3000/mturk</a>'
# jbragg: Commented out long-term bonus.
instructions = '<p>Instructions:</p><ul><li>You will be presented with sentences that have a person and a location highlighted.</li><li>Your task is to determine which of the 5 designated relations are expressed between the person and location.</li><li>You'll get paid $0.50 after each successful set of 20 questions<!-- -- plus a bonus of $2.00 after every 10 batches (equal to 200 questions)-->.</li><li>We know the correct answers to some of these sentence questions, and you can stay if you get these questions right.</li><li>You can start by going to the external link above now. After you finish all the questions, you will be provided with a confirm code, used for authentication and determining the appropriate amount of money as the payment.</li><li>In very rare cases where the website crashes, you could click backward and then forward on your browser to reload the question. It won\'t affect the payment because all the questions you have answered are recorded, on which the amount of payment is based. So please don\'t worry about that.</li></ul>'
overview_content = link + instructions
overview.append_field('Title', overview_title)
overview.append(FormattedContent(overview_content))
#--------------- BUILD QUESTION 1 -------------------
qc1 = QuestionContent()
qc1.append_field('Title', 'How looks the design ?')
fta1 = SelectionAnswer(min=1,
max=1,
style='dropdown',
selections=ratings,
type='text',
other=False)
q1 = Question(identifier='design',
content=qc1,
answer_spec=AnswerSpecification(fta1),
is_required=True)
#--------------- BUILD QUESTION 2 -------------------
qc2 = QuestionContent()
qc2.append_field(
'Title',
'Confirm code \n1. The code will be provided to you as you finish from the exercise link. \n2. The code will be verified before paying. \n3. By the end of every 20 questions (as a batch), You can choose to finish and get a confirm code, or continue.'
)
fta2 = FreeTextAnswer()
q2 = Question(identifier="Confirm_code",
content=qc2,
answer_spec=AnswerSpecification(fta2))
#--------------- BUILD THE QUESTION FORM -------------------
question_form = QuestionForm()
question_form.append(overview)
# question_form.append(q1)
question_form.append(q2)
#--------------- CREATE HITs -------------------
HIT_num = hitNum
for i in range(HIT_num):
# max_assignments: how many replicas this HIT has
mtc.create_hit(questions=question_form,
max_assignments=maxAssignments,
title=title,
description=description,
keywords=keywords,
duration=60 * 60 * 10,
reward=0.50)
ONE_DAY = ONE_HOUR * 24
ONE_WEEK = ONE_HOUR * 24 * 7
hits = defaultdict(lambda : defaultdict())
# <codecell>
for url in urls:
hit = mtc.create_hit(
annotation=None,
approval_delay=1,
# approval_delay=ONE_DAY,
description=description,
duration=ONE_HOUR,
keywords=keywords,
lifetime=ONE_WEEK,
max_assignments=1,
qualifications=qualifications,
questions=make_question(url, selections),
response_groups=[
'HITAssignmentSummary', 'HITDetail', 'HITQuestion', 'Minimal'],
reward=0.02,
title=title,
)
hits[hit[0].HITId] = {'url': url}
# <codecell>
print 'Created Hits: '
for hit in hits.keys():
print hit
def startHit(self, Url, index):
#--------------- CREATE THE HIT -------------------
mtc = MTurkConnection(aws_access_key_id=ACCESS_ID,
aws_secret_access_key=SECRET_KEY,
host=HOST)
title = 'Write subtitles for a short movie clip ' + str(index)
description_notag = (
'Watch a short clip and write down the dialogue between characters.'
'If there is no dialogue, but there are prominent sounds like music or a door closing,'
'then describe it briefly in parentheses, like (door closing)'
'If there is no prominent sound in the clip, just type a space in the box.'
'Please watch the clip more than once if you need to, we want the most accurate subtitles!'
)
description_tag = (
'<b>Watch a short clip and write down the dialogue between characters.<br/><br/><br/>'
'If there is no dialogue, but there are prominent sounds like music or a door closing,'
'then describe it briefly in parentheses, like (door closing)<br/><br/><br/>'
'If there is no prominent sound in the clip, just type a space in the box.<br/><br/><br/>'
'Please watch the clip more than once if you need to, we want the most accurate subtitles!</b>'
)
keywords = 'video, transcript, translate'
salary = self.ui.payTextBox.toPlainText()
#--------------- BUILD OVERVIEW -------------------
overview = Overview()
overview.append_field('Title', 'Transcribe a short video clip')
#--------------- BUILD MOVIE CLIP -------------------
qc1 = QuestionContent(
) # I will modify this to embed some codes indicating time frame
qc1.append_field(
'FormattedContent', '<![CDATA[<br/>' + description_tag +
'<br/><br/><br/><iframe width="600" height="400" src="' +
Url.replace('&', '&') + '">No video supported</iframe>]]>')
fta1 = FreeTextAnswer()
q1 = Question(identifier='Answer',
content=qc1,
answer_spec=AnswerSpecification(fta1),
is_required=True)
#looking for start time frame
start = Url.find('start=') + 6
end = Url.find('&end', start)
time_frame = [('Time', Url[start:end])]
qc2 = QuestionContent()
qc2.append_field('Title', ' ')
fta2 = SelectionAnswer(min=1,
max=1,
style='dropdown',
selections=time_frame,
type='text',
other=False)
q2 = Question(identifier='Time code',
content=qc2,
answer_spec=AnswerSpecification(fta2),
is_required=False)
#--------------- BUILD THE QUESTION FORM ------------------#
question_form = QuestionForm()
question_form.append(overview)
question_form.append(q1)
question_form.append(q2)
my_hit = mtc.create_hit(questions=question_form,
lifetime=60 * 60 * 12,
max_assignments=1,
title=title,
description=description_notag,
keywords=keywords,
duration=10 * 60,
reward=salary)
print "Hit type id 1: %s" % my_hit[0].HITTypeId
EQ = ExternalQuestion(external_url, frame_height)
title = "UFAL - Test an automated tourist information service (it takes 2 minutes on average)"
description = "Rate a speech enabled tourist infomation system."
keywords = "speech,test,voice,evaluation,call,conversation,dialog,dialogue,chat,quick,fast,mark,rate"
reward = 0.20
max_assignments = 1
duration = datetime.timedelta(minutes=100)
lifetime = datetime.timedelta(days=7)
approval_delay = datetime.timedelta(days=1)
q1 = PercentAssignmentsApprovedRequirement('GreaterThan', 95)
q2 = NumberHitsApprovedRequirement('GreaterThan', 500)
q3 = LocaleRequirement('EqualTo', 'US')
qualifications = Qualifications([q1, q2, q3])
response_groups = None
print "Submiting HITs"
conn = mturk.get_connection()
for n in range(n_hits):
print "Hit #", n
conn.create_hit(
question=EQ, lifetime=lifetime, max_assignments=max_assignments,
title=title, description=description, keywords=keywords, reward=reward,
duration=duration, approval_delay=approval_delay,
qualifications=qualifications, response_groups=response_groups)
class HitCreator():
def __init__(self):
if settings.IS_DEV_ENV or settings.USE_AMT_SANDBOX:
HOST = 'mechanicalturk.sandbox.amazonaws.com'
else:
HOST = 'mechanicalturk.amazonaws.com'
self.connection = MTurkConnection(
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY,
host=HOST)
def createHitFrom(self, audioSnippet, hitType, numIncorrectWords=None):
if hitType == "fix":
suffix = "fixHIT"
# half cent per incorrect word, up to eight words
assert isinstance(numIncorrectWords, int)
amount = max(min(.05, numIncorrectWords*.005), .02)
elif hitType == "check":
suffix = "checkHIT"
amount = 0.05
else:
assert False
if settings.IS_DEV_ENV:
baseurl = 'https://localhost:5000/hit/' + suffix
else:
baseurl = "https://transcroobie.herokuapp.com/hit/" + suffix
title = "Transcribe a short audio clip."
description = "Transcribe the audio. Words may be cut off at the beginning"\
" or end of the segment. Do not worry about correctly"\
" transcribing these words."
keywords = ["transcription"]
frame_height = 800
thisDocUrl = baseurl + "?docId=" + str(audioSnippet.pk)
questionform = ExternalQuestion(thisDocUrl, frame_height)
resultSet = self.connection.create_hit(
title=title,
description=description,
keywords=keywords,
max_assignments=1,
question=questionform,
reward=Price(amount=amount),
response_groups=('Minimal', 'HITDetail'), # I don't know what response groups are
)
assert len(resultSet) == 1
audioSnippet.activeHITId = resultSet[0].HITId
audioSnippet.save()
def deleteHit(self, hitID):
try:
self.connection.disable_hit(hitID)
except MTurkRequestError as e:
print "HIT already deleted", e
def deleteAllHits(self):
allHits = [hit for hit in self.connection.get_all_hits()]
for hit in allHits:
print "Disabling hit ", hit.HITId
self.deleteHit(hit.HITId)
def processHit(self, questionFormAnswers):
# Process each HIT only once. This function will set activeHITId to ""
# to let you know that the HIT is completed and processed.
hitType = None
response = None
audioSnippet = None
fixWords = {}
for questionFormAnswer in questionFormAnswers:
if questionFormAnswer.qid == "asFileId":
asFileId = questionFormAnswer.fields[0]
audioSnippet = get_object_or_404(AudioSnippet, pk = asFileId)
elif questionFormAnswer.qid == "fixedHITResult":
hitType = "fix"
response = None # need to look at word_%d based on audiosnippet
elif questionFormAnswer.qid.startswith("word_"):
fixWords[questionFormAnswer.qid] = questionFormAnswer.fields[0]
elif questionFormAnswer.qid == "checkedHITResult":
hitType = "check"
responseStr = questionFormAnswer.fields[0]
response = [val == 'true' for val in responseStr.split(',')]
numIncorrectWords = 0
if hitType == "fix":
# Get the list of words marked incorrect, and count them
incorrectWords = audioSnippet.incorrectWords['bools'][-1]
numIncorrectWords = len(incorrectWords)-sum(incorrectWords)
# Get the last prediction to interpret incorrectWords
prediction = audioSnippet.predictions[-1].split()
# Convert the last prediction to what was actually sent to
# the user
predictionSpaced = transcriptWithSpacesAndEllipses(prediction)
assert len(incorrectWords) == len(predictionSpaced)
words, isCorrect = combineConsecutiveDuplicates(predictionSpaced,
incorrectWords)
response = ""
for i in xrange(len(words)):
if not isCorrect[i]:
response += fixWords["word_" + str(i)] + " "
else:
# Only add punctuation (" ") and ellipses if marked incorrect
word = words[i]
if word.isspace() or word == "":
continue
elif i == 0 and word.startswith("..."):
word = word[3:] # remove initial ellipses
elif i == len(words)-1 and word.endswith("..."):
word = word[:-3] # remove trailing ellipses
response += word.strip() + " "
audioSnippet.predictions.append(response)
# Always do a check after a fix
completionStatus = CompletionStatus.incomplete
else:
audioSnippet.incorrectWords['bools'].append(response)
completionStatus = self.getCompletionStatus(audioSnippet, response)
if completionStatus == CompletionStatus.correct:
audioSnippet.hasBeenValidated = True
audioSnippet.isComplete = True
elif completionStatus == CompletionStatus.givenup:
audioSnippet.hasBeenValidated = False
audioSnippet.isComplete = True
audioSnippet.activeHITId = ""
if completionStatus == CompletionStatus.incomplete:
if hitType == "check":
# CHECK task complete. Create a FIX task (since not # hasBeenValidated)
self.createHitFrom(audioSnippet, 'fix', numIncorrectWords)
elif hitType == "fix":
# FIX task complete. Create a CHECK task.
self.createHitFrom(audioSnippet, 'check')
audioSnippet.save()
def getCompletionStatus(self, audioSnippet, response):
# only callwhen all hitTypes == "check"
# returns a CompletionStatus
MAX_NUM_PREDICTIONS = 2
completionStatus = CompletionStatus.incomplete
if all(response):
completionStatus = CompletionStatus.correct
elif len(audioSnippet.predictions) > MAX_NUM_PREDICTIONS:
completionStatus = CompletionStatus.givenup
return completionStatus
def processHits(self, doc):
""" Returns whether or not the doc had a newly-completed HIT
which was processed. """
assert not doc.completeTranscript
audioSnippets = doc.audioSnippets.order_by('id')
newHITCompleted = False
assignments = []
for audioSnippet in audioSnippets:
hitID = audioSnippet.activeHITId
if not hitID: continue
try:
hit = self.connection.get_hit(hitID)
except MTurkRequestError as e:
logger.error("Perhaps this HIT no longer exists: " + str(e))
continue
asgnForHit = self.connection.get_assignments(hit[0].HITId)
if asgnForHit:
# Hit is ready. Get the data.
for asgn in asgnForHit:
assignments.append(asgn)
questionFormAnswers = asgn.answers[0]
self.processHit(questionFormAnswers)
newHITCompleted = True
statuses = [a.isComplete for a in audioSnippets]
if all([a.hasBeenValidated for s in statuses]) or \
all([a.isComplete for a in audioSnippets]):
# Note: if the conditional is not met, predictions may be an empty
# array. Don't run this next line outside of this conditional.
# (Happens only in a race condition after the audioSnippet is
# uploaded, and before it adds its first prediction.)
responses = [a.predictions[-1] for a in audioSnippets]
# All tasks complete for first time
totalString = overlap.combineSeveral(responses)
doc.completeTranscript = totalString
doc.save()
return newHITCompleted
def isTaskReady(self, hitID):
return len(self.connection.get_assignments(hitID)) > 0
def approveAllHits(self):
# Approve hits:
for assignment in self.getAllAssignments():
try:
self.connection.approve_assignment(assignment.AssignmentId)
except MTurkRequestError as e:
# Maybe already approved?
logger.error("MTurk Request Error: " + str(e))
def checkIfHitsReady(self):
return True
def getAllAssignments(self):
allHits = [hit for hit in self.connection.get_all_hits()]
# Approve hits:
for hit in allHits:
assignments = self.connection.get_assignments(hit.HITId)
for assignment in assignments:
yield assignment
from boto.mturk.connection import MTurkConnection
from boto.mturk.question import HTMLQuestion
from boto.mturk.layoutparam import LayoutParameter
from boto.mturk.layoutparam import LayoutParameters
hit_url_file = open("hit-url.list", "w")
hit_id_file = open("hit-id.list", "a")
img_list = [
line.rstrip("\n") for line in open("C:/Users/David/autoturk/image.list")
]
num_img = len(img_list)
for i in range(num_img):
img_src = "https://s3.us-east-2.amazonaws.com/drone-net/" + img_list[i]
mtc = MTurkConnection(aws_access_key_id="[Your_access_key_ID]",
aws_secret_access_key="[Your_secret_access_key]",
host="mechanicalturk.amazonaws.com")
image_url = LayoutParameter("image_url", img_src)
obj_to_find = LayoutParameter("objects_to_find", "drone")
params = LayoutParameters([image_url, obj_to_find])
response = mtc.create_hit(hit_layout="[Your_hit_layout]",
layout_params=params,
hit_type="[Your_hit_type]")
hit_type_id = response[0].HITTypeId
hit_id = response[0].HITId
hit_id_file.write(hit_id + "\n")
print("HIT ID: {}".format(hit_id))
hit_url_file.write("https://www.mturk.com/mturk/preview?groupId=" +
hit_type_id + "\n")
print("HIT URL: https://www.mturk.com/mturk/preview?groupId={}".format(
hit_type_id))
class MTurkProvider(object):
description = 'This is a task authored by a requester on Daemo, a research crowdsourcing platform. ' \
'Mechanical Turk workers are welcome to do it'
keywords = ['daemo']
countries = ['US', 'CA']
min_hits = 1000
def __init__(self, host, aws_access_key_id, aws_secret_access_key):
self.host = host
self.connection = MTurkConnection(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
host=settings.MTURK_HOST)
self.connection.APIVersion = "2014-08-15"
if not self.host:
raise ValueError("Please provide a host url")
def get_connection(self):
return self.connection
@staticmethod
def _mturk_system_qualifications(qualification):
requirements = []
for item in qualification.items.all():
if item.expression['attribute'] not in [
'location', 'approval_rate', 'total_tasks'
]:
continue
requirement = None
if item.expression['attribute'] == 'location':
op = OP_IN if item.expression['operator'] == 'in' else OP_NOT_IN
requirement = MultiLocaleRequirement(op, [
val.strip() for val in item.expression['value']
if val is not None and val != ''
])
elif item.expression['attribute'] == 'approval_rate':
op = OP_GT if item.expression['operator'] == 'gt' else OP_LT
requirement = PercentAssignmentsApprovedRequirement(
op, item.expression['value'])
elif item.expression['attribute'] == 'total_tasks':
op = OP_GT if item.expression['operator'] == 'gt' else OP_LT
requirement = NumberHitsApprovedRequirement(
op, item.expression['value'])
requirements.append(requirement)
return requirements
def get_qualifications(self, project, boomerang_threshold, add_boomerang):
requirements = []
if project.qualification is not None:
requirements += self._mturk_system_qualifications(
project.qualification)
boomerang_qual, success = self.create_qualification_type(
owner_id=project.owner_id,
project_id=project.group_id,
name='Boomerang Score #{}'.format(project.group_id),
flag=FLAG_Q_BOOMERANG,
description='No description available')
boomerang = None
if boomerang_threshold <= int(settings.BOOMERANG_MIDPOINT * 100):
for i, bucket in enumerate(WAIT_LIST_BUCKETS):
if int(bucket[1] * 100) <= boomerang_threshold:
boomerang_blacklist, success = \
self.create_qualification_type(owner_id=project.owner_id,
name='Boomerang Waitlist #{}-{}'.format(project.group_id, len(
WAIT_LIST_BUCKETS) - i),
flag=FLAG_Q_BOOMERANG,
description='No description available',
deny=True,
project_id=project.group_id,
bucket=bucket)
if success and add_boomerang:
boomerang = BoomerangRequirement(
qualification_type_id=boomerang_blacklist.type_id,
comparator=OP_DNE,
integer_value=None)
requirements.append(boomerang)
else:
boomerang = BoomerangRequirement(
qualification_type_id=boomerang_qual.type_id,
comparator=OP_GTEQ,
integer_value=boomerang_threshold)
if success and add_boomerang:
requirements.append(boomerang)
return Qualifications(requirements), boomerang_qual
def create_hits(self, project, tasks=None, repetition=None):
# if project.min_rating > 0:
# return 'NOOP'
if not tasks:
cursor = connection.cursor()
# noinspection SqlResolve
query = '''
SELECT
max(id) id,
repetition,
group_id,
repetition - sum(existing_assignments) remaining_assignments,
min_rating
FROM (
SELECT
t_rev.id,
t.group_id,
t.min_rating,
p.repetition,
CASE WHEN ma.id IS NULL OR ma.status IN (%(skipped)s, %(rejected)s, %(expired)s)
THEN 0
ELSE 1 END existing_assignments
FROM crowdsourcing_task t
INNER JOIN crowdsourcing_project p ON t.project_id = p.id
INNER JOIN crowdsourcing_task t_rev ON t_rev.group_id = t.group_id
LEFT OUTER JOIN mturk_mturkhit mh ON mh.task_id = t_rev.id
LEFT OUTER JOIN mturk_mturkassignment ma ON ma.hit_id = mh.id
WHERE t.project_id = (%(project_id)s) AND t_rev.exclude_at IS NULL
AND t_rev.deleted_at IS NULL
) t
GROUP BY group_id, repetition, min_rating HAVING sum(existing_assignments) < repetition;
'''
cursor.execute(
query, {
'skipped': TaskWorker.STATUS_SKIPPED,
'rejected': TaskWorker.STATUS_REJECTED,
'expired': TaskWorker.STATUS_EXPIRED,
'project_id': project.id
})
tasks = cursor.fetchall()
rated_workers = Rating.objects.filter(
origin_type=Rating.RATING_REQUESTER).count()
add_boomerang = rated_workers > 0
duration = project.timeout if project.timeout is not None else datetime.timedelta(
hours=24)
lifetime = project.deadline - timezone.now(
) if project.deadline is not None else datetime.timedelta(days=7)
for task in tasks:
question = self.create_external_question(task[0])
mturk_hit = MTurkHIT.objects.filter(task_id=task[0]).first()
qualifications, boomerang_qual = self.get_qualifications(
project=project,
boomerang_threshold=int(round(task[4], 2) * 100),
add_boomerang=add_boomerang)
qualifications_mask = 0
if qualifications is not None:
qualifications_mask = FLAG_Q_LOCALE + FLAG_Q_HITS + FLAG_Q_RATE + FLAG_Q_BOOMERANG
hit_type, success = self.create_hit_type(
title=project.name,
description=self.description,
price=project.price,
duration=duration,
keywords=self.keywords,
approval_delay=datetime.timedelta(days=2),
qual_req=qualifications,
qualifications_mask=qualifications_mask,
boomerang_threshold=int(round(task[4], 2) * 100),
owner_id=project.owner_id,
boomerang_qual=boomerang_qual)
if not success:
return 'FAILURE'
if mturk_hit is None:
try:
hit = self.connection.create_hit(
hit_type=hit_type.string_id,
max_assignments=task[3],
lifetime=lifetime,
question=question)[0]
self.set_notification(hit_type_id=hit.HITTypeId)
mturk_hit = MTurkHIT(hit_id=hit.HITId,
hit_type=hit_type,
task_id=task[0])
except MTurkRequestError as e:
error = e.errors[0][0]
if error == 'AWS.MechanicalTurk.InsufficientFunds':
message = {
"type": "ERROR",
"detail":
"Insufficient funds on your Mechanical Turk account!",
"code": error
}
redis_publisher = RedisPublisher(facility='bot',
users=[project.owner])
message = RedisMessage(json.dumps(message))
redis_publisher.publish_message(message)
return 'FAILED'
else:
if mturk_hit.hit_type_id != hit_type.id:
result, success = self.change_hit_type_of_hit(
hit_id=mturk_hit.hit_id,
hit_type_id=hit_type.string_id)
if success:
mturk_hit.hit_type = hit_type
mturk_hit.save()
return 'SUCCESS'
def create_hit_type(self,
owner_id,
title,
description,
price,
duration,
boomerang_threshold,
keywords=None,
approval_delay=None,
qual_req=None,
qualifications_mask=0,
boomerang_qual=None):
hit_type = MTurkHITType.objects.filter(
owner_id=owner_id,
name=title,
description=description,
price=Decimal(str(price)),
duration=duration,
qualifications_mask=qualifications_mask,
boomerang_threshold=boomerang_threshold).first()
if hit_type is not None:
return hit_type, True
reward = Price(price)
try:
mturk_ht = self.connection.register_hit_type(
title=title,
description=description,
reward=reward,
duration=duration,
keywords=keywords,
approval_delay=approval_delay,
qual_req=qual_req)[0]
hit_type = MTurkHITType(owner_id=owner_id,
name=title,
description=description,
price=Decimal(str(price)),
keywords=keywords,
duration=duration,
qualifications_mask=qualifications_mask,
boomerang_qualification=boomerang_qual,
boomerang_threshold=boomerang_threshold)
hit_type.string_id = mturk_ht.HITTypeId
hit_type.save()
except MTurkRequestError:
return None, False
return hit_type, True
def create_external_question(self, task, frame_height=800):
task_hash = Hashids(salt=settings.SECRET_KEY,
min_length=settings.ID_HASH_MIN_LENGTH)
task_id = task_hash.encode(task)
url = self.host + '/mturk/task/?taskId=' + task_id
question = ExternalQuestion(external_url=url,
frame_height=frame_height)
return question
def update_max_assignments(self, task):
task = Task.objects.get(id=task['id'])
mturk_hit = task.mturk_hit
if not mturk_hit:
raise MTurkHIT.DoesNotExist(
"This task is not associated to any mturk hit")
assignments_completed = task.task_workers.filter(~Q(status__in=[
TaskWorker.STATUS_REJECTED, TaskWorker.STATUS_SKIPPED,
TaskWorker.STATUS_EXPIRED
])).count()
remaining_assignments = task.project.repetition - assignments_completed
if remaining_assignments > 0 and mturk_hit.num_assignments == mturk_hit.mturk_assignments. \
filter(status=TaskWorker.STATUS_SUBMITTED).count() and \
mturk_hit.mturk_assignments.filter(status=TaskWorker.STATUS_IN_PROGRESS).count() == 0:
self.add_assignments(hit_id=mturk_hit.hit_id, increment=1)
self.extend_hit(hit_id=mturk_hit.hit_id)
mturk_hit.status = MTurkHIT.STATUS_IN_PROGRESS
mturk_hit.num_assignments += 1
mturk_hit.save()
elif remaining_assignments == 0:
self.expire_hit(hit_id=mturk_hit.hit_id)
mturk_hit.status = MTurkHIT.STATUS_EXPIRED
mturk_hit.save()
elif remaining_assignments > 0 and \
mturk_hit.status == MTurkHIT.STATUS_EXPIRED:
self.extend_hit(hit_id=mturk_hit.hit_id)
mturk_hit.status = MTurkHIT.STATUS_IN_PROGRESS
return 'SUCCESS'
def get_assignment(self, assignment_id):
try:
return self.connection.get_assignment(assignment_id)[0], True
except MTurkRequestError as e:
error = e.errors[0][0]
if error == 'AWS.MechanicalTurk.InvalidAssignmentState':
return assignment_id, False
return None, False
def set_notification(self, hit_type_id):
self.connection.set_rest_notification(
hit_type=hit_type_id,
url=self.host + '/api/mturk/notification',
event_types=[
'AssignmentReturned', 'AssignmentAbandoned',
'AssignmentAccepted', 'AssignmentSubmitted'
])
def approve_assignment(self, task_worker):
task_worker_obj = TaskWorker.objects.get(id=task_worker['id'])
if hasattr(task_worker_obj, 'mturk_assignments'
) and task_worker_obj.mturk_assignments.first() is not None:
try:
self.connection.approve_assignment(
task_worker_obj.mturk_assignments.first().assignment_id)
except MTurkRequestError:
return False
return True
def reject_assignment(self, task_worker):
task_worker_obj = TaskWorker.objects.get(id=task_worker['id'])
if hasattr(task_worker_obj, 'mturk_assignments'
) and task_worker_obj.mturk_assignments.first() is not None:
try:
self.connection.reject_assignment(
task_worker_obj.mturk_assignments.first().assignment_id)
except MTurkRequestError:
return False
return True
def expire_hit(self, hit_id):
try:
self.connection.expire_hit(hit_id)
except MTurkRequestError:
return False
return True
def disable_hit(self, hit_id):
try:
self.connection.disable_hit(hit_id)
except MTurkRequestError:
return False
return True
def extend_hit(self, hit_id):
try:
self.connection.extend_hit(hit_id=hit_id,
expiration_increment=604800) # 7 days
except MTurkRequestError:
return False
return True
def add_assignments(self, hit_id, increment=1):
try:
self.connection.extend_hit(hit_id=hit_id,
assignments_increment=increment)
except MTurkRequestError:
return False
return True
def test_connection(self):
try:
return self.connection.get_account_balance()[0], True
except MTurkRequestError as e:
error = e.errors[0][0]
if error == 'AWS.NotAuthorized':
return None, False
return None, False
def get_account_balance(self):
try:
return self.connection.get_account_balance()[0]
except MTurkRequestError:
return None
def create_qualification_type(self,
owner_id,
name,
flag,
description,
project_id,
auto_granted=False,
auto_granted_value=None,
deny=False,
bucket=None):
# noinspection SqlResolve
query = '''
SELECT * FROM (
SELECT
task.target_id,
task.username,
round(task.task_w_avg::NUMERIC, 2) rating
--round(coalesce(task.task_w_avg, requester.requester_w_avg,
-- platform.platform_w_avg)::NUMERIC, 2) rating
FROM (
SELECT
target_id,
origin_id,
project_id,
username,
sum(weight * power((%(BOOMERANG_TASK_ALPHA)s), t.row_number))
/ sum(power((%(BOOMERANG_TASK_ALPHA)s), t.row_number)) task_w_avg
FROM (
SELECT
r.id,
r.origin_id,
p.group_id project_id,
weight,
r.target_id,
-1 + row_number()
OVER (PARTITION BY target_id
ORDER BY tw.created_at DESC) AS row_number,
u.username username
FROM crowdsourcing_rating r
INNER JOIN crowdsourcing_task t ON t.id = r.task_id
INNER JOIN crowdsourcing_project p ON p.id = t.project_id
INNER JOIN crowdsourcing_taskworker tw ON t.id = tw.task_id
AND tw.worker_id=r.target_id
INNER JOIN auth_user u ON u.id = r.target_id
WHERE origin_id = (%(origin_id)s) AND origin_type = (%(origin_type)s)) t
GROUP BY origin_id, target_id, project_id, username)
task WHERE task.project_id = (%(project_id)s)
) r
'''
extra_query = 'WHERE rating BETWEEN (%(lower_bound)s) AND (%(upper_bound)s);'
params = {
'origin_type': Rating.RATING_REQUESTER,
'origin_id': owner_id,
'project_id': project_id,
'BOOMERANG_REQUESTER_ALPHA': settings.BOOMERANG_REQUESTER_ALPHA,
'BOOMERANG_PLATFORM_ALPHA': settings.BOOMERANG_PLATFORM_ALPHA,
'BOOMERANG_TASK_ALPHA': settings.BOOMERANG_TASK_ALPHA
}
obj_params = {'upper_bound': 300, 'lower_bound': 100}
if deny and bucket is not None:
query += extra_query
params.update({'upper_bound': bucket[1], 'lower_bound': bucket[0]})
obj_params.update({
'upper_bound': bucket[1] * 100,
'lower_bound': bucket[0] * 100,
'is_blacklist': True
})
cursor = connection.cursor()
cursor.execute(query, params=params)
worker_ratings_raw = cursor.fetchall()
worker_ratings = [{
"worker_id": r[0],
"worker_username": r[1],
"rating": r[2]
} for r in worker_ratings_raw]
qualification = MTurkQualification.objects.filter(owner_id=owner_id,
flag=flag,
name=name).first()
assigned_workers = []
if qualification is None:
try:
qualification_type = self.connection. \
create_qualification_type(name=name, description=description,
status='Active',
auto_granted=auto_granted,
auto_granted_value=auto_granted_value)[0]
qualification = MTurkQualification.objects.create(
owner_id=owner_id,
flag=flag,
name=name,
description=description,
auto_granted=auto_granted,
auto_granted_value=auto_granted_value,
type_id=qualification_type.QualificationTypeId,
**obj_params)
except MTurkRequestError:
return None, False
else:
assigned_workers = MTurkWorkerQualification.objects.values(
'worker').filter(qualification=qualification).values_list(
'worker', flat=True)
for rating in worker_ratings:
user_name = rating["worker_username"].split('.')
if len(user_name) == 2 and user_name[0] == 'mturk':
mturk_worker_id = user_name[1].upper()
if mturk_worker_id not in assigned_workers:
self.assign_qualification(
qualification_type_id=qualification.type_id,
worker_id=mturk_worker_id,
value=int(rating['rating'] * 100))
defaults = {
'qualification': qualification,
'worker': mturk_worker_id,
'score': int(rating['rating'] * 100)
}
MTurkWorkerQualification.objects.update_or_create(
qualification=qualification,
worker=mturk_worker_id,
defaults=defaults)
return qualification, True
def change_hit_type_of_hit(self, hit_id, hit_type_id):
try:
result = self.connection.change_hit_type_of_hit(
hit_id=hit_id, hit_type=hit_type_id)
except MTurkRequestError:
return None, False
return result, True
def update_worker_boomerang(self, project_id, worker_id, task_avg,
requester_avg):
"""
Update boomerang for project
Args:
project_id:
worker_id:
task_avg:
requester_avg
Returns:
str
"""
hit = MTurkHIT.objects.select_related(
'hit_type__boomerang_qualification').filter(
task__project__group_id=project_id).first()
if hit is not None:
qualification = hit.hit_type.boomerang_qualification
worker_qual = MTurkWorkerQualification.objects.filter(
qualification=qualification, worker=worker_id).first()
if worker_qual is not None:
self.update_score(worker_qual,
score=int(task_avg * 100),
override=True)
else:
MTurkWorkerQualification.objects.create(
qualification=qualification,
worker=worker_id,
score=int(task_avg * 100),
overwritten=True)
self.assign_qualification(
qualification_type_id=qualification.type_id,
worker_id=worker_id,
value=int(task_avg * 100))
# other_quals = MTurkWorkerQualification.objects.filter(~Q(qualification=qualification),
# worker=worker_id,
# overwritten=False)
# for q in other_quals:
# self.update_score(q, score=int(requester_avg * 100))
return 'SUCCESS'
def update_score(self, worker_qual, score, override=False):
if worker_qual is None:
return False
try:
self.connection.update_qualification_score(
worker_qual.qualification.type_id, worker_qual.worker, score)
worker_qual.overwritten = override
worker_qual.score = score
worker_qual.save()
except MTurkRequestError:
return False
return True
def assign_qualification(self, qualification_type_id, worker_id, value=1):
"""
Revoke a qualification from a WorkerId
Args:
qualification_type_id:
worker_id:
value
Returns:
bool
"""
try:
self.connection.assign_qualification(qualification_type_id,
worker_id,
value,
send_notification=False)
return True
except MTurkRequestError:
return False
def revoke_qualification(self, qualification_type_id, worker_id):
try:
self.connection.revoke_qualification(
qualification_type_id=qualification_type_id,
subject_id=worker_id)
return True
except MTurkRequestError:
return False
def notify_workers(self, worker_ids, subject, message_text):
try:
self.connection.notify_workers(worker_ids, subject, message_text)
return True
except MTurkRequestError:
return False
host=AMAZON_HOST)
#frame_height in pixels
frame_height = 800
#Here, I create two sample qualifications
qualifications = Qualifications()
qualifications.add(
PercentAssignmentsApprovedRequirement(comparator="GreaterThan",
integer_value="90"))
qualifications.add(
NumberHitsApprovedRequirement(comparator="GreaterThan",
integer_value="100"))
#This url will be the url of your application, with appropriate GET parameters
url = "google.com"
questionform = ExternalQuestion(url, frame_height)
create_hit_result = connection.create_hit(
title="Insert the title of your HIT",
description="Insert your description here",
keywords=["add", "some", "keywords"],
#duration is in seconds
duration=60 * 60,
#max_assignments will set the amount of independent copies of the task (turkers can only see one)
max_assignments=15,
question=questionform,
reward=Price(amount=amount),
#Determines information returned by method in API, not super important
response_groups=('Minimal', 'HITDetail'),
qualifications=qualifications,
)
content=qc3,
answer_spec=AnswerSpecification(fta3))
#--------------- BUILD THE QUESTION FORM -------------------
question_form = QuestionForm()
question_form.append(overview)
question_form.append(q1)
question_form.append(q2)
question_form.append(q3)
#--------------- CREATE THE HIT -------------------
qualification = Qualifications()
qualification.add(LocaleRequirement('EqualTo','US'))
hitDetails = mtc.create_hit(questions=question_form,
max_assignments=5,
title=title,
description=description,
keywords=keywords,
duration = 60*60,
reward=0.1,
qualifications=qualification,
response_groups = ['Minimal'],
)
created_hits.append([count, hitDetails[0].HITId])
count += 1
createCsvFile('createdHITs_commission.csv', created_hits)
answer_spec=AnswerSpecification(fta1),
is_required=True)
#--------------- BUILD QUESTION 2 -------------------
qc2 = QuestionContent()
qc2.append_field('Title','Your personal comments')
fta2 = FreeTextAnswer()
q2 = Question(identifier="comments",
content=qc2,
answer_spec=AnswerSpecification(fta2))
#--------------- BUILD THE QUESTION FORM -------------------
question_form = QuestionForm()
question_form.append(overview)
question_form.append(q1)
question_form.append(q2)
#--------------- CREATE THE HIT -------------------
mtc.create_hit(questions=question_form,
max_assignments=1,
title=title,
description=description,
keywords=keywords,
duration = 60*5,
reward=0.05)
answer_spec=AnswerSpecification(fta1),
is_required=True)
#make q2
qc2 = QuestionContent()
qc2.append_field('Title','Comments about the HIT (Optional)')
fta2 = FreeTextAnswer()
q2 = Question(identifier="comments",
content=qc2,
answer_spec=AnswerSpecification(fta2))
#make question form
question_form = QuestionForm()
question_form.append(overview)
question_form.append(q1)
question_form.append(q2)
#--------------- CREATE THE HIT -------------------
mtc.create_hit(questions=question_form,
max_assignments=1,
title=title,
description=description,
keywords=keywords,
duration = 60*5,
reward=0.05)
class MTurk(object):
"""
A class that wraps a boto.mturk.connection object and provides methods for
the most common AI2 use cases
"""
def __init__(self,
aws_access_key_id,
aws_secret_access_key,
host=SANDBOX_HOST):
"""
initializes the instance with AWS credentials and a host
:param aws_access_key_id the access key id.
:param aws_secret_access_key the secret access key.
:param host the mturk host to connect to
"""
self.connection = MTurkConnection(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
host=host)
self.host = host
def __del__(self):
"""
close the connection whenever this object goes out of scope
"""
self.connection.close()
def get_account_balance(self):
"""
:return the balance on the mturk account
"""
return self.connection.get_account_balance()[0]
def _create_hit(self, params, **kwargs):
"""
internal helper function for creating a HIT
:param params the parameters (required and optional) common to all HITs
:param **kwargs any other parameters needed for a specific HIT type
:return the created HIT object
"""
return self.connection.create_hit(
title=params["title"],
description=params["description"],
keywords=params["keywords"],
max_assignments=params["max_assignments"],
reward=Price(amount=params["amount"]),
qualifications=params["qualifications"],
lifetime=params["lifetime"],
# optional params below
annotation=params.get("annotation"),
**kwargs)
def create_url_hit(self, params):
"""
creates a HIT for an external question with a specified URL
:param params a dict of the HIT parameters. must contain a "url" parameter
:return the created HIT object
"""
question = ExternalQuestion(params["url"], params["frame_height"])
return self._create_hit(params, question=question)
def create_html_hit(self, params):
"""
creates a HIT for a question with the specified HTML
:param params a dict of the HIT parameters, must contain a "html" parameter
:return the created HIT object
"""
question = HTMLQuestion(params["html"], params["frame_height"])
return self._create_hit(params, question=question)
def create_layout_hit(self, params):
"""
creates a HIT for a question using the supplied layout id
:param params a dict of the HIT parameters, must contain a "hit_layout"
parameters with the layout id, and a "layout_params" parameter
that's the dict of parameters to feed to the layout.
"""
# create the LayoutParameters object from the supplied params
layout_params = LayoutParameters([
LayoutParameter(name, value)
for name, value in params["layout_params"]
])
return self._create_hit(params,
hit_layout=params["hit_layout"],
layout_params=layout_params)
def delete_all_hits(self):
"""
Permanently disables/ deletes all of the user's active HITs.
:param mturk_connection: active mturk connection established by user in the notebook.
:return:
"""
my_hits = list(self.get_all_hits())
for hit in my_hits:
self.connection.disable_hit(hit.HITId)
def get_assignments_object_list(self, assignment_dict):
"""
Returns a list of "<boto.mturk.connection.Assignment object at...>" objects
assignment_dict: a dictionary of HITId-assignment object pairs
"""
assignments = []
for entry in assignment_dict:
for assignment_object in assignment_dict[entry]:
assignments.append(assignment_object)
return assignments
def get_results_dict(self, HIT_assignments):
"""
Takes a list of HIT assignment objects as input.
Returns a list of dictionaries of HITs containing:
HIT_id: the HIT ID
worker_id: the worker ID of the Turker who completed the HIT
answers: a dictionary of qid-answer field value pairs
"""
assignment_results = []
for assignment in HIT_assignments:
HIT_dict = {}
HIT_dict["assignment_object"] = assignment
HIT_dict["worker_Id"] = assignment.WorkerId
HIT_dict["HIT_id"] = assignment.HITId
answers_dict = {}
for answer in assignment.answers[0]:
answers_dict[answer.qid] = answer.fields
HIT_dict["answers"] = answers_dict
assignment_results.append(HIT_dict)
return assignment_results
def get_all_results(self, hits):
all_results = {}
for hid, assignments in self.get_assignments(hits).items():
all_results[hid] = self.get_results_dict(assignments)
return all_results
def get_reviewable_hits(self, annotations=None, detailed=False):
"""
Get all the reviewable HITs. By default returns minimal HIT objects, but
will return detailed ones (by necessity) if annotations is specified or
if detailed is True
:param annotations an optional set of annotations to retrieve HITs for
:param detailed do you want detailed HIT objects or minimal ones
:return a list of HIT objects
"""
minimal_hits = []
page_num = 1
while True:
more_hits = self.connection.get_reviewable_hits(
page_size=100, page_number=page_num)
if more_hits:
minimal_hits.extend(more_hits)
page_num += 1
else:
break
if detailed or annotations is not None:
detailed_hits = [
self.connection.get_hit(hit.HITId,
response_groups=('Minimal',
'HITDetail'))
for hit in minimal_hits
]
return [
hit for hit in detailed_hits
if annotation_filter(annotations, hit)
]
else:
return minimal_hits
def get_all_hits(self, annotations=None):
"""
Get all the HITs.
:param annotations a set of annotations to get HITs for, all HITs if
not specified
:return a list of HIT objects
"""
return [
hit for hit in self.connection.get_all_hits()
if annotation_filter(annotations, hit)
]
def get_assignments(self, hits=None, hit_ids=None, status=None):
"""
Retrieves individual assignments associated with the supplied HITs
:param hits the HITs to get assignments for
:status HIT status to filter by
:return dict from HITId to lists of assignments
"""
if hit_ids is None:
hit_ids = [hit.HITId for hit in hits]
return {
hit_id: self.connection.get_assignments(hit_id, status=status)
for hit_id in hit_ids
}
def disable_hit(self, hit=None, hit_id=None):
"""
disable the specified hit (or the hit with the specified id). must
specify either `hit` or `hit_id`
:param hit a HIT object to disable
:param hit_id a HITId to disable
"""
hit_id = hit.HITId if hit is not None else hit_id
return self.connection.disable_hit(hit_id)
def approve_assignment(self,
assignment=None,
assignment_id=None,
feedback=None):
"""
approve the specified assignment (or the assigment with the specified id)
must specify either `assignment` or `assignment_id`
:param assignment an assignment object to approve
:param assignment_id an AssignmentId to approve
:param feedback optional feedback for the worker
"""
assignment_id = assignment.AssignmentId if assignment is not None else assignment_id
return self.connection.approve_assignment(assignment_id, feedback)
def reject_assignment(self,
assignment=None,
assignment_id=None,
feedback=None):
"""
reject the specified assignment (or the assigment with the specified id)
must specify either `assignment` or `assignment_id`
:param assignment an assignment object to reject
:param assignment_id an AssignmentId to reject
:param feedback optional feedback for the worker
"""
assignment_id = assignment.AssignmentId if assignment is not None else assignment_id
return self.connection.reject_assignment(assignment_id, feedback)