def delete_hit(hit_id):
if SANDBOX:
mturk_url = 'mechanicalturk.sandbox.amazonaws.com'
preview_url = 'https://workersandbox.mturk.com/mturk/preview?groupId='
else:
mturk_url = 'mechanicalturk.amazonaws.com'
preview_url = 'https://mturk.com/mturk/preview?groupId='
conn = MTurkConnection(aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY, host=mturk_url)
conn.expire_hit(hit_id)
# Give the HIT a moment to expire.
time.sleep(0.25)
conn.dispose_hit(hit_id)
print("HIT " + hit_id + " was deleted!")
def delete_hit(hit_id):
if SANDBOX:
mturk_url = 'mechanicalturk.sandbox.amazonaws.com'
preview_url = 'https://workersandbox.mturk.com/mturk/preview?groupId='
else:
mturk_url = 'mechanicalturk.amazonaws.com'
preview_url = 'https://mturk.com/mturk/preview?groupId='
conn = MTurkConnection(aws_access_key_id=AWS_ACCESS_KEY,
aws_secret_access_key=AWS_SECRET_KEY,
host=mturk_url)
conn.expire_hit(hit_id)
# Give the HIT a moment to expire.
time.sleep(0.25)
conn.dispose_hit(hit_id)
print("HIT " + hit_id + " was deleted!")
class amusic(object):
engine = None
origconf = {'minPitch':12,'maxPitch':84,'minStartTime':0.0,'maxStartTime':200.0,
'minNoteDuration':0.5,'maxNoteDuration':5.0,'minNoteCount':50,'maxNoteCount':400,
'currentPopulation':'','bucket':None,'hitRewardPerAssignment':0.05,'mturkLayoutID':None,
'hitTitle':None,'hitDescription':None,'hitKeywords':None}
conf = origconf.copy()
s3bucket = None
mtc = None
class PopulationNotSet( Exception ): pass
class SongNotFound( Exception ): pass
def __init__(self,username,password,ACCESS_ID,SECRET_KEY,initialize=False):
self.ACCESS_ID = ACCESS_ID
self.SECRET_KEY = SECRET_KEY
self.engine = sqlalchemy.create_engine('mysql+mysqlconnector://%s:%s@localhost' % (username,password))
self.engine.connect()
self.mtc = MTurkConnection(self.ACCESS_ID,self.SECRET_KEY,host='mechanicalturk.sandbox.amazonaws.com')
try:
self.engine.execute('USE amusic;')
c = self.engine.execute('SELECT * FROM conf;').fetchone()
self.conf['minPitch'] = c[0]
self.conf['maxPitch'] = c[1]
self.conf['minStartTime'] = c[2]
self.conf['maxStartTime'] = c[3]
self.conf['minNoteDuration'] = c[4]
self.conf['maxNoteDuration'] = c[5]
self.conf['minNoteCount'] = c[6]
self.conf['maxNoteCount'] = c[7]
self.conf['currentPopulation'] = c[8]
self.conf['bucket'] = c[9]
self.conf['mturkLayoutID'] = c[10]
self.conf['hitTitle'] = c[11]
self.conf['hitDescription'] = c[12]
self.conf['hitKeywords'] = c[13]
self.conf['hitRewardPerAssignment'] = c[14]
except:pass
def inits3(self):
c = boto.connect_s3(self.ACCESS_ID,self.SECRET_KEY)
self.s3bucket = c.create_bucket(self.conf['bucket'])
self.s3bucket.set_acl('public-read')
def initialize(self):
try: self.engine.execute('DROP DATABASE IF EXISTS amusic;')
except: pass
self.engine.execute('CREATE DATABASE amusic;')
self.engine.execute('USE amusic;')
self.engine.execute('CREATE TABLE conf (minPitch INT,\
maxPitch INT,\
minStartTime FLOAT,\
maxStartTime FLOAT,\
minNoteDuration FLOAT,\
maxNoteDuration FLOAT,\
minNoteCount INT,\
maxNoteCount INT,\
currentPopulation VARCHAR(100),\
bucket VARCHAR(100),\
mturkLayoutID VARCHAR(100),\
hitTitle VARCHAR(100),\
hitDescription VARCHAR(100),\
hitKeywords VARCHAR(100),\
hitRewardPerAssignment FLOAT,\
CONSTRAINT fk_1 FOREIGN KEY (`currentPopulation`) REFERENCES population (title),\
PRIMARY KEY(minPitch)\
) ENGINE = MYISAM;')
self.engine.execute('INSERT INTO conf (minPitch,\
maxPitch,\
minStartTime,\
maxStartTime,\
minNoteDuration,\
maxNoteDuration,\
minNoteCount,\
maxNoteCount,\
currentPopulation,\
bucket,\
hitRewardPerAssignment,\
mturkLayoutID,\
hitTitle,\
hitKeywords,\
hitDescription)\
VALUES(%d,%d,%f,%f,%f,%f,%d,%d,"%s","%s",%f,"%s","%s","%s","%s");'%
(self.origconf['minPitch'],self.origconf['maxPitch'],self.origconf['minStartTime'],
self.origconf['maxStartTime'],self.origconf['minNoteDuration'],
self.origconf['maxNoteDuration'],self.origconf['minNoteCount'],
self.origconf['maxNoteCount'],self.origconf['currentPopulation'],
self.origconf['bucket'], self.origconf['hitRewardPerAssignment'],
self.origconf['mturkLayoutID'],self.origconf['hitTitle'],
self.origconf['hitKeywords'],self.origconf['hitDescription']))
self.engine.execute('CREATE TABLE population (title VARCHAR(100) NOT NULL,\
PRIMARY KEY(title)\
) ENGINE = MYISAM;')
self.engine.execute('CREATE TABLE song (id INT NOT NULL AUTO_INCREMENT,\
title VARCHAR(100),\
population VARCHAR(100),\
ppq INT,\
CONSTRAINT fk_1 FOREIGN KEY (`population`) REFERENCES population (title),\
PRIMARY KEY(id,title)\
) ENGINE = MYISAM;')
self.engine.execute('CREATE TABLE event (songID INT,\
track INT,\
id INT NOT NULL AUTO_INCREMENT,\
type VARCHAR(5),\
pitch INT,\
value INT,\
startTime FLOAT,\
duration FLOAT,\
velocity INT,\
CONSTRAINT fk_1 FOREIGN KEY (`songID`) REFERENCES song (id),\
PRIMARY KEY(songID,track,id)\
) ENGINE = MYISAM;')
def setPopulation(self,title):
self.engine.execute('UPDATE conf SET currentPopulation="%s";' % title)
self.conf['currentPopulation'] = title
def newPopulation(self,title):
return Population(self,title)
def getCurrentPopulation(self):
if self.conf['currentPopulation'] == '':
raise self.PopulationNotSet
return Population(self,self.conf['currentPopulation'])
def getPopulation(self,title):
return Population(self,title,create=False)
def listHITs(self):
print " ".join(('%s','%s','%s','%s','%s')) % ("HIT ID".ljust(30), "Status".ljust(22), "Amount", "Song1", "Song2")
for i in self.mtc.search_hits(response_groups=['Request','Minimal','HITDetail','HITQuestion']):
l = re.findall('<input type="hidden" value="([^"]+?)" name="song[12]" />',i.Question)
print ' '.join((i.HITId, i.HITStatus,i.HITReviewStatus,i.Amount,l[0].split('/')[-1],l[1].split('/')[-1]))
def deleteHITs(self):
for i in self.mtc.get_reviewable_hits():
self.mtc.dispose_hit(i.HITId)
def approveAssignment(self,aID):
self.mtc.approve_assignment(aID)
def rejectAssignment(self,aID):
self.mtc.reject_assignment(aID)
def approveAllAssignments(self):
for i in self.mtc.get_reviewable_hits():
for assignment in self.mtc.get_assignments(i.HITId):
if assignment.AssignmentStatus=="Submitted":
self.mtc.approve_assignment(assignment.AssignmentId)
def getResults(self):
print " ".join(('%s','%s','%s','%s','%s')) % ("Assignment ID".ljust(30), "Worker ID".ljust(14), "Song1", "Song2", "Answer")
for i in self.mtc.get_reviewable_hits():
for assignment in self.mtc.get_assignments(i.HITId):
ans = {}
for j in assignment.answers[0]:
if j.qid!='commit': ans[j.qid] = j.fields[0].split('/')[-1]
print assignment.AssignmentId, assignment.WorkerId,ans['song1'], ans['song2'], ans['boxradio']
class MTurkServices(object):
''' MTurk services '''
def __init__(self, aws_access_key_id, aws_secret_access_key, is_sandbox):
self.update_credentials(aws_access_key_id, aws_secret_access_key)
self.set_sandbox(is_sandbox)
self.valid_login = self.verify_aws_login()
if not self.valid_login:
print 'WARNING *****************************'
print 'Sorry, AWS Credentials invalid.\nYou will only be able to '\
'test experiments locally until you enter\nvalid '\
'credentials in the AWS Access section of ~/.psiturkconfig\n'
def update_credentials(self, aws_access_key_id, aws_secret_access_key):
''' Update credentials '''
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
def set_sandbox(self, is_sandbox):
''' Set sandbox '''
self.is_sandbox = is_sandbox
def get_reviewable_hits(self):
''' Get reviewable HITs '''
if not self.connect_to_turk():
return False
try:
hits = self.mtc.get_all_hits()
except MTurkRequestError:
return False
reviewable_hits = [hit for hit in hits if hit.HITStatus == "Reviewable" \
or hit.HITStatus == "Reviewing"]
hits_data = [MTurkHIT({
'hitid': hit.HITId,
'title': hit.Title,
'status': hit.HITStatus,
'max_assignments': hit.MaxAssignments,
'number_assignments_completed': hit.NumberOfAssignmentsCompleted,
'number_assignments_pending': hit.NumberOfAssignmentsPending,
'number_assignments_available': hit.NumberOfAssignmentsAvailable,
'creation_time': hit.CreationTime,
'expiration': hit.Expiration
}) for hit in reviewable_hits]
return hits_data
def get_all_hits(self):
''' Get all HITs '''
if not self.connect_to_turk():
return False
try:
hits = self.mtc.get_all_hits()
except MTurkRequestError:
return False
hits_data = [MTurkHIT({
'hitid': hit.HITId,
'title': hit.Title,
'status': hit.HITStatus,
'max_assignments': hit.MaxAssignments,
'number_assignments_completed': hit.NumberOfAssignmentsCompleted,
'number_assignments_pending': hit.NumberOfAssignmentsPending,
'number_assignments_available': hit.NumberOfAssignmentsAvailable,
'creation_time': hit.CreationTime,
'expiration': hit.Expiration,
}) for hit in hits]
return hits_data
def get_active_hits(self):
''' Get active HITs '''
if not self.connect_to_turk():
return False
# hits = self.mtc.search_hits()
try:
hits = self.mtc.get_all_hits()
except MTurkRequestError:
return False
active_hits = [hit for hit in hits if not hit.expired]
hits_data = [MTurkHIT({
'hitid': hit.HITId,
'title': hit.Title,
'status': hit.HITStatus,
'max_assignments': hit.MaxAssignments,
'number_assignments_completed': hit.NumberOfAssignmentsCompleted,
'number_assignments_pending': hit.NumberOfAssignmentsPending,
'number_assignments_available': hit.NumberOfAssignmentsAvailable,
'creation_time': hit.CreationTime,
'expiration': hit.Expiration,
}) for hit in active_hits]
return hits_data
def get_workers(self, assignment_status=None):
''' Get workers '''
if not self.connect_to_turk():
return False
try:
hits = self.mtc.get_all_hits()
hit_ids = [hit.HITId for hit in hits]
workers_nested = []
page_size=100
for hit_id in hit_ids:
current_page_number=1
hit_assignments = self.mtc.get_assignments(
hit_id,
status=assignment_status,
sort_by='SubmitTime',
page_size=page_size,
page_number=current_page_number
)
totalNumResults = int(hit_assignments.TotalNumResults)
total_pages = (totalNumResults // page_size) + (totalNumResults % page_size > 0) #do integer division then round up if necessary
while current_page_number < total_pages:
current_page_number += 1
hit_assignments += self.mtc.get_assignments(
hit_id,
status=assignment_status,
sort_by='SubmitTime',
page_size=page_size,
page_number=current_page_number
)
workers_nested.append(hit_assignments)
workers = [val for subl in workers_nested for val in subl] # Flatten nested lists
except MTurkRequestError:
return False
worker_data = [{
'hitId': worker.HITId,
'assignmentId': worker.AssignmentId,
'workerId': worker.WorkerId,
'submit_time': worker.SubmitTime,
'accept_time': worker.AcceptTime,
'status': worker.AssignmentStatus
} for worker in workers]
return worker_data
def bonus_worker(self, assignment_id, amount, reason=""):
''' Bonus worker '''
if not self.connect_to_turk():
return False
try:
bonus = MTurkConnection.get_price_as_price(amount)
assignment = self.mtc.get_assignment(assignment_id)[0]
worker_id = assignment.WorkerId
self.mtc.grant_bonus(worker_id, assignment_id, bonus, reason)
return True
except MTurkRequestError as exception:
print exception
return False
def approve_worker(self, assignment_id):
''' Approve worker '''
if not self.connect_to_turk():
return False
try:
self.mtc.approve_assignment(assignment_id, feedback=None)
return True
except MTurkRequestError:
return False
def reject_worker(self, assignment_id):
''' Reject worker '''
if not self.connect_to_turk():
return False
try:
self.mtc.reject_assignment(assignment_id, feedback=None)
return True
except MTurkRequestError:
return False
def unreject_worker(self, assignment_id):
''' Unreject worker '''
if not self.connect_to_turk():
return False
try:
self.mtc.approve_rejected_assignment(assignment_id)
return True
except MTurkRequestError:
return False
def verify_aws_login(self):
''' Verify AWS login '''
if ((self.aws_access_key_id == 'YourAccessKeyId') or
(self.aws_secret_access_key == 'YourSecretAccessKey')):
return False
else:
host = 'mechanicalturk.amazonaws.com'
mturkparams = dict(
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
host=host)
self.mtc = MTurkConnection(**mturkparams)
try:
self.mtc.get_account_balance()
except MTurkRequestError as exception:
print exception.error_message
return False
else:
return True
def connect_to_turk(self):
''' Connect to turk '''
if not self.valid_login:
print 'Sorry, unable to connect to Amazon Mechanical Turk. AWS '\
'credentials invalid.'
return False
if self.is_sandbox:
host = 'mechanicalturk.sandbox.amazonaws.com'
else:
host = 'mechanicalturk.amazonaws.com'
mturkparams = dict(
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
host=host)
self.mtc = MTurkConnection(**mturkparams)
return True
def configure_hit(self, hit_config):
''' Configure HIT '''
# configure question_url based on the id
experiment_portal_url = hit_config['ad_location']
frame_height = 600
mturk_question = ExternalQuestion(experiment_portal_url, frame_height)
# Qualification:
quals = Qualifications()
approve_requirement = hit_config['approve_requirement']
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if hit_config['us_only']:
quals.add(LocaleRequirement("EqualTo", "US"))
# Create a HIT type for this HIT.
hit_type = self.mtc.register_hit_type(
hit_config['title'],
hit_config['description'],
hit_config['reward'],
hit_config['duration'],
keywords=hit_config['keywords'],
approval_delay=None,
qual_req=None)[0]
# Check the config file to see if notifications are wanted.
config = PsiturkConfig()
config.load_config()
try:
url = config.get('Server Parameters', 'notification_url')
all_event_types = [
"AssignmentAccepted",
"AssignmentAbandoned",
"AssignmentReturned",
"AssignmentSubmitted",
"HITReviewable",
"HITExpired",
]
self.mtc.set_rest_notification(
hit_type.HITTypeId,
url,
event_types=all_event_types)
except:
pass
# Specify all the HIT parameters
self.param_dict = dict(
hit_type=hit_type.HITTypeId,
question=mturk_question,
lifetime=hit_config['lifetime'],
max_assignments=hit_config['max_assignments'],
title=hit_config['title'],
description=hit_config['description'],
keywords=hit_config['keywords'],
reward=hit_config['reward'],
duration=hit_config['duration'],
approval_delay=None,
questions=None,
qualifications=quals,
response_groups=[
'Minimal',
'HITDetail',
'HITQuestion',
'HITAssignmentSummary'
])
def check_balance(self):
''' Check balance '''
if not self.connect_to_turk():
return '-'
return self.mtc.get_account_balance()[0]
# TODO (if valid AWS credentials haven't been provided then
# connect_to_turk() will fail, not error checking here and elsewhere)
def create_hit(self, hit_config):
''' Create HIT '''
try:
if not self.connect_to_turk():
return False
self.configure_hit(hit_config)
myhit = self.mtc.create_hit(**self.param_dict)[0]
self.hitid = myhit.HITId
except:
return False
else:
return self.hitid
# TODO(Jay): Have a wrapper around functions that serializes them.
# Default output should not be serialized.
def expire_hit(self, hitid):
''' Expire HIT '''
if not self.connect_to_turk():
return False
try:
self.mtc.expire_hit(hitid)
return True
except MTurkRequestError:
print "Failed to expire HIT. Please check the ID and try again."
return False
def dispose_hit(self, hitid):
''' Dispose HIT '''
if not self.connect_to_turk():
return False
try:
self.mtc.dispose_hit(hitid)
except Exception, e:
print "Failed to dispose of HIT %s. Make sure there are no "\
"assignments remaining to be reviewed." % hitid
mainly information that you do not want to share.
""")
parser.add_argument('-c', "--configs", nargs='+',
help='additional configuration files')
args = parser.parse_args()
mturk_cfg_fname = as_project_path('resources/private/mturk.cfg')
cfg = Config.load_configs([mturk_cfg_fname] + args.configs, log=False)
print "Delete all HITs"
conn = MTurkConnection(aws_access_key_id = cfg['MTURK']['aws_access_key_id'],
aws_secret_access_key = cfg['MTURK']['aws_secret_access_key'],
host = cfg['MTURK']['host'])
for pnum in range(1, 50):
for hit in conn.get_reviewable_hits(page_size=100, page_number=pnum):
print "HITId:", hit.HITId
hitStatus = defaultdict(int)
for ass in conn.get_assignments(hit.HITId, status='Submitted', page_size=10, page_number=1):
#print "Dir ass:", dir(ass)
hitStatus[ass.AssignmentStatus] += 1
print hitStatus
print 'Deleting hit:', hit.HITId
conn.dispose_hit(hit.HITId)
class amusic(object):
engine = None
origconf = {'minPitch':12,'maxPitch':84,'minStartTime':0.0,'maxStartTime':200.0,
'minNoteDuration':0.5,'maxNoteDuration':5.0,'minNoteCount':50,'maxNoteCount':400,
'currentPopulation':'','bucket':None,'hitRewardPerAssignment':0.05,'mturkLayoutID':None,
'hitTitle':None,'hitDescription':None,'hitKeywords':None}
conf = origconf.copy()
s3bucket = None
mtc = None
class PopulationNotSet( Exception ): pass
class SongNotFound( Exception ): pass
def __init__(self,username,password,ACCESS_ID,SECRET_KEY,initialize=False):
self.ACCESS_ID = ACCESS_ID
self.SECRET_KEY = SECRET_KEY
self.engine = sqlalchemy.create_engine('mysql+mysqlconnector://%s:%s@localhost' % (username,password))
self.engine.connect()
self.mtc = MTurkConnection(self.ACCESS_ID,self.SECRET_KEY,host='mechanicalturk.sandbox.amazonaws.com')
try:
self.engine.execute('USE amusic;')
c = self.engine.execute('SELECT * FROM conf;').fetchone()
self.conf['minPitch'] = c[0]
self.conf['maxPitch'] = c[1]
self.conf['minStartTime'] = c[2]
self.conf['maxStartTime'] = c[3]
self.conf['minNoteDuration'] = c[4]
self.conf['maxNoteDuration'] = c[5]
self.conf['minNoteCount'] = c[6]
self.conf['maxNoteCount'] = c[7]
self.conf['currentPopulation'] = c[8]
self.conf['bucket'] = c[9]
self.conf['mturkLayoutID'] = c[10]
self.conf['hitTitle'] = c[11]
self.conf['hitDescription'] = c[12]
self.conf['hitKeywords'] = c[13]
self.conf['hitRewardPerAssignment'] = c[14]
except:pass
def inits3(self):
c = boto.connect_s3(self.ACCESS_ID,self.SECRET_KEY)
self.s3bucket = c.create_bucket(self.conf['bucket'])
self.s3bucket.set_acl('public-read')
def initialize(self):
try: self.engine.execute('DROP DATABASE IF EXISTS amusic;')
except: pass
self.engine.execute('CREATE DATABASE amusic;')
self.engine.execute('USE amusic;')
self.engine.execute('CREATE TABLE conf (minPitch INT,\
maxPitch INT,\
minStartTime FLOAT,\
maxStartTime FLOAT,\
minNoteDuration FLOAT,\
maxNoteDuration FLOAT,\
minNoteCount INT,\
maxNoteCount INT,\
currentPopulation VARCHAR(100),\
bucket VARCHAR(100),\
mturkLayoutID VARCHAR(100),\
hitTitle VARCHAR(100),\
hitDescription VARCHAR(100),\
hitKeywords VARCHAR(100),\
hitRewardPerAssignment FLOAT,\
CONSTRAINT fk_1 FOREIGN KEY (`currentPopulation`) REFERENCES population (title),\
PRIMARY KEY(minPitch)\
) ENGINE = MYISAM;')
self.engine.execute('INSERT INTO conf (minPitch,\
maxPitch,\
minStartTime,\
maxStartTime,\
minNoteDuration,\
maxNoteDuration,\
minNoteCount,\
maxNoteCount,\
currentPopulation,\
bucket,\
hitRewardPerAssignment,\
mturkLayoutID,\
hitTitle,\
hitKeywords,\
hitDescription)\
VALUES(%d,%d,%f,%f,%f,%f,%d,%d,"%s","%s",%f,"%s","%s","%s","%s");'%
(self.origconf['minPitch'],self.origconf['maxPitch'],self.origconf['minStartTime'],
self.origconf['maxStartTime'],self.origconf['minNoteDuration'],
self.origconf['maxNoteDuration'],self.origconf['minNoteCount'],
self.origconf['maxNoteCount'],self.origconf['currentPopulation'],
self.origconf['bucket'], self.origconf['hitRewardPerAssignment'],
self.origconf['mturkLayoutID'],self.origconf['hitTitle'],
self.origconf['hitKeywords'],self.origconf['hitDescription']))
self.engine.execute('CREATE TABLE population (title VARCHAR(100) NOT NULL,\
PRIMARY KEY(title)\
) ENGINE = MYISAM;')
self.engine.execute('CREATE TABLE song (id INT NOT NULL AUTO_INCREMENT,\
title VARCHAR(100),\
population VARCHAR(100),\
ppq INT,\
CONSTRAINT fk_1 FOREIGN KEY (`population`) REFERENCES population (title),\
PRIMARY KEY(id,title)\
) ENGINE = MYISAM;')
self.engine.execute('CREATE TABLE event (songID INT,\
track INT,\
id INT NOT NULL AUTO_INCREMENT,\
type VARCHAR(5),\
pitch INT,\
value INT,\
startTime FLOAT,\
duration FLOAT,\
velocity INT,\
CONSTRAINT fk_1 FOREIGN KEY (`songID`) REFERENCES song (id),\
PRIMARY KEY(songID,track,id)\
) ENGINE = MYISAM;')
def setPopulation(self,title):
self.engine.execute('UPDATE conf SET currentPopulation="%s";' % title)
self.conf['currentPopulation'] = title
def newPopulation(self,title):
return Population(self,title)
def getCurrentPopulation(self):
if self.conf['currentPopulation'] == '':
raise self.PopulationNotSet
return Population(self,self.conf['currentPopulation'])
def getPopulation(self,title):
return Population(self,title,create=False)
def listHITs(self):
print " ".join(('%s','%s','%s','%s','%s')) % ("HIT ID".ljust(30), "Status".ljust(22), "Amount", "Song1", "Song2")
for i in self.mtc.search_hits(response_groups=['Request','Minimal','HITDetail','HITQuestion']):
l = re.findall('<input type="hidden" value="([^"]+?)" name="song[12]" />',i.Question)
print ' '.join((i.HITId, i.HITStatus,i.HITReviewStatus,i.Amount,l[0].split('/')[-1],l[1].split('/')[-1]))
def deleteHITs(self):
for i in self.mtc.get_reviewable_hits():
self.mtc.dispose_hit(i.HITId)
def approveAssignment(self,aID):
self.mtc.approve_assignment(aID)
def rejectAssignment(self,aID):
self.mtc.reject_assignment(aID)
def approveAllAssignments(self):
for i in self.mtc.get_reviewable_hits():
for assignment in self.mtc.get_assignments(i.HITId):
if assignment.AssignmentStatus=="Submitted":
self.mtc.approve_assignment(assignment.AssignmentId)
def getResults(self):
print " ".join(('%s','%s','%s','%s','%s')) % ("Assignment ID".ljust(30), "Worker ID".ljust(14), "Song1", "Song2", "Answer")
for i in self.mtc.get_reviewable_hits():
for assignment in self.mtc.get_assignments(i.HITId):
ans = {}
for j in assignment.answers[0]:
if j.qid!='commit': ans[j.qid] = j.fields[0].split('/')[-1]
print assignment.AssignmentId, assignment.WorkerId,ans['song1'], ans['song2'], ans['boxradio']
help='additional configuration files')
args = parser.parse_args()
mturk_cfg_fname = as_project_path('resources/private/mturk.cfg')
cfg = Config.load_configs([mturk_cfg_fname] + args.configs, log=False)
print "Delete approved, rejected, or expired HITs"
conn = MTurkConnection(
aws_access_key_id=cfg['MTURK']['aws_access_key_id'],
aws_secret_access_key=cfg['MTURK']['aws_secret_access_key'],
host=cfg['MTURK']['host'])
for pnum in range(1, 350):
for hit in conn.get_reviewable_hits(page_size=100, page_number=pnum):
print "HITId:", hit.HITId
hitStatus = defaultdict(int)
for ass in conn.get_assignments(hit.HITId,
status='Submitted',
page_size=10,
page_number=1):
#print "Dir ass:", dir(ass)
hitStatus[ass.AssignmentStatus] += 1
print hitStatus
if 'Submitted' not in hitStatus:
print 'Deleting hit:', hit.HITId
conn.dispose_hit(hit.HITId)
class MTurkServices:
def __init__(self, aws_access_key_id, aws_secret_access_key, is_sandbox):
self.update_credentials(aws_access_key_id, aws_secret_access_key)
self.set_sandbox(is_sandbox)
self.validLogin = self.verify_aws_login()
if not self.validLogin:
print 'WARNING *****************************'
print 'Sorry, AWS Credentials invalid.\nYou will only be able to '\
+ 'test experiments locally until you enter\nvalid '\
+ 'credentials in the AWS Access section of ~/.psiturkconfig\n'
def update_credentials(self, aws_access_key_id, aws_secret_access_key):
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
def set_sandbox(self, is_sandbox):
self.is_sandbox = is_sandbox
def get_reviewable_hits(self):
if not self.connect_to_turk():
return False
try:
hits = self.mtc.get_all_hits()
except MTurkRequestError:
return False
reviewable_hits = [hit for hit in hits if (hit.HITStatus == "Reviewable" or hit.HITStatus == "Reviewing")]
hits_data = [MTurkHIT({'hitid': hit.HITId,
'title': hit.Title,
'status': hit.HITStatus,
'max_assignments': hit.MaxAssignments,
'number_assignments_completed': hit.NumberOfAssignmentsCompleted,
'number_assignments_pending': hit.NumberOfAssignmentsPending,
'number_assignments_available': hit.NumberOfAssignmentsAvailable,
'creation_time': hit.CreationTime,
'expiration': hit.Expiration,
}) for hit in reviewable_hits]
return(hits_data)
def get_all_hits(self):
if not self.connect_to_turk():
return False
try:
hits = self.mtc.get_all_hits()
except MTurkRequestError:
return False
hits_data = [MTurkHIT({'hitid': hit.HITId,
'title': hit.Title,
'status': hit.HITStatus,
'max_assignments': hit.MaxAssignments,
'number_assignments_completed': hit.NumberOfAssignmentsCompleted,
'number_assignments_pending': hit.NumberOfAssignmentsPending,
'number_assignments_available': hit.NumberOfAssignmentsAvailable,
'creation_time': hit.CreationTime,
'expiration': hit.Expiration,
}) for hit in hits]
return(hits_data)
def get_active_hits(self):
if not self.connect_to_turk():
return False
# hits = self.mtc.search_hits()
try:
hits = self.mtc.get_all_hits()
except MTurkRequestError:
return False
active_hits = [hit for hit in hits if not(hit.expired)]
hits_data = [MTurkHIT({'hitid': hit.HITId,
'title': hit.Title,
'status': hit.HITStatus,
'max_assignments': hit.MaxAssignments,
'number_assignments_completed': hit.NumberOfAssignmentsCompleted,
'number_assignments_pending': hit.NumberOfAssignmentsPending,
'number_assignments_available': hit.NumberOfAssignmentsAvailable,
'creation_time': hit.CreationTime,
'expiration': hit.Expiration,
}) for hit in active_hits]
return(hits_data)
def get_workers(self, assignmentStatus = None):
if not self.connect_to_turk():
return False
try:
hits = self.mtc.search_hits(sort_direction='Descending', page_size=20)
hit_ids = [hit.HITId for hit in hits]
workers_nested = [self.mtc.get_assignments(
hit_id,
status=assignmentStatus,
sort_by='SubmitTime',
page_size=100
) for hit_id in hit_ids]
workers = [val for subl in workers_nested for val in subl] # Flatten nested lists
except MTurkRequestError:
return(False)
worker_data = [{'hitId': worker.HITId,
'assignmentId': worker.AssignmentId,
'workerId': worker.WorkerId,
'submit_time': worker.SubmitTime,
'accept_time': worker.AcceptTime,
'status': worker.AssignmentStatus
} for worker in workers]
return(worker_data)
def bonus_worker(self, assignment_id, amount, reason=""):
if not self.connect_to_turk():
return False
try:
bonus = MTurkConnection.get_price_as_price(amount)
assignment = self.mtc.get_assignment(assignment_id)[0]
workerId = assignment.WorkerId
self.mtc.grant_bonus(workerId, assignment_id, bonus, reason)
return True
except MTurkRequestError as e:
print e
return False
def approve_worker(self, assignment_id):
if not self.connect_to_turk():
return(False)
try:
self.mtc.approve_assignment(assignment_id, feedback=None)
return True
except MTurkRequestError:
return(False)
def reject_worker(self, assignment_id):
if not self.connect_to_turk():
return False
try:
self.mtc.reject_assignment(assignment_id, feedback=None)
return True
except MTurkRequestError:
return(False)
def unreject_worker(self, assignment_id):
if not self.connect_to_turk():
return False
try:
self.mtc.approve_rejected_assignment(assignment_id)
return True
except MTurkRequestError:
return False
def verify_aws_login(self):
if (self.aws_access_key_id == 'YourAccessKeyId') or (self.aws_secret_access_key == 'YourSecretAccessKey'):
return False
else:
host = 'mechanicalturk.amazonaws.com'
mturkparams = dict(
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
host=host)
self.mtc = MTurkConnection(**mturkparams)
try:
self.mtc.get_account_balance()
except MTurkRequestError as e:
print(e.error_message)
return False
else:
return True
def connect_to_turk(self):
if not self.validLogin:
print 'Sorry, unable to connect to Amazon Mechanical Turk. AWS credentials invalid.'
return False
if self.is_sandbox:
host = 'mechanicalturk.sandbox.amazonaws.com'
else:
host = 'mechanicalturk.amazonaws.com'
mturkparams = dict(
aws_access_key_id = self.aws_access_key_id,
aws_secret_access_key = self.aws_secret_access_key,
host=host)
self.mtc = MTurkConnection(**mturkparams)
return True
def configure_hit(self, hit_config):
# configure question_url based on the id
experimentPortalURL = hit_config['ad_location']
frameheight = 600
mturkQuestion = ExternalQuestion(experimentPortalURL, frameheight)
# Qualification:
quals = Qualifications()
approve_requirement = hit_config['approve_requirement']
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if hit_config['us_only']:
quals.add(LocaleRequirement("EqualTo", "US"))
# Specify all the HIT parameters
self.paramdict = dict(
hit_type = None,
question = mturkQuestion,
lifetime = hit_config['lifetime'],
max_assignments = hit_config['max_assignments'],
title = hit_config['title'],
description = hit_config['description'],
keywords = hit_config['keywords'],
reward = hit_config['reward'],
duration = hit_config['duration'],
approval_delay = None,
questions = None,
qualifications = quals
)
def check_balance(self):
if not self.connect_to_turk():
return('-')
return(self.mtc.get_account_balance()[0])
# TODO (if valid AWS credentials haven't been provided then connect_to_turk() will
# fail, not error checking here and elsewhere)
def create_hit(self, hit_config):
try:
if not self.connect_to_turk():
return False
self.configure_hit(hit_config)
myhit = self.mtc.create_hit(**self.paramdict)[0]
self.hitid = myhit.HITId
except:
return False
else:
return self.hitid
# TODO(Jay): Have a wrapper around functions that serializes them.
# Default output should not be serialized.
def expire_hit(self, hitid):
if not self.connect_to_turk():
return False
try:
self.mtc.expire_hit(hitid)
return True
except MTurkRequestError:
print "Failed to expire HIT. Please check the ID and try again."
return False
def dispose_hit(self, hitid):
if not self.connect_to_turk():
return False
try:
self.mtc.dispose_hit(hitid)
except Exception, e:
print 'Failed to dispose of HIT %s. Make sure there are no assignments remaining to be reviewed' % hitid
class MTurkServices(object):
''' MTurk services '''
def __init__(self, aws_access_key_id, aws_secret_access_key, is_sandbox):
self.update_credentials(aws_access_key_id, aws_secret_access_key)
self.set_sandbox(is_sandbox)
self.valid_login = self.verify_aws_login()
if not self.valid_login:
print 'WARNING *****************************'
print 'Sorry, AWS Credentials invalid.\nYou will only be able to '\
'test experiments locally until you enter\nvalid '\
'credentials in the AWS Access section of ~/.psiturkconfig\n'
def update_credentials(self, aws_access_key_id, aws_secret_access_key):
''' Update credentials '''
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
def set_sandbox(self, is_sandbox):
''' Set sandbox '''
self.is_sandbox = is_sandbox
def get_reviewable_hits(self):
''' Get reviewable HITs '''
if not self.connect_to_turk():
return False
try:
hits = self.mtc.get_all_hits()
except MTurkRequestError:
return False
reviewable_hits = [hit for hit in hits if hit.HITStatus == "Reviewable" \
or hit.HITStatus == "Reviewing"]
hits_data = [MTurkHIT({
'hitid': hit.HITId,
'title': hit.Title,
'status': hit.HITStatus,
'max_assignments': hit.MaxAssignments,
'number_assignments_completed': hit.NumberOfAssignmentsCompleted,
'number_assignments_pending': hit.NumberOfAssignmentsPending,
'number_assignments_available': hit.NumberOfAssignmentsAvailable,
'creation_time': hit.CreationTime,
'expiration': hit.Expiration
}) for hit in reviewable_hits]
return hits_data
def get_all_hits(self):
''' Get all HITs '''
if not self.connect_to_turk():
return False
try:
hits = self.mtc.get_all_hits()
except MTurkRequestError:
return False
hits_data = [MTurkHIT({
'hitid': hit.HITId,
'title': hit.Title,
'status': hit.HITStatus,
'max_assignments': hit.MaxAssignments,
'number_assignments_completed': hit.NumberOfAssignmentsCompleted,
'number_assignments_pending': hit.NumberOfAssignmentsPending,
'number_assignments_available': hit.NumberOfAssignmentsAvailable,
'creation_time': hit.CreationTime,
'expiration': hit.Expiration,
}) for hit in hits]
return hits_data
def get_active_hits(self):
''' Get active HITs '''
if not self.connect_to_turk():
return False
# hits = self.mtc.search_hits()
try:
hits = self.mtc.get_all_hits()
except MTurkRequestError:
return False
active_hits = [hit for hit in hits if not hit.expired]
hits_data = [MTurkHIT({
'hitid': hit.HITId,
'title': hit.Title,
'status': hit.HITStatus,
'max_assignments': hit.MaxAssignments,
'number_assignments_completed': hit.NumberOfAssignmentsCompleted,
'number_assignments_pending': hit.NumberOfAssignmentsPending,
'number_assignments_available': hit.NumberOfAssignmentsAvailable,
'creation_time': hit.CreationTime,
'expiration': hit.Expiration,
}) for hit in active_hits]
return hits_data
def get_workers(self, assignment_status=None, chosen_hit=None):
''' Get workers '''
if not self.connect_to_turk():
return False
try:
if chosen_hit:
hit_ids = [chosen_hit]
else:
hits = self.mtc.get_all_hits()
hit_ids = [hit.HITId for hit in hits]
workers_nested = []
page_size=100
for hit_id in hit_ids:
current_page_number=1
hit_assignments = self.mtc.get_assignments(
hit_id,
status=assignment_status,
sort_by='SubmitTime',
page_size=page_size,
page_number=current_page_number
)
totalNumResults = int(hit_assignments.TotalNumResults)
total_pages = (totalNumResults // page_size) + (totalNumResults % page_size > 0) #do integer division then round up if necessary
while current_page_number < total_pages:
current_page_number += 1
hit_assignments += self.mtc.get_assignments(
hit_id,
status=assignment_status,
sort_by='SubmitTime',
page_size=page_size,
page_number=current_page_number
)
workers_nested.append(hit_assignments)
workers = [val for subl in workers_nested for val in subl] # Flatten nested lists
except MTurkRequestError:
return False
worker_data = [{
'hitId': worker.HITId,
'assignmentId': worker.AssignmentId,
'workerId': worker.WorkerId,
'submit_time': worker.SubmitTime,
'accept_time': worker.AcceptTime,
'status': worker.AssignmentStatus
} for worker in workers]
return worker_data
def get_worker(self, assignment_id):
if not self.connect_to_turk():
return False
try:
worker = self.mtc.get_assignment(assignment_id)[0]
except MTurkRequestError as e:
return False
worker_data = [{
'hitId': worker.HITId,
'assignmentId': worker.AssignmentId,
'workerId': worker.WorkerId,
'submit_time': worker.SubmitTime,
'accept_time': worker.AcceptTime,
'status': worker.AssignmentStatus
}]
return worker_data
def bonus_worker(self, assignment_id, amount, reason=""):
''' Bonus worker '''
if not self.connect_to_turk():
return False
try:
bonus = MTurkConnection.get_price_as_price(amount)
assignment = self.mtc.get_assignment(assignment_id)[0]
worker_id = assignment.WorkerId
self.mtc.grant_bonus(worker_id, assignment_id, bonus, reason)
return True
except MTurkRequestError as exception:
print exception
return False
def approve_worker(self, assignment_id):
''' Approve worker '''
if not self.connect_to_turk():
return False
try:
self.mtc.approve_assignment(assignment_id, feedback=None)
return True
except MTurkRequestError as e:
return False
def reject_worker(self, assignment_id):
''' Reject worker '''
if not self.connect_to_turk():
return False
try:
self.mtc.reject_assignment(assignment_id, feedback=None)
return True
except MTurkRequestError:
return False
def unreject_worker(self, assignment_id):
''' Unreject worker '''
if not self.connect_to_turk():
return False
try:
self.mtc.approve_rejected_assignment(assignment_id)
return True
except MTurkRequestError:
return False
def verify_aws_login(self):
''' Verify AWS login '''
if ((self.aws_access_key_id == 'YourAccessKeyId') or
(self.aws_secret_access_key == 'YourSecretAccessKey')):
return False
else:
host = 'mechanicalturk.amazonaws.com'
mturkparams = dict(
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
host=host)
self.mtc = MTurkConnection(**mturkparams)
try:
self.mtc.get_account_balance()
except MTurkRequestError as exception:
print exception.error_message
return False
else:
return True
def connect_to_turk(self):
''' Connect to turk '''
if not self.valid_login:
print 'Sorry, unable to connect to Amazon Mechanical Turk. AWS '\
'credentials invalid.'
return False
if self.is_sandbox:
host = 'mechanicalturk.sandbox.amazonaws.com'
else:
host = 'mechanicalturk.amazonaws.com'
mturkparams = dict(
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
host=host)
self.mtc = MTurkConnection(**mturkparams)
return True
def configure_hit(self, hit_config):
''' Configure HIT '''
# configure question_url based on the id
experiment_portal_url = hit_config['ad_location']
frame_height = 600
mturk_question = ExternalQuestion(experiment_portal_url, frame_height)
# Qualification:
quals = Qualifications()
approve_requirement = hit_config['approve_requirement']
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
number_hits_approved = hit_config['number_hits_approved']
quals.add(
NumberHitsApprovedRequirement("GreaterThanOrEqualTo",
number_hits_approved))
require_master_workers = hit_config['require_master_workers']
if require_master_workers:
quals.add(MasterRequirement(sandbox=self.is_sandbox))
if hit_config['us_only']:
quals.add(LocaleRequirement("EqualTo", "US"))
# Create a HIT type for this HIT.
hit_type = self.mtc.register_hit_type(
hit_config['title'],
hit_config['description'],
hit_config['reward'],
hit_config['duration'],
keywords=hit_config['keywords'],
approval_delay=None,
qual_req=quals)[0]
# Check the config file to see if notifications are wanted.
config = PsiturkConfig()
config.load_config()
try:
url = config.get('Server Parameters', 'notification_url')
all_event_types = [
"AssignmentAccepted",
"AssignmentAbandoned",
"AssignmentReturned",
"AssignmentSubmitted",
"HITReviewable",
"HITExpired",
]
self.mtc.set_rest_notification(
hit_type.HITTypeId,
url,
event_types=all_event_types)
except:
pass
# Specify all the HIT parameters
self.param_dict = dict(
hit_type=hit_type.HITTypeId,
question=mturk_question,
lifetime=hit_config['lifetime'],
max_assignments=hit_config['max_assignments'],
questions=None,
response_groups=[
'Minimal',
'HITDetail',
'HITQuestion',
'HITAssignmentSummary'
])
def check_balance(self):
''' Check balance '''
if not self.connect_to_turk():
return '-'
return self.mtc.get_account_balance()[0]
# TODO (if valid AWS credentials haven't been provided then
# connect_to_turk() will fail, not error checking here and elsewhere)
def create_hit(self, hit_config):
''' Create HIT '''
try:
if not self.connect_to_turk():
return False
self.configure_hit(hit_config)
myhit = self.mtc.create_hit(**self.param_dict)[0]
self.hitid = myhit.HITId
except MTurkRequestError as e:
print e
return False
else:
return self.hitid
# TODO(Jay): Have a wrapper around functions that serializes them.
# Default output should not be serialized.
def expire_hit(self, hitid):
''' Expire HIT '''
if not self.connect_to_turk():
return False
try:
self.mtc.expire_hit(hitid)
return True
except MTurkRequestError:
print "Failed to expire HIT. Please check the ID and try again."
return False
def dispose_hit(self, hitid):
''' Dispose HIT '''
if not self.connect_to_turk():
return False
try:
self.mtc.dispose_hit(hitid)
except Exception, e:
print "Failed to dispose of HIT %s. Make sure there are no "\
"assignments remaining to be reviewed." % hitid
class MTurkServices:
def __init__(self, aws_access_key_id, aws_secret_access_key, is_sandbox):
self.update_credentials(aws_access_key_id, aws_secret_access_key)
self.set_sandbox(is_sandbox)
self.validLogin = self.verify_aws_login()
if not self.validLogin:
print 'WARNING *****************************'
print 'Sorry, AWS Credentials invalid.\nYou will only be able to '\
+ 'test experiments locally until you enter\nvalid '\
+ 'credentials in the AWS Access section of ~/.psiturkconfig\n'
def update_credentials(self, aws_access_key_id, aws_secret_access_key):
self.aws_access_key_id = aws_access_key_id
self.aws_secret_access_key = aws_secret_access_key
def set_sandbox(self, is_sandbox):
self.is_sandbox = is_sandbox
def get_reviewable_hits(self):
if not self.connect_to_turk():
return False
try:
hits = self.mtc.get_all_hits()
except MTurkRequestError:
return False
reviewable_hits = [
hit for hit in hits
if (hit.HITStatus == "Reviewable" or hit.HITStatus == "Reviewing")
]
hits_data = [
MTurkHIT({
'hitid': hit.HITId,
'title': hit.Title,
'status': hit.HITStatus,
'max_assignments': hit.MaxAssignments,
'number_assignments_completed':
hit.NumberOfAssignmentsCompleted,
'number_assignments_pending': hit.NumberOfAssignmentsPending,
'number_assignments_available':
hit.NumberOfAssignmentsAvailable,
'creation_time': hit.CreationTime,
'expiration': hit.Expiration,
}) for hit in reviewable_hits
]
return (hits_data)
def get_all_hits(self):
if not self.connect_to_turk():
return False
try:
hits = self.mtc.get_all_hits()
except MTurkRequestError:
return False
hits_data = [
MTurkHIT({
'hitid': hit.HITId,
'title': hit.Title,
'status': hit.HITStatus,
'max_assignments': hit.MaxAssignments,
'number_assignments_completed':
hit.NumberOfAssignmentsCompleted,
'number_assignments_pending': hit.NumberOfAssignmentsPending,
'number_assignments_available':
hit.NumberOfAssignmentsAvailable,
'creation_time': hit.CreationTime,
'expiration': hit.Expiration,
}) for hit in hits
]
return (hits_data)
def get_active_hits(self):
if not self.connect_to_turk():
return False
# hits = self.mtc.search_hits()
try:
hits = self.mtc.get_all_hits()
except MTurkRequestError:
return False
active_hits = [hit for hit in hits if not (hit.expired)]
hits_data = [
MTurkHIT({
'hitid': hit.HITId,
'title': hit.Title,
'status': hit.HITStatus,
'max_assignments': hit.MaxAssignments,
'number_assignments_completed':
hit.NumberOfAssignmentsCompleted,
'number_assignments_pending': hit.NumberOfAssignmentsPending,
'number_assignments_available':
hit.NumberOfAssignmentsAvailable,
'creation_time': hit.CreationTime,
'expiration': hit.Expiration,
}) for hit in active_hits
]
return (hits_data)
def get_workers(self, assignmentStatus=None):
if not self.connect_to_turk():
return False
try:
hits = self.mtc.search_hits(sort_direction='Descending',
page_size=20)
hit_ids = [hit.HITId for hit in hits]
workers_nested = [
self.mtc.get_assignments(hit_id,
status=assignmentStatus,
sort_by='SubmitTime',
page_size=100) for hit_id in hit_ids
]
workers = [val for subl in workers_nested
for val in subl] # Flatten nested lists
except MTurkRequestError:
return (False)
worker_data = [{
'hitId': worker.HITId,
'assignmentId': worker.AssignmentId,
'workerId': worker.WorkerId,
'submit_time': worker.SubmitTime,
'accept_time': worker.AcceptTime,
'status': worker.AssignmentStatus
} for worker in workers]
return (worker_data)
def bonus_worker(self, assignment_id, amount, reason=""):
if not self.connect_to_turk():
return False
try:
bonus = MTurkConnection.get_price_as_price(amount)
assignment = self.mtc.get_assignment(assignment_id)[0]
workerId = assignment.WorkerId
self.mtc.grant_bonus(workerId, assignment_id, bonus, reason)
return True
except MTurkRequestError as e:
print e
return False
def approve_worker(self, assignment_id):
if not self.connect_to_turk():
return (False)
try:
self.mtc.approve_assignment(assignment_id, feedback=None)
return True
except MTurkRequestError:
return (False)
def reject_worker(self, assignment_id):
if not self.connect_to_turk():
return False
try:
self.mtc.reject_assignment(assignment_id, feedback=None)
return True
except MTurkRequestError:
return (False)
def unreject_worker(self, assignment_id):
if not self.connect_to_turk():
return False
try:
self.mtc.approve_rejected_assignment(assignment_id)
return True
except MTurkRequestError:
return False
def verify_aws_login(self):
if (self.aws_access_key_id
== 'YourAccessKeyId') or (self.aws_secret_access_key
== 'YourSecretAccessKey'):
return False
else:
host = 'mechanicalturk.amazonaws.com'
mturkparams = dict(
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
host=host)
self.mtc = MTurkConnection(**mturkparams)
try:
self.mtc.get_account_balance()
except MTurkRequestError as e:
print(e.error_message)
return False
else:
return True
def connect_to_turk(self):
if not self.validLogin:
print 'Sorry, unable to connect to Amazon Mechanical Turk. AWS credentials invalid.'
return False
if self.is_sandbox:
host = 'mechanicalturk.sandbox.amazonaws.com'
else:
host = 'mechanicalturk.amazonaws.com'
mturkparams = dict(aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
host=host)
self.mtc = MTurkConnection(**mturkparams)
return True
def configure_hit(self, hit_config):
# configure question_url based on the id
experimentPortalURL = hit_config['ad_location']
frameheight = 600
mturkQuestion = ExternalQuestion(experimentPortalURL, frameheight)
# Qualification:
quals = Qualifications()
approve_requirement = hit_config['approve_requirement']
quals.add(
PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo",
approve_requirement))
if hit_config['us_only']:
quals.add(LocaleRequirement("EqualTo", "US"))
# Specify all the HIT parameters
self.paramdict = dict(hit_type=None,
question=mturkQuestion,
lifetime=hit_config['lifetime'],
max_assignments=hit_config['max_assignments'],
title=hit_config['title'],
description=hit_config['description'],
keywords=hit_config['keywords'],
reward=hit_config['reward'],
duration=hit_config['duration'],
approval_delay=None,
questions=None,
qualifications=quals)
def check_balance(self):
if not self.connect_to_turk():
return ('-')
return (self.mtc.get_account_balance()[0])
# TODO (if valid AWS credentials haven't been provided then connect_to_turk() will
# fail, not error checking here and elsewhere)
def create_hit(self, hit_config):
try:
if not self.connect_to_turk():
return False
self.configure_hit(hit_config)
myhit = self.mtc.create_hit(**self.paramdict)[0]
self.hitid = myhit.HITId
except:
return False
else:
return self.hitid
# TODO(Jay): Have a wrapper around functions that serializes them.
# Default output should not be serialized.
def expire_hit(self, hitid):
if not self.connect_to_turk():
return False
try:
self.mtc.expire_hit(hitid)
return True
except MTurkRequestError:
print "Failed to expire HIT. Please check the ID and try again."
return False
def dispose_hit(self, hitid):
if not self.connect_to_turk():
return False
try:
self.mtc.dispose_hit(hitid)
except Exception, e:
print 'Failed to dispose of HIT %s. Make sure there are no assignments remaining to be reviewed' % hitid
class HaCRSTurker:
def __init__(self):
self.config = HaCRSUtil.get_config('../config.ini')
HOST = self.config.get('mturk', 'host')
AWS_ACCESS_KEY_ID = self.config.get('mturk', 'access_key_id')
AWS_SECRET_ACCESS_KEY = self.config.get('mturk', 'secret_access_key')
self.MTconnection = MTurkConnection(
aws_access_key_id=AWS_ACCESS_KEY_ID,
aws_secret_access_key=AWS_SECRET_ACCESS_KEY,
host=HOST)
self.db = HaCRSDB()
def get_balance(self):
#print self.MTconnection.get_account_balance()
pass
def expire_all_hits(self):
all_hits = self.MTconnection.get_all_hits()
for hit in all_hits:
if hit.expired:
continue
try:
self.MTconnection.expire_hit(hit.HITId)
#print 'Expired HIT'
except Exception as e:
#print 'Could not expire: {}'.format(e)
pass
def delete_all_mturk_hits(self):
all_hits = self.MTconnection.get_all_hits()
for hit in all_hits:
#print 'expire/dispose'
self.MTconnection.expire_hit(hit.HITId)
self.MTconnection.dispose_hit(hit.HITId)
def get_all_mturk_hits(self):
all_hits = self.MTconnection.get_all_hits()
return all_hits
# TODO: HITs available via API, but not via Amazon Web Sandbox
def push_tasklet_mturk(self, keywords):
sdescription = self.config.get('mturk', 'shortdescr')
frame_height = self.config.get('mturk', 'frameheight')
#url = "https://cgcturk.hacked.jp/tasklet/{}/".format(tasklet['id'])
url = "https://cgcturk.hacked.jp/pick_tasklet/{}/".format(keywords)
#keywords = tasklet['keywords']
#amount = tasklet['amount']
if keywords == 'easy':
amount = 1.00
elif keywords in ['medium', 'hard', 'very_hard']:
amount = 2.00
elif keywords == 'priority':
amount = 4.00
else:
#print 'Error'
sys.exit(1)
questionform = ExternalQuestion(url, frame_height)
title = 'HELP AN AI!!! We are students building an artificial intelligence to find bugs in programs to keep the internet safe'
sdescription = 'We are students building an artificial intelligence system that finds bugs in programs and keeps the internet safe from malware. BUT IT NEEDS YOUR HELP! Play with programs to find functions that it missed, and get $$$!'
hit_result = self.MTconnection.create_hit(
title='[{}] {}'.format(keywords, title),
description=sdescription,
keywords=keywords,
max_assignments=1,
question=questionform,
reward=Price(amount=amount),
response_groups=('Minimal', 'HITDetail'), # ?
)
assert len(hit_result) == 1
mturkid = self.db.create_mturk_resource(hit_result[0].HITId,
hit_result[0].HITGroupId)
#self.db.add_mturk_tasklet_association(tasklet['id'], mturkid)
#self.db.commit()
return mturkid, hit_result
def push_tasks_mturk(self):
frame_height = self.config.get('mturk', 'frameheight')
amount = 0.01
tasklets = self.db.get_unassigned_tasklets()
sdescription = self.config.get('mturk', 'shortdescr')
for tasklet in tasklets:
#print 'pushing!'
url = "https://cgcturk.hacked.jp/tasklet/{}/".format(tasklet['id'])
keywords = ["easy"]
questionform = ExternalQuestion(url, frame_height)
hit_result = self.MTconnection.create_hit(
title=HaCRSUtil.get_tasklet_name(tasklet),
description=sdescription,
keywords=keywords,
max_assignments=1,
question=questionform,
reward=Price(amount=amount),
response_groups=('Minimal', 'HITDetail'), # ?
)
assert len(hit_result) == 1
mturkid = self.db.create_mturk_resource(hit_result[0].HITId,
hit_result[0].HITGroupId)
self.db.add_mturk_tasklet_association(tasklet['id'], mturkid)
self.db.commit()
def show_seed_tasklets(self):
pprint(self.db.get_seed_tasklets())
def get_hit(self, hitid):
try:
hit = self.MTconnection.get_hit(hitid)
except Exception as e:
return None
if hit != None:
return hit[0]
def get_assignment_from_hit(self, hitid):
try:
assignments = self.MTconnection.get_assignments(hitid)
return assignments[0]
except Exception as e:
return None
def get_approved_seeding_tasklets(self):
for program in json.load(
open(self.config.get('general', 'programsjson'))):
pid = self.db.lookup_program(program)
program = None
approved = set()
for tasklet in self.db.get_latest_seed_tasklets():
turkinfos = self.db.get_mturk_infos(tasklet['id'])
try:
#hit = self.MTconnection.get_hit(turkinfos['hitid'])
assignments = self.MTconnection.get_assignments(
turkinfos['hitid'])
if len(assignments) == 0:
continue
if assignments[0].AssignmentStatus == 'Approved':
approved.add(self.db.get_tasklet_program(tasklet['id']))
except Exception as e:
#print e
pass
return list(approved)
def cleanup():
"""Remove any boto test related HIT's"""
conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com')
current_page = 1
page_size = 10
total_disabled = 0
ignored = []
while True:
# reset the total for this loop
disabled_count = 0
# search all the hits in the sandbox
search_rs = conn.search_hits(page_size=page_size, page_number=current_page)
# success?
if search_rs.status:
for hit in search_rs:
# delete any with Boto in the description
print 'hit id:%s Status:%s, desc:%s' %(hit.HITId, hit.HITStatus, hit.Description)
if hit.Description.find('Boto') != -1:
if hit.HITStatus != 'Reviewable':
print 'Disabling hit id:%s %s' %(hit.HITId, hit.Description)
disable_rs = conn.disable_hit(hit.HITId)
if disable_rs.status:
disabled_count += 1
# update the running total
total_disabled += 1
else:
print 'Error when disabling, code:%s, message:%s' %(disable_rs.Code, disable_rs.Message)
else:
print 'Disposing hit id:%s %s' %(hit.HITId, hit.Description)
dispose_rs = conn.dispose_hit(hit.HITId)
if dispose_rs.status:
disabled_count += 1
# update the running total
total_disabled += 1
else:
print 'Error when disposing, code:%s, message:%s' %(dispose_rs.Code, dispose_rs.Message)
else:
if hit.HITId not in ignored:
print 'ignored:%s' %hit.HITId
ignored.append(hit.HITId)
# any more results?
if int(search_rs.TotalNumResults) > current_page*page_size:
# if we have disabled any HITs on this page
# then we don't need to go to a new page
# otherwise we do
if not disabled_count:
current_page += 1
else:
# no, we're done
break
else:
print 'Error performing search, code:%s, message:%s' %(search_rs.Code, search_rs.Message)
break
total_ignored = len(ignored)
print 'Processed: %d HITs, disabled/disposed: %d, ignored: %d' %(total_ignored + total_disabled, total_disabled, total_ignored)
def cleanup():
"""Remove any boto test related HIT's"""
conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com')
current_page = 1
page_size = 10
total_disabled = 0
ignored = []
while True:
# reset the total for this loop
disabled_count = 0
# search all the hits in the sandbox
search_rs = conn.search_hits(page_size=page_size,
page_number=current_page)
# success?
if search_rs.status:
for hit in search_rs:
# delete any with Boto in the description
print 'hit id:%s Status:%s, desc:%s' % (
hit.HITId, hit.HITStatus, hit.Description)
if hit.Description.find('Boto') != -1:
if hit.HITStatus != 'Reviewable':
print 'Disabling hit id:%s %s' % (hit.HITId,
hit.Description)
disable_rs = conn.disable_hit(hit.HITId)
if disable_rs.status:
disabled_count += 1
# update the running total
total_disabled += 1
else:
print 'Error when disabling, code:%s, message:%s' % (
disable_rs.Code, disable_rs.Message)
else:
print 'Disposing hit id:%s %s' % (hit.HITId,
hit.Description)
dispose_rs = conn.dispose_hit(hit.HITId)
if dispose_rs.status:
disabled_count += 1
# update the running total
total_disabled += 1
else:
print 'Error when disposing, code:%s, message:%s' % (
dispose_rs.Code, dispose_rs.Message)
else:
if hit.HITId not in ignored:
print 'ignored:%s' % hit.HITId
ignored.append(hit.HITId)
# any more results?
if int(search_rs.TotalNumResults) > current_page * page_size:
# if we have disabled any HITs on this page
# then we don't need to go to a new page
# otherwise we do
if not disabled_count:
current_page += 1
else:
# no, we're done
break
else:
print 'Error performing search, code:%s, message:%s' % (
search_rs.Code, search_rs.Message)
break
total_ignored = len(ignored)
print 'Processed: %d HITs, disabled/disposed: %d, ignored: %d' % (
total_ignored + total_disabled, total_disabled, total_ignored)
