def delete_hit(hit_id): if SANDBOX: mturk_url = 'mechanicalturk.sandbox.amazonaws.com' preview_url = 'https://workersandbox.mturk.com/mturk/preview?groupId=' else: mturk_url = 'mechanicalturk.amazonaws.com' preview_url = 'https://mturk.com/mturk/preview?groupId=' conn = MTurkConnection(aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY, host=mturk_url) conn.expire_hit(hit_id) # Give the HIT a moment to expire. time.sleep(0.25) conn.dispose_hit(hit_id) print("HIT " + hit_id + " was deleted!")
def delete_hit(hit_id): if SANDBOX: mturk_url = 'mechanicalturk.sandbox.amazonaws.com' preview_url = 'https://workersandbox.mturk.com/mturk/preview?groupId=' else: mturk_url = 'mechanicalturk.amazonaws.com' preview_url = 'https://mturk.com/mturk/preview?groupId=' conn = MTurkConnection(aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_KEY, host=mturk_url) conn.expire_hit(hit_id) # Give the HIT a moment to expire. time.sleep(0.25) conn.dispose_hit(hit_id) print("HIT " + hit_id + " was deleted!")
class amusic(object): engine = None origconf = {'minPitch':12,'maxPitch':84,'minStartTime':0.0,'maxStartTime':200.0, 'minNoteDuration':0.5,'maxNoteDuration':5.0,'minNoteCount':50,'maxNoteCount':400, 'currentPopulation':'','bucket':None,'hitRewardPerAssignment':0.05,'mturkLayoutID':None, 'hitTitle':None,'hitDescription':None,'hitKeywords':None} conf = origconf.copy() s3bucket = None mtc = None class PopulationNotSet( Exception ): pass class SongNotFound( Exception ): pass def __init__(self,username,password,ACCESS_ID,SECRET_KEY,initialize=False): self.ACCESS_ID = ACCESS_ID self.SECRET_KEY = SECRET_KEY self.engine = sqlalchemy.create_engine('mysql+mysqlconnector://%s:%s@localhost' % (username,password)) self.engine.connect() self.mtc = MTurkConnection(self.ACCESS_ID,self.SECRET_KEY,host='mechanicalturk.sandbox.amazonaws.com') try: self.engine.execute('USE amusic;') c = self.engine.execute('SELECT * FROM conf;').fetchone() self.conf['minPitch'] = c[0] self.conf['maxPitch'] = c[1] self.conf['minStartTime'] = c[2] self.conf['maxStartTime'] = c[3] self.conf['minNoteDuration'] = c[4] self.conf['maxNoteDuration'] = c[5] self.conf['minNoteCount'] = c[6] self.conf['maxNoteCount'] = c[7] self.conf['currentPopulation'] = c[8] self.conf['bucket'] = c[9] self.conf['mturkLayoutID'] = c[10] self.conf['hitTitle'] = c[11] self.conf['hitDescription'] = c[12] self.conf['hitKeywords'] = c[13] self.conf['hitRewardPerAssignment'] = c[14] except:pass def inits3(self): c = boto.connect_s3(self.ACCESS_ID,self.SECRET_KEY) self.s3bucket = c.create_bucket(self.conf['bucket']) self.s3bucket.set_acl('public-read') def initialize(self): try: self.engine.execute('DROP DATABASE IF EXISTS amusic;') except: pass self.engine.execute('CREATE DATABASE amusic;') self.engine.execute('USE amusic;') self.engine.execute('CREATE TABLE conf (minPitch INT,\ maxPitch INT,\ minStartTime FLOAT,\ maxStartTime FLOAT,\ minNoteDuration FLOAT,\ maxNoteDuration FLOAT,\ minNoteCount INT,\ maxNoteCount INT,\ currentPopulation VARCHAR(100),\ bucket VARCHAR(100),\ mturkLayoutID VARCHAR(100),\ hitTitle VARCHAR(100),\ hitDescription VARCHAR(100),\ hitKeywords VARCHAR(100),\ hitRewardPerAssignment FLOAT,\ CONSTRAINT fk_1 FOREIGN KEY (`currentPopulation`) REFERENCES population (title),\ PRIMARY KEY(minPitch)\ ) ENGINE = MYISAM;') self.engine.execute('INSERT INTO conf (minPitch,\ maxPitch,\ minStartTime,\ maxStartTime,\ minNoteDuration,\ maxNoteDuration,\ minNoteCount,\ maxNoteCount,\ currentPopulation,\ bucket,\ hitRewardPerAssignment,\ mturkLayoutID,\ hitTitle,\ hitKeywords,\ hitDescription)\ VALUES(%d,%d,%f,%f,%f,%f,%d,%d,"%s","%s",%f,"%s","%s","%s","%s");'% (self.origconf['minPitch'],self.origconf['maxPitch'],self.origconf['minStartTime'], self.origconf['maxStartTime'],self.origconf['minNoteDuration'], self.origconf['maxNoteDuration'],self.origconf['minNoteCount'], self.origconf['maxNoteCount'],self.origconf['currentPopulation'], self.origconf['bucket'], self.origconf['hitRewardPerAssignment'], self.origconf['mturkLayoutID'],self.origconf['hitTitle'], self.origconf['hitKeywords'],self.origconf['hitDescription'])) self.engine.execute('CREATE TABLE population (title VARCHAR(100) NOT NULL,\ PRIMARY KEY(title)\ ) ENGINE = MYISAM;') self.engine.execute('CREATE TABLE song (id INT NOT NULL AUTO_INCREMENT,\ title VARCHAR(100),\ population VARCHAR(100),\ ppq INT,\ CONSTRAINT fk_1 FOREIGN KEY (`population`) REFERENCES population (title),\ PRIMARY KEY(id,title)\ ) ENGINE = MYISAM;') self.engine.execute('CREATE TABLE event (songID INT,\ track INT,\ id INT NOT NULL AUTO_INCREMENT,\ type VARCHAR(5),\ pitch INT,\ value INT,\ startTime FLOAT,\ duration FLOAT,\ velocity INT,\ CONSTRAINT fk_1 FOREIGN KEY (`songID`) REFERENCES song (id),\ PRIMARY KEY(songID,track,id)\ ) ENGINE = MYISAM;') def setPopulation(self,title): self.engine.execute('UPDATE conf SET currentPopulation="%s";' % title) self.conf['currentPopulation'] = title def newPopulation(self,title): return Population(self,title) def getCurrentPopulation(self): if self.conf['currentPopulation'] == '': raise self.PopulationNotSet return Population(self,self.conf['currentPopulation']) def getPopulation(self,title): return Population(self,title,create=False) def listHITs(self): print " ".join(('%s','%s','%s','%s','%s')) % ("HIT ID".ljust(30), "Status".ljust(22), "Amount", "Song1", "Song2") for i in self.mtc.search_hits(response_groups=['Request','Minimal','HITDetail','HITQuestion']): l = re.findall('<input type="hidden" value="([^"]+?)" name="song[12]" />',i.Question) print ' '.join((i.HITId, i.HITStatus,i.HITReviewStatus,i.Amount,l[0].split('/')[-1],l[1].split('/')[-1])) def deleteHITs(self): for i in self.mtc.get_reviewable_hits(): self.mtc.dispose_hit(i.HITId) def approveAssignment(self,aID): self.mtc.approve_assignment(aID) def rejectAssignment(self,aID): self.mtc.reject_assignment(aID) def approveAllAssignments(self): for i in self.mtc.get_reviewable_hits(): for assignment in self.mtc.get_assignments(i.HITId): if assignment.AssignmentStatus=="Submitted": self.mtc.approve_assignment(assignment.AssignmentId) def getResults(self): print " ".join(('%s','%s','%s','%s','%s')) % ("Assignment ID".ljust(30), "Worker ID".ljust(14), "Song1", "Song2", "Answer") for i in self.mtc.get_reviewable_hits(): for assignment in self.mtc.get_assignments(i.HITId): ans = {} for j in assignment.answers[0]: if j.qid!='commit': ans[j.qid] = j.fields[0].split('/')[-1] print assignment.AssignmentId, assignment.WorkerId,ans['song1'], ans['song2'], ans['boxradio']
class MTurkServices(object): ''' MTurk services ''' def __init__(self, aws_access_key_id, aws_secret_access_key, is_sandbox): self.update_credentials(aws_access_key_id, aws_secret_access_key) self.set_sandbox(is_sandbox) self.valid_login = self.verify_aws_login() if not self.valid_login: print 'WARNING *****************************' print 'Sorry, AWS Credentials invalid.\nYou will only be able to '\ 'test experiments locally until you enter\nvalid '\ 'credentials in the AWS Access section of ~/.psiturkconfig\n' def update_credentials(self, aws_access_key_id, aws_secret_access_key): ''' Update credentials ''' self.aws_access_key_id = aws_access_key_id self.aws_secret_access_key = aws_secret_access_key def set_sandbox(self, is_sandbox): ''' Set sandbox ''' self.is_sandbox = is_sandbox def get_reviewable_hits(self): ''' Get reviewable HITs ''' if not self.connect_to_turk(): return False try: hits = self.mtc.get_all_hits() except MTurkRequestError: return False reviewable_hits = [hit for hit in hits if hit.HITStatus == "Reviewable" \ or hit.HITStatus == "Reviewing"] hits_data = [MTurkHIT({ 'hitid': hit.HITId, 'title': hit.Title, 'status': hit.HITStatus, 'max_assignments': hit.MaxAssignments, 'number_assignments_completed': hit.NumberOfAssignmentsCompleted, 'number_assignments_pending': hit.NumberOfAssignmentsPending, 'number_assignments_available': hit.NumberOfAssignmentsAvailable, 'creation_time': hit.CreationTime, 'expiration': hit.Expiration }) for hit in reviewable_hits] return hits_data def get_all_hits(self): ''' Get all HITs ''' if not self.connect_to_turk(): return False try: hits = self.mtc.get_all_hits() except MTurkRequestError: return False hits_data = [MTurkHIT({ 'hitid': hit.HITId, 'title': hit.Title, 'status': hit.HITStatus, 'max_assignments': hit.MaxAssignments, 'number_assignments_completed': hit.NumberOfAssignmentsCompleted, 'number_assignments_pending': hit.NumberOfAssignmentsPending, 'number_assignments_available': hit.NumberOfAssignmentsAvailable, 'creation_time': hit.CreationTime, 'expiration': hit.Expiration, }) for hit in hits] return hits_data def get_active_hits(self): ''' Get active HITs ''' if not self.connect_to_turk(): return False # hits = self.mtc.search_hits() try: hits = self.mtc.get_all_hits() except MTurkRequestError: return False active_hits = [hit for hit in hits if not hit.expired] hits_data = [MTurkHIT({ 'hitid': hit.HITId, 'title': hit.Title, 'status': hit.HITStatus, 'max_assignments': hit.MaxAssignments, 'number_assignments_completed': hit.NumberOfAssignmentsCompleted, 'number_assignments_pending': hit.NumberOfAssignmentsPending, 'number_assignments_available': hit.NumberOfAssignmentsAvailable, 'creation_time': hit.CreationTime, 'expiration': hit.Expiration, }) for hit in active_hits] return hits_data def get_workers(self, assignment_status=None): ''' Get workers ''' if not self.connect_to_turk(): return False try: hits = self.mtc.get_all_hits() hit_ids = [hit.HITId for hit in hits] workers_nested = [] page_size=100 for hit_id in hit_ids: current_page_number=1 hit_assignments = self.mtc.get_assignments( hit_id, status=assignment_status, sort_by='SubmitTime', page_size=page_size, page_number=current_page_number ) totalNumResults = int(hit_assignments.TotalNumResults) total_pages = (totalNumResults // page_size) + (totalNumResults % page_size > 0) #do integer division then round up if necessary while current_page_number < total_pages: current_page_number += 1 hit_assignments += self.mtc.get_assignments( hit_id, status=assignment_status, sort_by='SubmitTime', page_size=page_size, page_number=current_page_number ) workers_nested.append(hit_assignments) workers = [val for subl in workers_nested for val in subl] # Flatten nested lists except MTurkRequestError: return False worker_data = [{ 'hitId': worker.HITId, 'assignmentId': worker.AssignmentId, 'workerId': worker.WorkerId, 'submit_time': worker.SubmitTime, 'accept_time': worker.AcceptTime, 'status': worker.AssignmentStatus } for worker in workers] return worker_data def bonus_worker(self, assignment_id, amount, reason=""): ''' Bonus worker ''' if not self.connect_to_turk(): return False try: bonus = MTurkConnection.get_price_as_price(amount) assignment = self.mtc.get_assignment(assignment_id)[0] worker_id = assignment.WorkerId self.mtc.grant_bonus(worker_id, assignment_id, bonus, reason) return True except MTurkRequestError as exception: print exception return False def approve_worker(self, assignment_id): ''' Approve worker ''' if not self.connect_to_turk(): return False try: self.mtc.approve_assignment(assignment_id, feedback=None) return True except MTurkRequestError: return False def reject_worker(self, assignment_id): ''' Reject worker ''' if not self.connect_to_turk(): return False try: self.mtc.reject_assignment(assignment_id, feedback=None) return True except MTurkRequestError: return False def unreject_worker(self, assignment_id): ''' Unreject worker ''' if not self.connect_to_turk(): return False try: self.mtc.approve_rejected_assignment(assignment_id) return True except MTurkRequestError: return False def verify_aws_login(self): ''' Verify AWS login ''' if ((self.aws_access_key_id == 'YourAccessKeyId') or (self.aws_secret_access_key == 'YourSecretAccessKey')): return False else: host = 'mechanicalturk.amazonaws.com' mturkparams = dict( aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, host=host) self.mtc = MTurkConnection(**mturkparams) try: self.mtc.get_account_balance() except MTurkRequestError as exception: print exception.error_message return False else: return True def connect_to_turk(self): ''' Connect to turk ''' if not self.valid_login: print 'Sorry, unable to connect to Amazon Mechanical Turk. AWS '\ 'credentials invalid.' return False if self.is_sandbox: host = 'mechanicalturk.sandbox.amazonaws.com' else: host = 'mechanicalturk.amazonaws.com' mturkparams = dict( aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, host=host) self.mtc = MTurkConnection(**mturkparams) return True def configure_hit(self, hit_config): ''' Configure HIT ''' # configure question_url based on the id experiment_portal_url = hit_config['ad_location'] frame_height = 600 mturk_question = ExternalQuestion(experiment_portal_url, frame_height) # Qualification: quals = Qualifications() approve_requirement = hit_config['approve_requirement'] quals.add( PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo", approve_requirement)) if hit_config['us_only']: quals.add(LocaleRequirement("EqualTo", "US")) # Create a HIT type for this HIT. hit_type = self.mtc.register_hit_type( hit_config['title'], hit_config['description'], hit_config['reward'], hit_config['duration'], keywords=hit_config['keywords'], approval_delay=None, qual_req=None)[0] # Check the config file to see if notifications are wanted. config = PsiturkConfig() config.load_config() try: url = config.get('Server Parameters', 'notification_url') all_event_types = [ "AssignmentAccepted", "AssignmentAbandoned", "AssignmentReturned", "AssignmentSubmitted", "HITReviewable", "HITExpired", ] self.mtc.set_rest_notification( hit_type.HITTypeId, url, event_types=all_event_types) except: pass # Specify all the HIT parameters self.param_dict = dict( hit_type=hit_type.HITTypeId, question=mturk_question, lifetime=hit_config['lifetime'], max_assignments=hit_config['max_assignments'], title=hit_config['title'], description=hit_config['description'], keywords=hit_config['keywords'], reward=hit_config['reward'], duration=hit_config['duration'], approval_delay=None, questions=None, qualifications=quals, response_groups=[ 'Minimal', 'HITDetail', 'HITQuestion', 'HITAssignmentSummary' ]) def check_balance(self): ''' Check balance ''' if not self.connect_to_turk(): return '-' return self.mtc.get_account_balance()[0] # TODO (if valid AWS credentials haven't been provided then # connect_to_turk() will fail, not error checking here and elsewhere) def create_hit(self, hit_config): ''' Create HIT ''' try: if not self.connect_to_turk(): return False self.configure_hit(hit_config) myhit = self.mtc.create_hit(**self.param_dict)[0] self.hitid = myhit.HITId except: return False else: return self.hitid # TODO(Jay): Have a wrapper around functions that serializes them. # Default output should not be serialized. def expire_hit(self, hitid): ''' Expire HIT ''' if not self.connect_to_turk(): return False try: self.mtc.expire_hit(hitid) return True except MTurkRequestError: print "Failed to expire HIT. Please check the ID and try again." return False def dispose_hit(self, hitid): ''' Dispose HIT ''' if not self.connect_to_turk(): return False try: self.mtc.dispose_hit(hitid) except Exception, e: print "Failed to dispose of HIT %s. Make sure there are no "\ "assignments remaining to be reviewed." % hitid
mainly information that you do not want to share. """) parser.add_argument('-c', "--configs", nargs='+', help='additional configuration files') args = parser.parse_args() mturk_cfg_fname = as_project_path('resources/private/mturk.cfg') cfg = Config.load_configs([mturk_cfg_fname] + args.configs, log=False) print "Delete all HITs" conn = MTurkConnection(aws_access_key_id = cfg['MTURK']['aws_access_key_id'], aws_secret_access_key = cfg['MTURK']['aws_secret_access_key'], host = cfg['MTURK']['host']) for pnum in range(1, 50): for hit in conn.get_reviewable_hits(page_size=100, page_number=pnum): print "HITId:", hit.HITId hitStatus = defaultdict(int) for ass in conn.get_assignments(hit.HITId, status='Submitted', page_size=10, page_number=1): #print "Dir ass:", dir(ass) hitStatus[ass.AssignmentStatus] += 1 print hitStatus print 'Deleting hit:', hit.HITId conn.dispose_hit(hit.HITId)
class amusic(object): engine = None origconf = {'minPitch':12,'maxPitch':84,'minStartTime':0.0,'maxStartTime':200.0, 'minNoteDuration':0.5,'maxNoteDuration':5.0,'minNoteCount':50,'maxNoteCount':400, 'currentPopulation':'','bucket':None,'hitRewardPerAssignment':0.05,'mturkLayoutID':None, 'hitTitle':None,'hitDescription':None,'hitKeywords':None} conf = origconf.copy() s3bucket = None mtc = None class PopulationNotSet( Exception ): pass class SongNotFound( Exception ): pass def __init__(self,username,password,ACCESS_ID,SECRET_KEY,initialize=False): self.ACCESS_ID = ACCESS_ID self.SECRET_KEY = SECRET_KEY self.engine = sqlalchemy.create_engine('mysql+mysqlconnector://%s:%s@localhost' % (username,password)) self.engine.connect() self.mtc = MTurkConnection(self.ACCESS_ID,self.SECRET_KEY,host='mechanicalturk.sandbox.amazonaws.com') try: self.engine.execute('USE amusic;') c = self.engine.execute('SELECT * FROM conf;').fetchone() self.conf['minPitch'] = c[0] self.conf['maxPitch'] = c[1] self.conf['minStartTime'] = c[2] self.conf['maxStartTime'] = c[3] self.conf['minNoteDuration'] = c[4] self.conf['maxNoteDuration'] = c[5] self.conf['minNoteCount'] = c[6] self.conf['maxNoteCount'] = c[7] self.conf['currentPopulation'] = c[8] self.conf['bucket'] = c[9] self.conf['mturkLayoutID'] = c[10] self.conf['hitTitle'] = c[11] self.conf['hitDescription'] = c[12] self.conf['hitKeywords'] = c[13] self.conf['hitRewardPerAssignment'] = c[14] except:pass def inits3(self): c = boto.connect_s3(self.ACCESS_ID,self.SECRET_KEY) self.s3bucket = c.create_bucket(self.conf['bucket']) self.s3bucket.set_acl('public-read') def initialize(self): try: self.engine.execute('DROP DATABASE IF EXISTS amusic;') except: pass self.engine.execute('CREATE DATABASE amusic;') self.engine.execute('USE amusic;') self.engine.execute('CREATE TABLE conf (minPitch INT,\ maxPitch INT,\ minStartTime FLOAT,\ maxStartTime FLOAT,\ minNoteDuration FLOAT,\ maxNoteDuration FLOAT,\ minNoteCount INT,\ maxNoteCount INT,\ currentPopulation VARCHAR(100),\ bucket VARCHAR(100),\ mturkLayoutID VARCHAR(100),\ hitTitle VARCHAR(100),\ hitDescription VARCHAR(100),\ hitKeywords VARCHAR(100),\ hitRewardPerAssignment FLOAT,\ CONSTRAINT fk_1 FOREIGN KEY (`currentPopulation`) REFERENCES population (title),\ PRIMARY KEY(minPitch)\ ) ENGINE = MYISAM;') self.engine.execute('INSERT INTO conf (minPitch,\ maxPitch,\ minStartTime,\ maxStartTime,\ minNoteDuration,\ maxNoteDuration,\ minNoteCount,\ maxNoteCount,\ currentPopulation,\ bucket,\ hitRewardPerAssignment,\ mturkLayoutID,\ hitTitle,\ hitKeywords,\ hitDescription)\ VALUES(%d,%d,%f,%f,%f,%f,%d,%d,"%s","%s",%f,"%s","%s","%s","%s");'% (self.origconf['minPitch'],self.origconf['maxPitch'],self.origconf['minStartTime'], self.origconf['maxStartTime'],self.origconf['minNoteDuration'], self.origconf['maxNoteDuration'],self.origconf['minNoteCount'], self.origconf['maxNoteCount'],self.origconf['currentPopulation'], self.origconf['bucket'], self.origconf['hitRewardPerAssignment'], self.origconf['mturkLayoutID'],self.origconf['hitTitle'], self.origconf['hitKeywords'],self.origconf['hitDescription'])) self.engine.execute('CREATE TABLE population (title VARCHAR(100) NOT NULL,\ PRIMARY KEY(title)\ ) ENGINE = MYISAM;') self.engine.execute('CREATE TABLE song (id INT NOT NULL AUTO_INCREMENT,\ title VARCHAR(100),\ population VARCHAR(100),\ ppq INT,\ CONSTRAINT fk_1 FOREIGN KEY (`population`) REFERENCES population (title),\ PRIMARY KEY(id,title)\ ) ENGINE = MYISAM;') self.engine.execute('CREATE TABLE event (songID INT,\ track INT,\ id INT NOT NULL AUTO_INCREMENT,\ type VARCHAR(5),\ pitch INT,\ value INT,\ startTime FLOAT,\ duration FLOAT,\ velocity INT,\ CONSTRAINT fk_1 FOREIGN KEY (`songID`) REFERENCES song (id),\ PRIMARY KEY(songID,track,id)\ ) ENGINE = MYISAM;') def setPopulation(self,title): self.engine.execute('UPDATE conf SET currentPopulation="%s";' % title) self.conf['currentPopulation'] = title def newPopulation(self,title): return Population(self,title) def getCurrentPopulation(self): if self.conf['currentPopulation'] == '': raise self.PopulationNotSet return Population(self,self.conf['currentPopulation']) def getPopulation(self,title): return Population(self,title,create=False) def listHITs(self): print " ".join(('%s','%s','%s','%s','%s')) % ("HIT ID".ljust(30), "Status".ljust(22), "Amount", "Song1", "Song2") for i in self.mtc.search_hits(response_groups=['Request','Minimal','HITDetail','HITQuestion']): l = re.findall('<input type="hidden" value="([^"]+?)" name="song[12]" />',i.Question) print ' '.join((i.HITId, i.HITStatus,i.HITReviewStatus,i.Amount,l[0].split('/')[-1],l[1].split('/')[-1])) def deleteHITs(self): for i in self.mtc.get_reviewable_hits(): self.mtc.dispose_hit(i.HITId) def approveAssignment(self,aID): self.mtc.approve_assignment(aID) def rejectAssignment(self,aID): self.mtc.reject_assignment(aID) def approveAllAssignments(self): for i in self.mtc.get_reviewable_hits(): for assignment in self.mtc.get_assignments(i.HITId): if assignment.AssignmentStatus=="Submitted": self.mtc.approve_assignment(assignment.AssignmentId) def getResults(self): print " ".join(('%s','%s','%s','%s','%s')) % ("Assignment ID".ljust(30), "Worker ID".ljust(14), "Song1", "Song2", "Answer") for i in self.mtc.get_reviewable_hits(): for assignment in self.mtc.get_assignments(i.HITId): ans = {} for j in assignment.answers[0]: if j.qid!='commit': ans[j.qid] = j.fields[0].split('/')[-1] print assignment.AssignmentId, assignment.WorkerId,ans['song1'], ans['song2'], ans['boxradio']
help='additional configuration files') args = parser.parse_args() mturk_cfg_fname = as_project_path('resources/private/mturk.cfg') cfg = Config.load_configs([mturk_cfg_fname] + args.configs, log=False) print "Delete approved, rejected, or expired HITs" conn = MTurkConnection( aws_access_key_id=cfg['MTURK']['aws_access_key_id'], aws_secret_access_key=cfg['MTURK']['aws_secret_access_key'], host=cfg['MTURK']['host']) for pnum in range(1, 350): for hit in conn.get_reviewable_hits(page_size=100, page_number=pnum): print "HITId:", hit.HITId hitStatus = defaultdict(int) for ass in conn.get_assignments(hit.HITId, status='Submitted', page_size=10, page_number=1): #print "Dir ass:", dir(ass) hitStatus[ass.AssignmentStatus] += 1 print hitStatus if 'Submitted' not in hitStatus: print 'Deleting hit:', hit.HITId conn.dispose_hit(hit.HITId)
class MTurkServices: def __init__(self, aws_access_key_id, aws_secret_access_key, is_sandbox): self.update_credentials(aws_access_key_id, aws_secret_access_key) self.set_sandbox(is_sandbox) self.validLogin = self.verify_aws_login() if not self.validLogin: print 'WARNING *****************************' print 'Sorry, AWS Credentials invalid.\nYou will only be able to '\ + 'test experiments locally until you enter\nvalid '\ + 'credentials in the AWS Access section of ~/.psiturkconfig\n' def update_credentials(self, aws_access_key_id, aws_secret_access_key): self.aws_access_key_id = aws_access_key_id self.aws_secret_access_key = aws_secret_access_key def set_sandbox(self, is_sandbox): self.is_sandbox = is_sandbox def get_reviewable_hits(self): if not self.connect_to_turk(): return False try: hits = self.mtc.get_all_hits() except MTurkRequestError: return False reviewable_hits = [hit for hit in hits if (hit.HITStatus == "Reviewable" or hit.HITStatus == "Reviewing")] hits_data = [MTurkHIT({'hitid': hit.HITId, 'title': hit.Title, 'status': hit.HITStatus, 'max_assignments': hit.MaxAssignments, 'number_assignments_completed': hit.NumberOfAssignmentsCompleted, 'number_assignments_pending': hit.NumberOfAssignmentsPending, 'number_assignments_available': hit.NumberOfAssignmentsAvailable, 'creation_time': hit.CreationTime, 'expiration': hit.Expiration, }) for hit in reviewable_hits] return(hits_data) def get_all_hits(self): if not self.connect_to_turk(): return False try: hits = self.mtc.get_all_hits() except MTurkRequestError: return False hits_data = [MTurkHIT({'hitid': hit.HITId, 'title': hit.Title, 'status': hit.HITStatus, 'max_assignments': hit.MaxAssignments, 'number_assignments_completed': hit.NumberOfAssignmentsCompleted, 'number_assignments_pending': hit.NumberOfAssignmentsPending, 'number_assignments_available': hit.NumberOfAssignmentsAvailable, 'creation_time': hit.CreationTime, 'expiration': hit.Expiration, }) for hit in hits] return(hits_data) def get_active_hits(self): if not self.connect_to_turk(): return False # hits = self.mtc.search_hits() try: hits = self.mtc.get_all_hits() except MTurkRequestError: return False active_hits = [hit for hit in hits if not(hit.expired)] hits_data = [MTurkHIT({'hitid': hit.HITId, 'title': hit.Title, 'status': hit.HITStatus, 'max_assignments': hit.MaxAssignments, 'number_assignments_completed': hit.NumberOfAssignmentsCompleted, 'number_assignments_pending': hit.NumberOfAssignmentsPending, 'number_assignments_available': hit.NumberOfAssignmentsAvailable, 'creation_time': hit.CreationTime, 'expiration': hit.Expiration, }) for hit in active_hits] return(hits_data) def get_workers(self, assignmentStatus = None): if not self.connect_to_turk(): return False try: hits = self.mtc.search_hits(sort_direction='Descending', page_size=20) hit_ids = [hit.HITId for hit in hits] workers_nested = [self.mtc.get_assignments( hit_id, status=assignmentStatus, sort_by='SubmitTime', page_size=100 ) for hit_id in hit_ids] workers = [val for subl in workers_nested for val in subl] # Flatten nested lists except MTurkRequestError: return(False) worker_data = [{'hitId': worker.HITId, 'assignmentId': worker.AssignmentId, 'workerId': worker.WorkerId, 'submit_time': worker.SubmitTime, 'accept_time': worker.AcceptTime, 'status': worker.AssignmentStatus } for worker in workers] return(worker_data) def bonus_worker(self, assignment_id, amount, reason=""): if not self.connect_to_turk(): return False try: bonus = MTurkConnection.get_price_as_price(amount) assignment = self.mtc.get_assignment(assignment_id)[0] workerId = assignment.WorkerId self.mtc.grant_bonus(workerId, assignment_id, bonus, reason) return True except MTurkRequestError as e: print e return False def approve_worker(self, assignment_id): if not self.connect_to_turk(): return(False) try: self.mtc.approve_assignment(assignment_id, feedback=None) return True except MTurkRequestError: return(False) def reject_worker(self, assignment_id): if not self.connect_to_turk(): return False try: self.mtc.reject_assignment(assignment_id, feedback=None) return True except MTurkRequestError: return(False) def unreject_worker(self, assignment_id): if not self.connect_to_turk(): return False try: self.mtc.approve_rejected_assignment(assignment_id) return True except MTurkRequestError: return False def verify_aws_login(self): if (self.aws_access_key_id == 'YourAccessKeyId') or (self.aws_secret_access_key == 'YourSecretAccessKey'): return False else: host = 'mechanicalturk.amazonaws.com' mturkparams = dict( aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, host=host) self.mtc = MTurkConnection(**mturkparams) try: self.mtc.get_account_balance() except MTurkRequestError as e: print(e.error_message) return False else: return True def connect_to_turk(self): if not self.validLogin: print 'Sorry, unable to connect to Amazon Mechanical Turk. AWS credentials invalid.' return False if self.is_sandbox: host = 'mechanicalturk.sandbox.amazonaws.com' else: host = 'mechanicalturk.amazonaws.com' mturkparams = dict( aws_access_key_id = self.aws_access_key_id, aws_secret_access_key = self.aws_secret_access_key, host=host) self.mtc = MTurkConnection(**mturkparams) return True def configure_hit(self, hit_config): # configure question_url based on the id experimentPortalURL = hit_config['ad_location'] frameheight = 600 mturkQuestion = ExternalQuestion(experimentPortalURL, frameheight) # Qualification: quals = Qualifications() approve_requirement = hit_config['approve_requirement'] quals.add( PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo", approve_requirement)) if hit_config['us_only']: quals.add(LocaleRequirement("EqualTo", "US")) # Specify all the HIT parameters self.paramdict = dict( hit_type = None, question = mturkQuestion, lifetime = hit_config['lifetime'], max_assignments = hit_config['max_assignments'], title = hit_config['title'], description = hit_config['description'], keywords = hit_config['keywords'], reward = hit_config['reward'], duration = hit_config['duration'], approval_delay = None, questions = None, qualifications = quals ) def check_balance(self): if not self.connect_to_turk(): return('-') return(self.mtc.get_account_balance()[0]) # TODO (if valid AWS credentials haven't been provided then connect_to_turk() will # fail, not error checking here and elsewhere) def create_hit(self, hit_config): try: if not self.connect_to_turk(): return False self.configure_hit(hit_config) myhit = self.mtc.create_hit(**self.paramdict)[0] self.hitid = myhit.HITId except: return False else: return self.hitid # TODO(Jay): Have a wrapper around functions that serializes them. # Default output should not be serialized. def expire_hit(self, hitid): if not self.connect_to_turk(): return False try: self.mtc.expire_hit(hitid) return True except MTurkRequestError: print "Failed to expire HIT. Please check the ID and try again." return False def dispose_hit(self, hitid): if not self.connect_to_turk(): return False try: self.mtc.dispose_hit(hitid) except Exception, e: print 'Failed to dispose of HIT %s. Make sure there are no assignments remaining to be reviewed' % hitid
class MTurkServices(object): ''' MTurk services ''' def __init__(self, aws_access_key_id, aws_secret_access_key, is_sandbox): self.update_credentials(aws_access_key_id, aws_secret_access_key) self.set_sandbox(is_sandbox) self.valid_login = self.verify_aws_login() if not self.valid_login: print 'WARNING *****************************' print 'Sorry, AWS Credentials invalid.\nYou will only be able to '\ 'test experiments locally until you enter\nvalid '\ 'credentials in the AWS Access section of ~/.psiturkconfig\n' def update_credentials(self, aws_access_key_id, aws_secret_access_key): ''' Update credentials ''' self.aws_access_key_id = aws_access_key_id self.aws_secret_access_key = aws_secret_access_key def set_sandbox(self, is_sandbox): ''' Set sandbox ''' self.is_sandbox = is_sandbox def get_reviewable_hits(self): ''' Get reviewable HITs ''' if not self.connect_to_turk(): return False try: hits = self.mtc.get_all_hits() except MTurkRequestError: return False reviewable_hits = [hit for hit in hits if hit.HITStatus == "Reviewable" \ or hit.HITStatus == "Reviewing"] hits_data = [MTurkHIT({ 'hitid': hit.HITId, 'title': hit.Title, 'status': hit.HITStatus, 'max_assignments': hit.MaxAssignments, 'number_assignments_completed': hit.NumberOfAssignmentsCompleted, 'number_assignments_pending': hit.NumberOfAssignmentsPending, 'number_assignments_available': hit.NumberOfAssignmentsAvailable, 'creation_time': hit.CreationTime, 'expiration': hit.Expiration }) for hit in reviewable_hits] return hits_data def get_all_hits(self): ''' Get all HITs ''' if not self.connect_to_turk(): return False try: hits = self.mtc.get_all_hits() except MTurkRequestError: return False hits_data = [MTurkHIT({ 'hitid': hit.HITId, 'title': hit.Title, 'status': hit.HITStatus, 'max_assignments': hit.MaxAssignments, 'number_assignments_completed': hit.NumberOfAssignmentsCompleted, 'number_assignments_pending': hit.NumberOfAssignmentsPending, 'number_assignments_available': hit.NumberOfAssignmentsAvailable, 'creation_time': hit.CreationTime, 'expiration': hit.Expiration, }) for hit in hits] return hits_data def get_active_hits(self): ''' Get active HITs ''' if not self.connect_to_turk(): return False # hits = self.mtc.search_hits() try: hits = self.mtc.get_all_hits() except MTurkRequestError: return False active_hits = [hit for hit in hits if not hit.expired] hits_data = [MTurkHIT({ 'hitid': hit.HITId, 'title': hit.Title, 'status': hit.HITStatus, 'max_assignments': hit.MaxAssignments, 'number_assignments_completed': hit.NumberOfAssignmentsCompleted, 'number_assignments_pending': hit.NumberOfAssignmentsPending, 'number_assignments_available': hit.NumberOfAssignmentsAvailable, 'creation_time': hit.CreationTime, 'expiration': hit.Expiration, }) for hit in active_hits] return hits_data def get_workers(self, assignment_status=None, chosen_hit=None): ''' Get workers ''' if not self.connect_to_turk(): return False try: if chosen_hit: hit_ids = [chosen_hit] else: hits = self.mtc.get_all_hits() hit_ids = [hit.HITId for hit in hits] workers_nested = [] page_size=100 for hit_id in hit_ids: current_page_number=1 hit_assignments = self.mtc.get_assignments( hit_id, status=assignment_status, sort_by='SubmitTime', page_size=page_size, page_number=current_page_number ) totalNumResults = int(hit_assignments.TotalNumResults) total_pages = (totalNumResults // page_size) + (totalNumResults % page_size > 0) #do integer division then round up if necessary while current_page_number < total_pages: current_page_number += 1 hit_assignments += self.mtc.get_assignments( hit_id, status=assignment_status, sort_by='SubmitTime', page_size=page_size, page_number=current_page_number ) workers_nested.append(hit_assignments) workers = [val for subl in workers_nested for val in subl] # Flatten nested lists except MTurkRequestError: return False worker_data = [{ 'hitId': worker.HITId, 'assignmentId': worker.AssignmentId, 'workerId': worker.WorkerId, 'submit_time': worker.SubmitTime, 'accept_time': worker.AcceptTime, 'status': worker.AssignmentStatus } for worker in workers] return worker_data def get_worker(self, assignment_id): if not self.connect_to_turk(): return False try: worker = self.mtc.get_assignment(assignment_id)[0] except MTurkRequestError as e: return False worker_data = [{ 'hitId': worker.HITId, 'assignmentId': worker.AssignmentId, 'workerId': worker.WorkerId, 'submit_time': worker.SubmitTime, 'accept_time': worker.AcceptTime, 'status': worker.AssignmentStatus }] return worker_data def bonus_worker(self, assignment_id, amount, reason=""): ''' Bonus worker ''' if not self.connect_to_turk(): return False try: bonus = MTurkConnection.get_price_as_price(amount) assignment = self.mtc.get_assignment(assignment_id)[0] worker_id = assignment.WorkerId self.mtc.grant_bonus(worker_id, assignment_id, bonus, reason) return True except MTurkRequestError as exception: print exception return False def approve_worker(self, assignment_id): ''' Approve worker ''' if not self.connect_to_turk(): return False try: self.mtc.approve_assignment(assignment_id, feedback=None) return True except MTurkRequestError as e: return False def reject_worker(self, assignment_id): ''' Reject worker ''' if not self.connect_to_turk(): return False try: self.mtc.reject_assignment(assignment_id, feedback=None) return True except MTurkRequestError: return False def unreject_worker(self, assignment_id): ''' Unreject worker ''' if not self.connect_to_turk(): return False try: self.mtc.approve_rejected_assignment(assignment_id) return True except MTurkRequestError: return False def verify_aws_login(self): ''' Verify AWS login ''' if ((self.aws_access_key_id == 'YourAccessKeyId') or (self.aws_secret_access_key == 'YourSecretAccessKey')): return False else: host = 'mechanicalturk.amazonaws.com' mturkparams = dict( aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, host=host) self.mtc = MTurkConnection(**mturkparams) try: self.mtc.get_account_balance() except MTurkRequestError as exception: print exception.error_message return False else: return True def connect_to_turk(self): ''' Connect to turk ''' if not self.valid_login: print 'Sorry, unable to connect to Amazon Mechanical Turk. AWS '\ 'credentials invalid.' return False if self.is_sandbox: host = 'mechanicalturk.sandbox.amazonaws.com' else: host = 'mechanicalturk.amazonaws.com' mturkparams = dict( aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, host=host) self.mtc = MTurkConnection(**mturkparams) return True def configure_hit(self, hit_config): ''' Configure HIT ''' # configure question_url based on the id experiment_portal_url = hit_config['ad_location'] frame_height = 600 mturk_question = ExternalQuestion(experiment_portal_url, frame_height) # Qualification: quals = Qualifications() approve_requirement = hit_config['approve_requirement'] quals.add( PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo", approve_requirement)) number_hits_approved = hit_config['number_hits_approved'] quals.add( NumberHitsApprovedRequirement("GreaterThanOrEqualTo", number_hits_approved)) require_master_workers = hit_config['require_master_workers'] if require_master_workers: quals.add(MasterRequirement(sandbox=self.is_sandbox)) if hit_config['us_only']: quals.add(LocaleRequirement("EqualTo", "US")) # Create a HIT type for this HIT. hit_type = self.mtc.register_hit_type( hit_config['title'], hit_config['description'], hit_config['reward'], hit_config['duration'], keywords=hit_config['keywords'], approval_delay=None, qual_req=quals)[0] # Check the config file to see if notifications are wanted. config = PsiturkConfig() config.load_config() try: url = config.get('Server Parameters', 'notification_url') all_event_types = [ "AssignmentAccepted", "AssignmentAbandoned", "AssignmentReturned", "AssignmentSubmitted", "HITReviewable", "HITExpired", ] self.mtc.set_rest_notification( hit_type.HITTypeId, url, event_types=all_event_types) except: pass # Specify all the HIT parameters self.param_dict = dict( hit_type=hit_type.HITTypeId, question=mturk_question, lifetime=hit_config['lifetime'], max_assignments=hit_config['max_assignments'], questions=None, response_groups=[ 'Minimal', 'HITDetail', 'HITQuestion', 'HITAssignmentSummary' ]) def check_balance(self): ''' Check balance ''' if not self.connect_to_turk(): return '-' return self.mtc.get_account_balance()[0] # TODO (if valid AWS credentials haven't been provided then # connect_to_turk() will fail, not error checking here and elsewhere) def create_hit(self, hit_config): ''' Create HIT ''' try: if not self.connect_to_turk(): return False self.configure_hit(hit_config) myhit = self.mtc.create_hit(**self.param_dict)[0] self.hitid = myhit.HITId except MTurkRequestError as e: print e return False else: return self.hitid # TODO(Jay): Have a wrapper around functions that serializes them. # Default output should not be serialized. def expire_hit(self, hitid): ''' Expire HIT ''' if not self.connect_to_turk(): return False try: self.mtc.expire_hit(hitid) return True except MTurkRequestError: print "Failed to expire HIT. Please check the ID and try again." return False def dispose_hit(self, hitid): ''' Dispose HIT ''' if not self.connect_to_turk(): return False try: self.mtc.dispose_hit(hitid) except Exception, e: print "Failed to dispose of HIT %s. Make sure there are no "\ "assignments remaining to be reviewed." % hitid
class MTurkServices: def __init__(self, aws_access_key_id, aws_secret_access_key, is_sandbox): self.update_credentials(aws_access_key_id, aws_secret_access_key) self.set_sandbox(is_sandbox) self.validLogin = self.verify_aws_login() if not self.validLogin: print 'WARNING *****************************' print 'Sorry, AWS Credentials invalid.\nYou will only be able to '\ + 'test experiments locally until you enter\nvalid '\ + 'credentials in the AWS Access section of ~/.psiturkconfig\n' def update_credentials(self, aws_access_key_id, aws_secret_access_key): self.aws_access_key_id = aws_access_key_id self.aws_secret_access_key = aws_secret_access_key def set_sandbox(self, is_sandbox): self.is_sandbox = is_sandbox def get_reviewable_hits(self): if not self.connect_to_turk(): return False try: hits = self.mtc.get_all_hits() except MTurkRequestError: return False reviewable_hits = [ hit for hit in hits if (hit.HITStatus == "Reviewable" or hit.HITStatus == "Reviewing") ] hits_data = [ MTurkHIT({ 'hitid': hit.HITId, 'title': hit.Title, 'status': hit.HITStatus, 'max_assignments': hit.MaxAssignments, 'number_assignments_completed': hit.NumberOfAssignmentsCompleted, 'number_assignments_pending': hit.NumberOfAssignmentsPending, 'number_assignments_available': hit.NumberOfAssignmentsAvailable, 'creation_time': hit.CreationTime, 'expiration': hit.Expiration, }) for hit in reviewable_hits ] return (hits_data) def get_all_hits(self): if not self.connect_to_turk(): return False try: hits = self.mtc.get_all_hits() except MTurkRequestError: return False hits_data = [ MTurkHIT({ 'hitid': hit.HITId, 'title': hit.Title, 'status': hit.HITStatus, 'max_assignments': hit.MaxAssignments, 'number_assignments_completed': hit.NumberOfAssignmentsCompleted, 'number_assignments_pending': hit.NumberOfAssignmentsPending, 'number_assignments_available': hit.NumberOfAssignmentsAvailable, 'creation_time': hit.CreationTime, 'expiration': hit.Expiration, }) for hit in hits ] return (hits_data) def get_active_hits(self): if not self.connect_to_turk(): return False # hits = self.mtc.search_hits() try: hits = self.mtc.get_all_hits() except MTurkRequestError: return False active_hits = [hit for hit in hits if not (hit.expired)] hits_data = [ MTurkHIT({ 'hitid': hit.HITId, 'title': hit.Title, 'status': hit.HITStatus, 'max_assignments': hit.MaxAssignments, 'number_assignments_completed': hit.NumberOfAssignmentsCompleted, 'number_assignments_pending': hit.NumberOfAssignmentsPending, 'number_assignments_available': hit.NumberOfAssignmentsAvailable, 'creation_time': hit.CreationTime, 'expiration': hit.Expiration, }) for hit in active_hits ] return (hits_data) def get_workers(self, assignmentStatus=None): if not self.connect_to_turk(): return False try: hits = self.mtc.search_hits(sort_direction='Descending', page_size=20) hit_ids = [hit.HITId for hit in hits] workers_nested = [ self.mtc.get_assignments(hit_id, status=assignmentStatus, sort_by='SubmitTime', page_size=100) for hit_id in hit_ids ] workers = [val for subl in workers_nested for val in subl] # Flatten nested lists except MTurkRequestError: return (False) worker_data = [{ 'hitId': worker.HITId, 'assignmentId': worker.AssignmentId, 'workerId': worker.WorkerId, 'submit_time': worker.SubmitTime, 'accept_time': worker.AcceptTime, 'status': worker.AssignmentStatus } for worker in workers] return (worker_data) def bonus_worker(self, assignment_id, amount, reason=""): if not self.connect_to_turk(): return False try: bonus = MTurkConnection.get_price_as_price(amount) assignment = self.mtc.get_assignment(assignment_id)[0] workerId = assignment.WorkerId self.mtc.grant_bonus(workerId, assignment_id, bonus, reason) return True except MTurkRequestError as e: print e return False def approve_worker(self, assignment_id): if not self.connect_to_turk(): return (False) try: self.mtc.approve_assignment(assignment_id, feedback=None) return True except MTurkRequestError: return (False) def reject_worker(self, assignment_id): if not self.connect_to_turk(): return False try: self.mtc.reject_assignment(assignment_id, feedback=None) return True except MTurkRequestError: return (False) def unreject_worker(self, assignment_id): if not self.connect_to_turk(): return False try: self.mtc.approve_rejected_assignment(assignment_id) return True except MTurkRequestError: return False def verify_aws_login(self): if (self.aws_access_key_id == 'YourAccessKeyId') or (self.aws_secret_access_key == 'YourSecretAccessKey'): return False else: host = 'mechanicalturk.amazonaws.com' mturkparams = dict( aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, host=host) self.mtc = MTurkConnection(**mturkparams) try: self.mtc.get_account_balance() except MTurkRequestError as e: print(e.error_message) return False else: return True def connect_to_turk(self): if not self.validLogin: print 'Sorry, unable to connect to Amazon Mechanical Turk. AWS credentials invalid.' return False if self.is_sandbox: host = 'mechanicalturk.sandbox.amazonaws.com' else: host = 'mechanicalturk.amazonaws.com' mturkparams = dict(aws_access_key_id=self.aws_access_key_id, aws_secret_access_key=self.aws_secret_access_key, host=host) self.mtc = MTurkConnection(**mturkparams) return True def configure_hit(self, hit_config): # configure question_url based on the id experimentPortalURL = hit_config['ad_location'] frameheight = 600 mturkQuestion = ExternalQuestion(experimentPortalURL, frameheight) # Qualification: quals = Qualifications() approve_requirement = hit_config['approve_requirement'] quals.add( PercentAssignmentsApprovedRequirement("GreaterThanOrEqualTo", approve_requirement)) if hit_config['us_only']: quals.add(LocaleRequirement("EqualTo", "US")) # Specify all the HIT parameters self.paramdict = dict(hit_type=None, question=mturkQuestion, lifetime=hit_config['lifetime'], max_assignments=hit_config['max_assignments'], title=hit_config['title'], description=hit_config['description'], keywords=hit_config['keywords'], reward=hit_config['reward'], duration=hit_config['duration'], approval_delay=None, questions=None, qualifications=quals) def check_balance(self): if not self.connect_to_turk(): return ('-') return (self.mtc.get_account_balance()[0]) # TODO (if valid AWS credentials haven't been provided then connect_to_turk() will # fail, not error checking here and elsewhere) def create_hit(self, hit_config): try: if not self.connect_to_turk(): return False self.configure_hit(hit_config) myhit = self.mtc.create_hit(**self.paramdict)[0] self.hitid = myhit.HITId except: return False else: return self.hitid # TODO(Jay): Have a wrapper around functions that serializes them. # Default output should not be serialized. def expire_hit(self, hitid): if not self.connect_to_turk(): return False try: self.mtc.expire_hit(hitid) return True except MTurkRequestError: print "Failed to expire HIT. Please check the ID and try again." return False def dispose_hit(self, hitid): if not self.connect_to_turk(): return False try: self.mtc.dispose_hit(hitid) except Exception, e: print 'Failed to dispose of HIT %s. Make sure there are no assignments remaining to be reviewed' % hitid
class HaCRSTurker: def __init__(self): self.config = HaCRSUtil.get_config('../config.ini') HOST = self.config.get('mturk', 'host') AWS_ACCESS_KEY_ID = self.config.get('mturk', 'access_key_id') AWS_SECRET_ACCESS_KEY = self.config.get('mturk', 'secret_access_key') self.MTconnection = MTurkConnection( aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, host=HOST) self.db = HaCRSDB() def get_balance(self): #print self.MTconnection.get_account_balance() pass def expire_all_hits(self): all_hits = self.MTconnection.get_all_hits() for hit in all_hits: if hit.expired: continue try: self.MTconnection.expire_hit(hit.HITId) #print 'Expired HIT' except Exception as e: #print 'Could not expire: {}'.format(e) pass def delete_all_mturk_hits(self): all_hits = self.MTconnection.get_all_hits() for hit in all_hits: #print 'expire/dispose' self.MTconnection.expire_hit(hit.HITId) self.MTconnection.dispose_hit(hit.HITId) def get_all_mturk_hits(self): all_hits = self.MTconnection.get_all_hits() return all_hits # TODO: HITs available via API, but not via Amazon Web Sandbox def push_tasklet_mturk(self, keywords): sdescription = self.config.get('mturk', 'shortdescr') frame_height = self.config.get('mturk', 'frameheight') #url = "https://cgcturk.hacked.jp/tasklet/{}/".format(tasklet['id']) url = "https://cgcturk.hacked.jp/pick_tasklet/{}/".format(keywords) #keywords = tasklet['keywords'] #amount = tasklet['amount'] if keywords == 'easy': amount = 1.00 elif keywords in ['medium', 'hard', 'very_hard']: amount = 2.00 elif keywords == 'priority': amount = 4.00 else: #print 'Error' sys.exit(1) questionform = ExternalQuestion(url, frame_height) title = 'HELP AN AI!!! We are students building an artificial intelligence to find bugs in programs to keep the internet safe' sdescription = 'We are students building an artificial intelligence system that finds bugs in programs and keeps the internet safe from malware. BUT IT NEEDS YOUR HELP! Play with programs to find functions that it missed, and get $$$!' hit_result = self.MTconnection.create_hit( title='[{}] {}'.format(keywords, title), description=sdescription, keywords=keywords, max_assignments=1, question=questionform, reward=Price(amount=amount), response_groups=('Minimal', 'HITDetail'), # ? ) assert len(hit_result) == 1 mturkid = self.db.create_mturk_resource(hit_result[0].HITId, hit_result[0].HITGroupId) #self.db.add_mturk_tasklet_association(tasklet['id'], mturkid) #self.db.commit() return mturkid, hit_result def push_tasks_mturk(self): frame_height = self.config.get('mturk', 'frameheight') amount = 0.01 tasklets = self.db.get_unassigned_tasklets() sdescription = self.config.get('mturk', 'shortdescr') for tasklet in tasklets: #print 'pushing!' url = "https://cgcturk.hacked.jp/tasklet/{}/".format(tasklet['id']) keywords = ["easy"] questionform = ExternalQuestion(url, frame_height) hit_result = self.MTconnection.create_hit( title=HaCRSUtil.get_tasklet_name(tasklet), description=sdescription, keywords=keywords, max_assignments=1, question=questionform, reward=Price(amount=amount), response_groups=('Minimal', 'HITDetail'), # ? ) assert len(hit_result) == 1 mturkid = self.db.create_mturk_resource(hit_result[0].HITId, hit_result[0].HITGroupId) self.db.add_mturk_tasklet_association(tasklet['id'], mturkid) self.db.commit() def show_seed_tasklets(self): pprint(self.db.get_seed_tasklets()) def get_hit(self, hitid): try: hit = self.MTconnection.get_hit(hitid) except Exception as e: return None if hit != None: return hit[0] def get_assignment_from_hit(self, hitid): try: assignments = self.MTconnection.get_assignments(hitid) return assignments[0] except Exception as e: return None def get_approved_seeding_tasklets(self): for program in json.load( open(self.config.get('general', 'programsjson'))): pid = self.db.lookup_program(program) program = None approved = set() for tasklet in self.db.get_latest_seed_tasklets(): turkinfos = self.db.get_mturk_infos(tasklet['id']) try: #hit = self.MTconnection.get_hit(turkinfos['hitid']) assignments = self.MTconnection.get_assignments( turkinfos['hitid']) if len(assignments) == 0: continue if assignments[0].AssignmentStatus == 'Approved': approved.add(self.db.get_tasklet_program(tasklet['id'])) except Exception as e: #print e pass return list(approved)
def cleanup(): """Remove any boto test related HIT's""" conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') current_page = 1 page_size = 10 total_disabled = 0 ignored = [] while True: # reset the total for this loop disabled_count = 0 # search all the hits in the sandbox search_rs = conn.search_hits(page_size=page_size, page_number=current_page) # success? if search_rs.status: for hit in search_rs: # delete any with Boto in the description print 'hit id:%s Status:%s, desc:%s' %(hit.HITId, hit.HITStatus, hit.Description) if hit.Description.find('Boto') != -1: if hit.HITStatus != 'Reviewable': print 'Disabling hit id:%s %s' %(hit.HITId, hit.Description) disable_rs = conn.disable_hit(hit.HITId) if disable_rs.status: disabled_count += 1 # update the running total total_disabled += 1 else: print 'Error when disabling, code:%s, message:%s' %(disable_rs.Code, disable_rs.Message) else: print 'Disposing hit id:%s %s' %(hit.HITId, hit.Description) dispose_rs = conn.dispose_hit(hit.HITId) if dispose_rs.status: disabled_count += 1 # update the running total total_disabled += 1 else: print 'Error when disposing, code:%s, message:%s' %(dispose_rs.Code, dispose_rs.Message) else: if hit.HITId not in ignored: print 'ignored:%s' %hit.HITId ignored.append(hit.HITId) # any more results? if int(search_rs.TotalNumResults) > current_page*page_size: # if we have disabled any HITs on this page # then we don't need to go to a new page # otherwise we do if not disabled_count: current_page += 1 else: # no, we're done break else: print 'Error performing search, code:%s, message:%s' %(search_rs.Code, search_rs.Message) break total_ignored = len(ignored) print 'Processed: %d HITs, disabled/disposed: %d, ignored: %d' %(total_ignored + total_disabled, total_disabled, total_ignored)
def cleanup(): """Remove any boto test related HIT's""" conn = MTurkConnection(host='mechanicalturk.sandbox.amazonaws.com') current_page = 1 page_size = 10 total_disabled = 0 ignored = [] while True: # reset the total for this loop disabled_count = 0 # search all the hits in the sandbox search_rs = conn.search_hits(page_size=page_size, page_number=current_page) # success? if search_rs.status: for hit in search_rs: # delete any with Boto in the description print 'hit id:%s Status:%s, desc:%s' % ( hit.HITId, hit.HITStatus, hit.Description) if hit.Description.find('Boto') != -1: if hit.HITStatus != 'Reviewable': print 'Disabling hit id:%s %s' % (hit.HITId, hit.Description) disable_rs = conn.disable_hit(hit.HITId) if disable_rs.status: disabled_count += 1 # update the running total total_disabled += 1 else: print 'Error when disabling, code:%s, message:%s' % ( disable_rs.Code, disable_rs.Message) else: print 'Disposing hit id:%s %s' % (hit.HITId, hit.Description) dispose_rs = conn.dispose_hit(hit.HITId) if dispose_rs.status: disabled_count += 1 # update the running total total_disabled += 1 else: print 'Error when disposing, code:%s, message:%s' % ( dispose_rs.Code, dispose_rs.Message) else: if hit.HITId not in ignored: print 'ignored:%s' % hit.HITId ignored.append(hit.HITId) # any more results? if int(search_rs.TotalNumResults) > current_page * page_size: # if we have disabled any HITs on this page # then we don't need to go to a new page # otherwise we do if not disabled_count: current_page += 1 else: # no, we're done break else: print 'Error performing search, code:%s, message:%s' % ( search_rs.Code, search_rs.Message) break total_ignored = len(ignored) print 'Processed: %d HITs, disabled/disposed: %d, ignored: %d' % ( total_ignored + total_disabled, total_disabled, total_ignored)