def main(): """This script should be used to compensate workers that have not recieved proper payment for the completion of tasks due to issues on our end. It's important to make sure you keep a requester reputation up. """ parser = argparse.ArgumentParser(description='Bonus workers directly') parser.add_argument( '--sandbox', dest='sandbox', default=False, action='store_true', help='Test bonus in sandbox', ) parser.add_argument( '--hit-id', dest='use_hit_id', default=False, action='store_true', help='Use HIT id instead of assignment id', ) opt = parser.parse_args() sandbox = opt.sandbox if sandbox: print('About to connect to the SANDBOX server. These bonuses will not ' 'actually be paid out.') else: print('About to connect to the LIVE server. These bonuses will be ' 'deducted from your account balance.') mturk_utils.setup_aws_credentials() client = mturk_utils.get_mturk_client(sandbox) while True: worker_id = input("Enter worker id: ") if len(worker_id) == 0: break bonus_amount = input("Enter bonus amount: ") if opt.use_hit_id: hit_id = input("Enter HIT id: ") listed = client.list_assignments_for_hit(HITId=hit_id) assignment_id = listed['Assignments'][0]['AssignmentId'] else: assignment_id = input("Enter assignment id: ") reason = input("Enter reason: ") input("Press enter to bonus {} to worker {} for reason '{}' on " "assignment {}.".format(bonus_amount, worker_id, reason, assignment_id)) resp = client.send_bonus( WorkerId=worker_id, BonusAmount=str(bonus_amount), AssignmentId=assignment_id, Reason=reason, ) print(resp)
def main(): """This script should be used to compensate workers that have not recieved proper payment for the completion of tasks due to issues on our end. It's important to make sure you keep a requester reputation up. """ parser = argparse.ArgumentParser(description='Bonus workers directly') parser.add_argument('--sandbox', dest='sandbox', default=False, action='store_true', help='Test bonus in sandbox') parser.add_argument('--hit-id', dest='use_hit_id', default=False, action='store_true', help='Use HIT id instead of assignment id') opt = parser.parse_args() sandbox = opt.sandbox if sandbox: print( 'About to connect to the SANDBOX server. These bonuses will not ' 'actually be paid out.' ) else: print( 'About to connect to the LIVE server. These bonuses will be ' 'deducted from your account balance.' ) mturk_utils.setup_aws_credentials() client = mturk_utils.get_mturk_client(sandbox) while True: worker_id = input("Enter worker id: ") if len(worker_id) == 0: break bonus_amount = input("Enter bonus amount: ") if opt.use_hit_id: hit_id = input("Enter HIT id: ") listed = client.list_assignments_for_hit(HITId=hit_id) assignment_id = listed['Assignments'][0]['AssignmentId'] else: assignment_id = input("Enter assignment id: ") reason = input("Enter reason: ") input( "Press enter to bonus {} to worker {} for reason '{}' on " "assignment {}.".format(bonus_amount, worker_id, reason, assignment_id) ) resp = client.send_bonus( WorkerId=worker_id, BonusAmount=str(bonus_amount), AssignmentId=assignment_id, Reason=reason ) print(resp)
def __init__(self, opt, mturk_agents): mturk_utils.setup_aws_credentials() is_sandbox = True self.client = mturk_utils.get_mturk_client(is_sandbox) self.bonus_amount = 1.0 self.mturk_agents = mturk_agents for agent in mturk_agents: self.assignment_id = agent.assignment_id print(agent.assignment_id) if agent.demo_role == 'Drawer': self.drawer = agent elif agent.demo_role == 'Teller': self.teller = agent self.episodeDone = False self.turn_idx = 0 self.dialog = [] self.should_pay_bonus = False self.selectImageForTask() self.unique_task_id = self.drawer.worker_id + self.teller.worker_id + self.teller.assignment_id + self.drawer.assignment_id return
def main(): """This script should be used after some error occurs that leaves HITs live while the ParlAI MTurk server down. This will search through live HITs and list them by task ID, letting you close down HITs that do not link to any server and are thus irrecoverable. """ parser = argparse.ArgumentParser(description='Delete HITs by expiring') parser.add_argument('--sandbox', dest='sandbox', default=False, action='store_true', help='Delete HITs from sandbox') parser.add_argument( '--ignore-assigned', dest='ignore', default=False, action='store_true', help='Ignore HITs that may already be completed or assigned', ) parser.add_argument( '--approve', dest='approve', default=False, action='store_true', help='Approve HITs that have been completed on deletion', ) parser.add_argument( '--verbose', dest='verbose', default=False, action='store_true', help='List actions to individual HITs', ) opt = parser.parse_args() sandbox = opt.sandbox ignore = opt.ignore approve = opt.approve verbose = opt.verbose task_group_ids = [] group_to_hit = {} processed = 0 found = 0 spinner_vals = ['-', '\\', '|', '/'] if sandbox: print( 'About to query the SANDBOX server, these HITs will be active HITs' ' from within the MTurk requester sandbox') else: print('About to query the LIVE server, these HITs will be active HITs ' 'potentially being worked on by real Turkers right now') print('Getting HITs from amazon MTurk server, please wait...\n' 'or use CTRL-C to skip to expiring some of what is found.\n') mturk_utils.setup_aws_credentials() client = mturk_utils.get_mturk_client(sandbox) response = client.list_hits(MaxResults=100) try: while (True): processed += response['NumResults'] for hit in response['HITs']: if ignore: if hit['NumberOfAssignmentsAvailable'] == 0: # Ignore hits with no assignable assignments continue if hit['HITStatus'] != 'Assignable' and \ hit['HITStatus'] != 'Unassignable': # Ignore completed hits continue question = hit['Question'] try: if 'ExternalURL' in question: url = question.split('ExternalURL')[1] group_id = url.split('task_group_id=')[1] group_id = group_id.split('&')[0] group_id = group_id.split('<')[0] if group_id not in task_group_ids: sys.stdout.write( '\rFound group {} ' ' ' '\n'.format(group_id)) group_to_hit[group_id] = {} task_group_ids.append(group_id) group_to_hit[group_id][hit['HITId']] = hit['HITStatus'] found += 1 except IndexError: pass # This wasn't the right HIT sys.stdout.write('\r{} HITs processed, {} active hits' ' found amongst {} tasks. {} '.format( processed, found, len(task_group_ids), spinner_vals[((int)(processed / 100)) % 4])) if 'NextToken' not in response: break response = client.list_hits(NextToken=response['NextToken'], MaxResults=100) except BaseException as e: print(e) pass if not approve: print('\n\nTask group id - Active HITs') for group_id in task_group_ids: print('{} - {}'.format(group_id, len(group_to_hit[group_id]))) else: print('\n\nTask group id - Active HITs - Reviewable') for group_id in task_group_ids: print('{} - {} - {}'.format( group_id, len(group_to_hit[group_id]), len([ s for s in group_to_hit[group_id].values() if s == 'Reviewable' ]), )) print( 'To clear a task, please enter the task group id of the task that you ' 'want to expire the HITs for. To exit, enter nothing') while True: task_group_id = input("Enter task group id: ") if len(task_group_id) == 0: break elif (task_group_id not in task_group_ids): print('Sorry, the id you entered was not found, try again') else: num_hits = input( 'Confirm by entering the number of HITs that will be deleted: ' ) if '{}'.format(len(group_to_hit[task_group_id])) == num_hits: hits_expired = 0 for hit_id, status in group_to_hit[task_group_id].items(): if approve: response = client.list_assignments_for_hit( HITId=hit_id, ) if response['NumResults'] == 0: if verbose: print('No results for hit {} with status {}\n' ''.format(hit_id, status)) else: assignment = response['Assignments'][0] assignment_id = assignment['AssignmentId'] client.approve_assignment( AssignmentId=assignment_id) if verbose: print('Approved assignment {}'.format( assignment_id)) mturk_utils.expire_hit(sandbox, hit_id) if verbose: print('Expired hit {}'.format(hit_id)) hits_expired += 1 sys.stdout.write('\rExpired hits {}'.format(hits_expired)) print('\nAll hits for group {} have been expired.'.format( task_group_id)) else: print( 'You entered {} but there are {} HITs to expire, please ' "try again to confirm you're ending the right task".format( num_hits, len(group_to_hit[task_group_id])))
def setup_server(self, task_directory_path=None): """Prepare the MTurk server for the new HIT we would like to submit""" completion_type = 'start' if self.opt['count_complete']: completion_type = 'finish' print_and_log( '\nYou are going to allow workers from Amazon ' 'Mechanical Turk to be an agent in ParlAI.\nDuring this ' 'process, Internet connection is required, and you should ' 'turn off your computer\'s auto-sleep feature.\n' 'Enough HITs will be created to fulfill {} times the number of ' 'conversations requested, extra HITs will be expired once the ' 'desired conversations {}.'.format(HIT_MULT, completion_type)) key_input = input('Please press Enter to continue... ') print_and_log('') setup_aws_credentials() # See if there's enough money in the account to fund the HITs requested num_assignments = self.required_hits payment_opt = { 'type': 'reward', 'num_total_assignments': num_assignments, 'reward': self.opt['reward'], # in dollars 'unique': self.opt['unique_worker'] } total_cost = calculate_mturk_cost(payment_opt=payment_opt) if not check_mturk_balance(balance_needed=total_cost, is_sandbox=self.opt['is_sandbox']): raise SystemExit('Insufficient funds') if total_cost > 100 or self.opt['reward'] > 1: confirm_string = '$%.2f' % total_cost print_and_log( 'You are going to create {} HITs at {} per assignment, for a ' 'total cost of {} after MTurk fees. Please enter "{}" to ' 'confirm and continue, and anything else to cancel'.format( self.required_hits, '$%.2f' % self.opt['reward'], confirm_string, confirm_string)) check = input('Enter here: ') if (check != confirm_string): raise SystemExit('Cancelling') print_and_log('Setting up MTurk server...') create_hit_config(task_description=self.opt['task_description'], unique_worker=self.opt['unique_worker'], is_sandbox=self.opt['is_sandbox']) # Poplulate files to copy over to the server if not self.task_files_to_copy: self.task_files_to_copy = [] if not task_directory_path: task_directory_path = os.path.join(self.opt['parlai_home'], 'parlai', 'mturk', 'tasks', self.opt['task']) self.task_files_to_copy.append( os.path.join(task_directory_path, 'html', 'cover_page.html')) for mturk_agent_id in self.mturk_agent_ids + ['onboarding']: self.task_files_to_copy.append( os.path.join(task_directory_path, 'html', '{}_index.html'.format(mturk_agent_id))) # Setup the server with a likely-unique app-name task_name = '{}-{}'.format(str(uuid.uuid4())[:8], self.opt['task']) self.server_task_name = \ ''.join(e for e in task_name if e.isalnum() or e == '-') self.server_url = \ setup_server(self.server_task_name, self.task_files_to_copy) print_and_log(self.server_url, False) print_and_log("MTurk server setup done.\n")
def setup_server(self, task_directory_path=None): """Prepare the MTurk server for the new HIT we would like to submit""" fin_word = 'start' if self.opt['count_complete']: fin_word = 'finish' shared_utils.print_and_log( logging.INFO, '\nYou are going to allow workers from Amazon Mechanical Turk to ' 'be an agent in ParlAI.\nDuring this process, Internet connection ' 'is required, and you should turn off your computer\'s auto-sleep ' 'feature.\nEnough HITs will be created to fulfill {} times the ' 'number of conversations requested, extra HITs will be expired ' 'once the desired conversations {}.'.format(HIT_MULT, fin_word), should_print=True ) key_input = input('Please press Enter to continue... ') shared_utils.print_and_log(logging.NOTSET, '', True) mturk_utils.setup_aws_credentials() # See if there's enough money in the account to fund the HITs requested num_assignments = self.required_hits payment_opt = { 'type': 'reward', 'num_total_assignments': num_assignments, 'reward': self.opt['reward'], # in dollars 'unique': self.opt['unique_worker'] } total_cost = mturk_utils.calculate_mturk_cost(payment_opt=payment_opt) if not mturk_utils.check_mturk_balance( balance_needed=total_cost, is_sandbox=self.opt['is_sandbox']): raise SystemExit('Insufficient funds') if ((not self.opt['is_sandbox']) and (total_cost > 100 or self.opt['reward'] > 1)): confirm_string = '$%.2f' % total_cost expected_cost = total_cost / HIT_MULT expected_string = '$%.2f' % expected_cost shared_utils.print_and_log( logging.INFO, 'You are going to create {} HITs at {} per assignment, for a ' 'total cost up to {} after MTurk fees. Please enter "{}" to ' 'confirm and continue, and anything else to cancel.\nNote that' ' of the {}, the target amount to spend is {}.'.format( self.required_hits, '$%.2f' % self.opt['reward'], confirm_string, confirm_string, confirm_string, expected_string ), should_print=True ) check = input('Enter here: ') if (check != confirm_string and ('$' + check) != confirm_string): raise SystemExit('Cancelling') shared_utils.print_and_log(logging.INFO, 'Setting up MTurk server...', should_print=True) mturk_utils.create_hit_config( task_description=self.opt['task_description'], unique_worker=self.opt['unique_worker'], is_sandbox=self.opt['is_sandbox'] ) # Poplulate files to copy over to the server if not self.task_files_to_copy: self.task_files_to_copy = [] if not task_directory_path: task_directory_path = os.path.join( self.opt['parlai_home'], 'parlai', 'mturk', 'tasks', self.opt['task'] ) self.task_files_to_copy.append( os.path.join(task_directory_path, 'html', 'cover_page.html')) for file_name in os.listdir(os.path.join(task_directory_path, 'html')): self.task_files_to_copy.append(os.path.join( task_directory_path, 'html', file_name )) for mturk_agent_id in self.mturk_agent_ids + ['onboarding']: self.task_files_to_copy.append(os.path.join( task_directory_path, 'html', '{}_index.html'.format(mturk_agent_id) )) # Setup the server with a likely-unique app-name task_name = '{}-{}'.format(str(uuid.uuid4())[:8], self.opt['task']) self.server_task_name = \ ''.join(e for e in task_name.lower() if e.isalnum() or e == '-') self.server_url = server_utils.setup_server(self.server_task_name, self.task_files_to_copy) shared_utils.print_and_log(logging.INFO, self.server_url) shared_utils.print_and_log(logging.INFO, "MTurk server setup done.\n", should_print=True)
def setup_server(self, task_directory_path=None): """Prepare the MTurk server for the new HIT we would like to submit""" fin_word = 'start' if self.opt['count_complete']: fin_word = 'finish' shared_utils.print_and_log( logging.INFO, '\nYou are going to allow workers from Amazon Mechanical Turk to ' 'be an agent in ParlAI.\nDuring this process, Internet connection ' 'is required, and you should turn off your computer\'s auto-sleep ' 'feature.\nEnough HITs will be created to fulfill {} times the ' 'number of conversations requested, extra HITs will be expired ' 'once the desired conversations {}.'.format(HIT_MULT, fin_word), should_print=True ) key_input = input('Please press Enter to continue... ') shared_utils.print_and_log(logging.NOTSET, '', True) mturk_utils.setup_aws_credentials() # See if there's enough money in the account to fund the HITs requested num_assignments = self.required_hits payment_opt = { 'type': 'reward', 'num_total_assignments': num_assignments, 'reward': self.opt['reward'], # in dollars 'unique': self.opt['unique_worker'] } total_cost = mturk_utils.calculate_mturk_cost(payment_opt=payment_opt) if not mturk_utils.check_mturk_balance( balance_needed=total_cost, is_sandbox=self.opt['is_sandbox']): raise SystemExit('Insufficient funds') if ((not self.opt['is_sandbox']) and (total_cost > 100 or self.opt['reward'] > 1)): confirm_string = '$%.2f' % total_cost expected_cost = total_cost / HIT_MULT expected_string = '$%.2f' % expected_cost shared_utils.print_and_log( logging.INFO, 'You are going to create {} HITs at {} per assignment, for a ' 'total cost up to {} after MTurk fees. Please enter "{}" to ' 'confirm and continue, and anything else to cancel.\nNote that' ' of the {}, the target amount to spend is {}.'.format( self.required_hits, '$%.2f' % self.opt['reward'], confirm_string, confirm_string, confirm_string, expected_string ), should_print=True ) check = input('Enter here: ') if (check != confirm_string and ('$' + check) != confirm_string): raise SystemExit('Cancelling') shared_utils.print_and_log(logging.INFO, 'Setting up MTurk server...', should_print=True) mturk_utils.create_hit_config( task_description=self.opt['task_description'], unique_worker=self.opt['unique_worker'], is_sandbox=self.opt['is_sandbox'] ) # Poplulate files to copy over to the server if not self.task_files_to_copy: self.task_files_to_copy = [] if not task_directory_path: task_directory_path = os.path.join( self.opt['parlai_home'], 'parlai', 'mturk', 'tasks', self.opt['task'] ) self.task_files_to_copy.append( os.path.join(task_directory_path, 'html', 'cover_page.html')) for mturk_agent_id in self.mturk_agent_ids + ['onboarding']: self.task_files_to_copy.append(os.path.join( task_directory_path, 'html', '{}_index.html'.format(mturk_agent_id) )) # Setup the server with a likely-unique app-name task_name = '{}-{}'.format(str(uuid.uuid4())[:8], self.opt['task']) self.server_task_name = \ ''.join(e for e in task_name if e.isalnum() or e == '-') self.server_url = server_utils.setup_server(self.server_task_name, self.task_files_to_copy) shared_utils.print_and_log(logging.INFO, self.server_url) shared_utils.print_and_log(logging.INFO, "MTurk server setup done.\n", should_print=True)
def main(): """This script should be used after some error occurs that leaves HITs live while the ParlAI MTurk server down. This will search through live HITs and list them by task ID, letting you close down HITs that do not link to any server and are thus irrecoverable. """ parser = argparse.ArgumentParser(description='Delete HITs by expiring') parser.add_argument('--sandbox', dest='sandbox', default=False, action='store_true', help='Delete HITs from sandbox') parser.add_argument( '--ignore-assigned', dest='ignore', default=False, action='store_true', help='Ignore HITs that may already be completed or assigned', ) opt = parser.parse_args() sandbox = opt.sandbox ignore = opt.ignore task_group_ids = [] group_to_hit = {} hits = [] processed = 0 found = 0 spinner_vals = ['-', '\\', '|', '/'] if sandbox: print( 'About to query the SANDBOX server, these HITs will be active HITs' ' from within the MTurk requester sandbox' ) else: print( 'About to query the LIVE server, these HITs will be active HITs ' 'potentially being worked on by real Turkers right now' ) print('Getting HITs from amazon MTurk server, please wait...\n' 'or use CTRL-C to skip to expiring some of what is found.\n') mturk_utils.setup_aws_credentials() client = mturk_utils.get_mturk_client(sandbox) response = client.list_hits(MaxResults=100) try: while (True): processed += response['NumResults'] for hit in response['HITs']: if ignore: if hit['NumberOfAssignmentsAvailable'] == 0: # Ignore hits with no assignable assignments continue if hit['HITStatus'] != 'Assignable' and \ hit['HITStatus'] != 'Unassignable': # Ignore completed hits continue question = hit['Question'] if 'ExternalURL' in question: url = question.split('ExternalURL')[1] group_id = url.split('task_group_id=')[1] group_id = group_id.split('&')[0] group_id = group_id.split('<')[0] if group_id not in task_group_ids: group_to_hit[group_id] = [] task_group_ids.append(group_id) group_to_hit[group_id].append(hit['HITId']) found += 1 sys.stdout.write( '\r{} HITs processed, {} active hits' ' found amongst {} tasks. {} ' .format( processed, found, len(task_group_ids), spinner_vals[((int)(processed / 100)) % 4] ) ) if 'NextToken' not in response: break response = client.list_hits( NextToken=response['NextToken'], MaxResults=100 ) except BaseException: pass print('\n\nTask group id - Active HITs') for group_id in task_group_ids: print('{} - {}'.format(group_id, len(group_to_hit[group_id]))) print( 'To clear a task, please enter the task group id of the task that you ' 'want to expire the HITs for. To exit, enter nothing' ) while True: task_group_id = input("Enter task group id: ") if len(task_group_id) == 0: break elif (task_group_id not in task_group_ids): print('Sorry, the id you entered was not found, try again') else: num_hits = input( 'Confirm by entering the number of HITs that will be deleted: ' ) if '{}'.format(len(group_to_hit[task_group_id])) == num_hits: hits_expired = 0 for hit_id in group_to_hit[task_group_id]: mturk_utils.expire_hit(sandbox, hit_id) hits_expired += 1 sys.stdout.write('\rExpired hits {}'.format(hits_expired)) print('\nAll hits for group {} have been expired.'.format( task_group_id )) else: print( 'You entered {} but there are {} HITs to expire, please ' "try again to confirm you're ending the right task".format( num_hits, len(group_to_hit[task_group_id]) ) )
def main(): """This script should be used after some error occurs that leaves HITs live while the ParlAI MTurk server down. This will search through live HITs and list them by task ID, letting you close down HITs that do not link to any server and are thus irrecoverable. """ parser = argparse.ArgumentParser(description='Delete HITs by expiring') parser.add_argument('--sandbox', dest='sandbox', default=False, action='store_true', help='Delete HITs from sandbox') opt = parser.parse_args() sandbox = opt.sandbox task_group_ids = [] group_to_hit = {} hits = [] processed = 0 found = 0 spinner_vals = ['-','\\','|','/'] if sandbox: print( 'About to query the SANDBOX server, these HITs will be active HITs' ' from within the MTurk requester sandbox' ) else: print( 'About to query the LIVE server, these HITs will be active HITs ' 'potentially being worked on by real Turkers right now' ) print('Getting HITs from amazon MTurk server, please wait...\n') mturk_utils.setup_aws_credentials() client = mturk_utils.get_mturk_client(sandbox) response = client.list_hits(MaxResults=100) while (True): processed += response['NumResults'] for hit in response['HITs']: if hit['NumberOfAssignmentsAvailable'] == 0: # Ignore hits with no assignable assignments continue if hit['HITStatus'] != 'Assignable' and \ hit['HITStatus'] != 'Unassignable': # Ignore completed hits continue question = hit['Question'] if 'ExternalURL' in question: url = question.split('ExternalURL')[1] group_id = url.split('task_group_id=')[1] group_id = group_id.split('&')[0] group_id = group_id.split('<')[0] if group_id not in task_group_ids: group_to_hit[group_id] = [] task_group_ids.append(group_id) group_to_hit[group_id].append(hit['HITId']) found += 1 sys.stdout.write( '\r{} HITs processed, {} active hits found amongst {} tasks. {} ' .format( processed, found, len(task_group_ids), spinner_vals[((int) (processed / 100)) % 4] ) ) if 'NextToken' not in response: break response = client.list_hits( NextToken=response['NextToken'], MaxResults=100 ) print('\n\nTask group id - Active HITs') for group_id in task_group_ids: print('{} - {}'.format(group_id, len(group_to_hit[group_id]))) print( 'To clear a task, please enter the task group id of the task that you ' 'want to expire the HITs for. To exit, enter nothing' ) while True: task_group_id = input("Enter task group id: ") if len(task_group_id) == 0: break elif (task_group_id not in task_group_ids): print('Sorry, the id you entered was not found, try again') else: num_hits = input( 'Confirm by entering the number of HITs that will be deleted: ' ) if '{}'.format(len(group_to_hit[task_group_id])) == num_hits: hits_expired = 0 for hit_id in group_to_hit[task_group_id]: mturk_utils.expire_hit(sandbox, hit_id) hits_expired += 1 sys.stdout.write('\rExpired hits {}'.format(hits_expired)) print('\nAll hits for group {} have been expired.'.format( task_group_id )) else: print( 'You entered {} but there are {} HITs to expire, please ' 'try again to confirm you are ending the right task'.format( num_hits, len(group_to_hit[task_group_id]) ) )
def main(opt): setup_aws_credentials() if opt['no_sandbox']: db_file, all_runs_dir = PATHS['live'] opt['is_sandbox'] = False else: db_file, all_runs_dir = PATHS['sandbox'] assert os.path.exists(db_file), f"DB file {db_file} doesn't exist!" assert os.path.isdir( all_runs_dir), f"run directory {all_runs_dir} doesn't exist!" db = MTurkDataHandler(file_name=db_file) mturk_manager = MTurkManager(opt, []) client = mturk_utils.get_mturk_client(not opt['no_sandbox']) # Get run IDs if opt['run_ids'] is None: run_ids = list(os.listdir(all_runs_dir)) run2worker = defaultdict(lambda: dict()) worker2run = defaultdict(lambda: dict()) for run_id in run_ids: run_dir = os.path.join(all_runs_dir, run_id) hits = os.listdir(run_dir) for hit in hits: # t_*/workers/{WORKER_ID}.json resps = os.listdir(f"{run_dir}/{hit}/workers/") assert len(resps) == 1, "More than one response found!" worker_id = resps[0].split('.')[0] worker_data = json.load( open(os.path.join(run_dir, hit, "workers", resps[0]))) run2worker[run_id][worker_id] = worker_data worker2run[worker_id][run_id] = worker_data else: run_ids = opt['run_ids'].split(',') def get_all_hits(): """ """ all_hits = [] resp = client.list_hits() all_hits.append(resp['HITs']) while 'NextToken' in resp and resp['NextToken']: resp = client.list_hits(NextToken=resp['NextToken']) all_hits += resp['HITs'] time.sleep(0.5) return all_hits def get_run_id_data(run_ids): """ """ print(f"Found following run IDs: ") n_hits = 0 run_data = list() for run_id in run_ids: run_datum = db.get_run_data(run_id) run_data.append((run_id, run_datum)) run_data.sort(key=lambda x: x[1]['launch_time']) for run_id, run_datum in run_data: start_time = datetime.fromtimestamp(run_datum['launch_time']) hits = db.get_pairings_for_run(run_id) n_hits += len(hits) print(f"{run_id} {len(hits)} HITS, started {start_time}") print(f"Total {n_hits} HITS over {len(run_ids)} runs") def approve_run_hits(run_id): """ """ to_approve = [] n_to_approve, n_approved = 0, 0 hits = db.get_pairings_for_run(run_id) data = [] for hit in hits: if hit['conversation_id'] is None: continue try: full_data = db.get_full_conversation_data( run_id, hit['conversation_id'], False) except FileNotFoundError: continue datum = next(iter(full_data['worker_data'].values())) if datum['response']['text'] in BAD_RESPONSES: continue n_to_approve += 1 to_approve.append(datum['assignment_id']) data.append(datum) print(f"To approve: {datum['assignment_id']}") print(f"Run ID {run_id}: to approve {n_to_approve} HITs") conf = input("Confirm? (y/n): ") if conf == "y": didnt_approve = list() for asgn_id in to_approve: try: mturk_manager.approve_work(asgn_id) n_approved += 1 print(f"Approved {asgn_id}") except: didnt_approve.append(asgn_id) print(f"Failed to approve: {asgn_id}") print(f"\tApproved {n_approved} HITs") if didnt_approve: print( f"\tFailed to approve assignments {','.join(didnt_approve)}" ) else: print("\tCancelled approvals") def approve_assignment(asgn_id): """ """ conf = input(f"Confirm approving assignment {asgn_id}? (y/n): ") if conf == "y": try: mturk_manager.approve_work(asgn_id, override_rejection=True) print(f"\tSuccessfully approved!") except: print(f"\tFailed to approve.") else: print("\tCancelled approvals.") def award_from_file(bonus_file, msg): awards = [r.split(',') for r in open(bonus_file, encoding="utf-8")] total_bonus = sum(float(award[-1]) for award in awards) conf = input( f"Confirm awarding total bonus ${total_bonus} to {len(awards)} workers? " ) if conf == "y": n_awarded = 0 amt_awarded = 0.0 didnt_award = list() for award in tqdm(awards): try: worker_id, asgn_id, request_tok, bonus_amt = award except ValueError: ipdb.set_trace() bonus_amt = float(bonus_amt) try: mturk_manager.pay_bonus(worker_id=worker_id, bonus_amount=bonus_amt, assignment_id=asgn_id, reason=msg, unique_request_token=request_tok) n_awarded += 1 amt_awarded += bonus_amt except: didnt_award.append( (worker_id, asgn_id, request_tok, bonus_amt)) #print(f"\tFailed to award bonus to {worker_id}") print(f"Awarded {amt_awarded} to {n_awarded} workers.") if didnt_award: print("Failed on:") for worker_id, asgn_id, request_tok, bonus_amt in didnt_award: #print(f"\tFailed to award bonus {bonus_amt} to {worker_id} for assignment {asgn_id} (tok: {request_tok})") print(f"{worker_id},{asgn_id},{request_tok},{bonus_amt}") else: print("\tCancelled bonus.") return def award_bonus(worker_id, bonus_amt, asgn_id, msg, request_tok): conf = input(f"Confirm awarding ${bonus_amt} to {worker_id}?") if conf == "y": try: mturk_manager.pay_bonus(worker_id=worker_id, bonus_amount=bonus_amt, assignment_id=asgn_id, reason=msg, unique_request_token=request_tok) print(f"\tSuccessfully approved!") except: print(f"\tFailed to approve.") else: print("\tCancelled bonus.") def inspect_assignment(asgn_id): """ """ raise NotImplementedError #asgn_data = db.get_assignment_data(asgn_id) #if asgn_data is None: # print("Assignment ID {asgn_id} not found.") def inspect_hit(hit_id): """ """ raise NotImplementedError #hit_data = db.get_hit_data(hit_id) #if hit_data is None: # print("HIT ID {hit_id} not found.") def inspect_run_worker_pair(run_id, worker_id): """ """ worker_data = run2worker[run_id][worker_id] asgn_id = worker_data['assignment_id'] answers = list() qsts = list() ctx = worker_data['task_data'][0]['conversations'][0]['dialog'][0][ 'text'] for task_datum in worker_data['task_data']: qst_d = task_datum['conversations'][1] qsts.append(qst_d['dialog'][0]['text']) if 'answer' in qst_d and 'answer' is not None: answers.append(qst_d['answer']) else: answers.append(None) try: choices = [ CHOICE2ANS[r['speakerChoice']] for r in worker_data['response']['task_data'] ] reasons = [ r['textReason'] for r in worker_data['response']['task_data'] ] except KeyError as e: print("Key error!") print("task_data not in worker response!") ipdb.set_trace() try: pair = db.get_worker_assignment_pairing(worker_id, asgn_id) hit_time = pair['task_end'] - pair['task_start'] except: ipdb.set_trace() print(f"\nAssignment ID: {worker_data['assignment_id']}") print(f"CONTEXT: {ctx}\n") for qst, ans, choice, reason in zip(qsts, answers, choices, reasons): print(f"QUESTION: {qst}") print(f"ANSWER: {ans}") print(f"CHOICE: {choice}") print(f"REASON: {reason}") print() print(f"HIT time: {hit_time}") resp = input("Accept (y/n) ? ") if resp == "y": #try: # mturk_manager.approve_work(asgn_id, override_rejection=True) # print("\tApproved!") #except: # ipdb.set_trace() mturk_manager.approve_work(asgn_id, override_rejection=True) print("\tApproved!") def inspect_hit_worker_pair(hit_id, worker_id): """ """ resp = client.list_assignments_for_hit(HITId=hit_id) all_asgns = list(resp['Assignments']) while 'NextToken' in resp and resp['NextToken']: resp = client.list_assignments_for_hit(HITId=hit_id, NextToken=resp['NextToken']) if resp['Assignments']: all_asgns.append(resp['Assignments']) time.sleep(0.5) assert len(all_asgns) == 1, ipdb.set_trace() asgn_ids = [a['AssignmentId'] for a in all_asgns] run_ids = list() worker_runs = worker2run[worker_id] for asgn_id in asgn_ids: for run_id, run_d in worker_runs.items(): if run_d['assignment_id'] == asgn_id: run_ids.append(run_id) print(f"Assignment ID: {asgn_ids[0]}") print(f"Submit date: {all_asgns[0]['SubmitTime'].strftime('%m/%d')}") #assert len(run_ids) == 1, ipdb.set_trace() #run_id = run_ids[0] #asgn_id = asgn_ids[0] #worker_data = run2worker[run_id][worker_id] #answers = list() #qsts = list() #ctx = worker_data['task_data'][0]['conversations'][0]['dialog'][0]['text'] #for task_datum in worker_data['task_data']: # qst_d = task_datum['conversations'][1] # qsts.append(qst_d['dialog'][0]['text']) # if 'answer' in qst_d and 'answer' is not None: # answers.append(qst_d['answer']) # else: # answers.append(None) #try: # choices = [CHOICE2ANS[r['speakerChoice']] for r in worker_data['response']['task_data']] # reasons = [r['textReason'] for r in worker_data['response']['task_data']] #except KeyError as e: # print("Key error!") # print("task_data not in worker response!") # ipdb.set_trace() #try: # pair = db.get_worker_assignment_pairing(worker_id, asgn_id) # hit_time = pair['task_end'] - pair['task_start'] #except: # ipdb.set_trace() #print(f"\nAssignment ID: {worker_data['assignment_id']}") #print(f"CONTEXT: {ctx}\n") #for qst, ans, choice, reason in zip(qsts, answers, choices, reasons): # print(f"QUESTION: {qst}") # print(f"ANSWER: {ans}") # print(f"CHOICE: {choice}") # print(f"REASON: {reason}") # print() #print(f"HIT time: {hit_time}") #resp = input("Accept (y/n) ? ") #if resp == "y": # try: # mturk_manager.approve_work(asgn_id, override_rejection=True) # print("\tApproved!") # except: # ipdb.set_trace() # main loop while True: print("Enter 'p' to print runs") cmd = input("Enter command: ") if len(cmd) == 0 or cmd == "exit": break cmd_parts = cmd.split() if cmd_parts[0] == "p": get_run_id_data(run_ids) elif cmd_parts[0] == "inspect": assert len(cmd_parts) == 3, "Insufficient arguments!" inspect_run_worker_pair(cmd_parts[1], cmd_parts[2]) elif cmd_parts[0] in ["get-asgn", 'ga']: assert len( cmd_parts ) == 3, "Insufficient arguments! Please provide worker_id and ..." inspect_hit_worker_pair(cmd_parts[1], cmd_parts[2]) elif cmd_parts[0] == "inspect-asgn": assert len(cmd_parts) > 1, "No assignment ID provided." inspect_assignment(cmd_parts[1]) elif cmd_parts[0] == "inspect-hit": assert len(cmd_parts) > 1, "No HIT ID provided." inspect_hit(cmd_parts[1]) elif cmd_parts[0] == "approve": assert len(cmd_parts) > 1, "No run ID provided." run_id = cmd_parts[1] if run_id in run_ids: approve_run_hits(run_id) else: print(f"Run ID {run_id} not found!") elif cmd_parts[0] == "approve-asgn": assert len(cmd_parts) > 1, "No assignment ID provided." approve_assignment(cmd_parts[1]) elif cmd_parts[0] == "award-from-file": assert len(cmd_parts) > 1, "No file provided." if not os.path.exists(cmd_parts[1]): print(f"File {cmd_parts[1]} not found!") continue award_from_file(cmd_parts[1], BONUS_MSG) elif cmd_parts[0] in ["d", "debug"]: ipdb.set_trace() else: print(f"Command `{cmd}` not understood.")