def save_data(self): convo_finished = True bad_workers = [] if not self.opt['is_sandbox']: if self.opt.get('unique_workers') and self.opt.get( 'unique_qualif_id'): # assign qualification to evaluating agent only qual = self.opt['unique_qualif_id'] mutils.give_worker_qualification(self.eval_agent.worker_id, qual, value=None, is_sandbox=False) if self.dialog == [] or self.gmark_score == -1: convo_finished = False data_path = self.opt['data_path'] if not os.path.exists(data_path): os.makedirs(data_path) if convo_finished: filename = os.path.join( data_path, '{}_{}_{}.pkl'.format( time.strftime("%Y%m%d-%H%M%S"), np.random.randint(0, 1000), self.task_type, ), ) else: filename = os.path.join( data_path, '{}_{}_{}_incomplete.pkl'.format( time.strftime("%Y%m%d-%H%M%S"), np.random.randint(0, 1000), self.task_type, ), ) print(self.world_tag, ': Data successfully saved at {}.'.format(filename)) pickle.dump( { 'chosen_topic': self.chosen_topic, 'topic_choices': self.topic_choices, 'seen': self.seen, 'dialog': self.dialog, 'dialog_list': self.dialog_list, 'other_first': self.other_first, 'total_time': time.time() - self.start_time, 'workers': [ag.worker_id for ag in self.agents], 'hit_id': [ag.hit_id for ag in self.agents], 'assignment_id': [ag.assignment_id for ag in self.agents], 'bad_workers': bad_workers, 'n_turn': self.n_turn, 'gmark_score': self.gmark_score, 'inference': 'nucleus', }, open(filename, 'wb'), )
def shutdown(self): """ Here is where the filtering occurs. If a worker hasn't successfully answered all the questions correctly, they are given the qualification that marks that they should be blocked from this task. """ if self.firstTime and self.correct != len(self.questions): mturk_utils.give_worker_qualification( self.mturk_agent.worker_id, self.qualification_id ) self.mturk_agent.shutdown()
def save_data(self): convo_finished = True bad_workers = [] if not self.opt['is_sandbox']: if (self.opt.get('unique_workers') and self.opt.get('unique_qualif_id')): # assign qualification to evaluating agent only qual = self.opt['unique_qualif_id'] mutils.give_worker_qualification(self.eval_agent.worker_id, qual, value=None, is_sandbox=False) if (self.dialog == [] or self.persona_scores == []): convo_finished = False data_path = self.opt['save_data_path'] if not os.path.exists(data_path): os.makedirs(data_path) if convo_finished: filename = os.path.join( data_path, '{}_{}_{}.pkl'.format(time.strftime("%Y%m%d-%H%M%S"), np.random.randint(0, 1000), self.task_type)) else: filename = os.path.join( data_path, '{}_{}_{}_incomplete.pkl'.format( time.strftime("%Y%m%d-%H%M%S"), np.random.randint(0, 1000), self.task_type)) print(self.world_tag, ': Data successfully saved at {}.'.format(filename)) pickle.dump( { 'dialog': self.dialog, 'mtbeam_log': self.mtbeam_log, 'dialog_list': self.dialog_list, 'other_first': self.other_first, 'total_time': time.time() - self.start_time, 'workers': [ag.worker_id for ag in self.agents], 'hit_id': [ag.hit_id for ag in self.agents], 'assignment_id': [ag.assignment_id for ag in self.agents], 'human_personas': [ag.personas for ag in self.agents], 'model_personas': self.model_personas, 'bad_workers': bad_workers, 'n_turn': self.n_turn, 'engagingness': self.engagingness_scores, 'fluency': self.fluency_scores, 'consistency': self.consistency_scores, 'persona': self.persona_scores }, open(filename, 'wb'))
def load_records(): global worker_record global worker_bans try: with open('records/worker_record.dill', 'rb') as infile: worker_record = dill.load(infile) print("Loaded worker records.") except FileNotFoundError: pass try: with open('records/worker_bans.json', 'r') as infile: worker_bans = json.load(infile) except FileNotFoundError: pass # Exclude players who are blocked by completing the maximum number of games in a previous HIT for worker_id in worker_bans: print("Excluded banned worker {}".format(worker_id)) mturk_utils.give_worker_qualification(worker_id, qualification_id)
def update_worker_record(worker, partner, played_game_id): """ Updates the record for a specific worker - partner pairing with a given game ID :param worker: worker agent :param partner: partner agent :param played_game_id: assigned game ID :return: Nothing. """ global worker_record global worker_bans player_id = worker.worker_id partner_id = partner.worker_id worker_record[player_id]["games"].append(played_game_id) worker_record[player_id]["partners"].append(partner_id) if len(worker_record[player_id]["games"]) == 5: mturk_utils.give_worker_qualification(player_id, qualification_id) worker_bans.append(player_id) worker.block_worker( "Reached the maxiumum of 5 games in the DMG Pilot")