def save_data(self):
convo_finished = True
bad_workers = []
if not self.opt['is_sandbox']:
if self.opt.get('unique_workers') and self.opt.get(
'unique_qualif_id'):
# assign qualification to evaluating agent only
qual = self.opt['unique_qualif_id']
mutils.give_worker_qualification(self.eval_agent.worker_id,
qual,
value=None,
is_sandbox=False)
if self.dialog == [] or self.gmark_score == -1:
convo_finished = False
data_path = self.opt['data_path']
if not os.path.exists(data_path):
os.makedirs(data_path)
if convo_finished:
filename = os.path.join(
data_path,
'{}_{}_{}.pkl'.format(
time.strftime("%Y%m%d-%H%M%S"),
np.random.randint(0, 1000),
self.task_type,
),
)
else:
filename = os.path.join(
data_path,
'{}_{}_{}_incomplete.pkl'.format(
time.strftime("%Y%m%d-%H%M%S"),
np.random.randint(0, 1000),
self.task_type,
),
)
print(self.world_tag,
': Data successfully saved at {}.'.format(filename))
pickle.dump(
{
'chosen_topic': self.chosen_topic,
'topic_choices': self.topic_choices,
'seen': self.seen,
'dialog': self.dialog,
'dialog_list': self.dialog_list,
'other_first': self.other_first,
'total_time': time.time() - self.start_time,
'workers': [ag.worker_id for ag in self.agents],
'hit_id': [ag.hit_id for ag in self.agents],
'assignment_id': [ag.assignment_id for ag in self.agents],
'bad_workers': bad_workers,
'n_turn': self.n_turn,
'gmark_score': self.gmark_score,
'inference': 'nucleus',
},
open(filename, 'wb'),
)
def shutdown(self):
"""
Here is where the filtering occurs. If a worker hasn't successfully
answered all the questions correctly, they are given the qualification
that marks that they should be blocked from this task.
"""
if self.firstTime and self.correct != len(self.questions):
mturk_utils.give_worker_qualification(
self.mturk_agent.worker_id,
self.qualification_id
)
self.mturk_agent.shutdown()
def save_data(self):
convo_finished = True
bad_workers = []
if not self.opt['is_sandbox']:
if (self.opt.get('unique_workers')
and self.opt.get('unique_qualif_id')):
# assign qualification to evaluating agent only
qual = self.opt['unique_qualif_id']
mutils.give_worker_qualification(self.eval_agent.worker_id,
qual,
value=None,
is_sandbox=False)
if (self.dialog == [] or self.persona_scores == []):
convo_finished = False
data_path = self.opt['save_data_path']
if not os.path.exists(data_path):
os.makedirs(data_path)
if convo_finished:
filename = os.path.join(
data_path,
'{}_{}_{}.pkl'.format(time.strftime("%Y%m%d-%H%M%S"),
np.random.randint(0, 1000),
self.task_type))
else:
filename = os.path.join(
data_path, '{}_{}_{}_incomplete.pkl'.format(
time.strftime("%Y%m%d-%H%M%S"), np.random.randint(0, 1000),
self.task_type))
print(self.world_tag,
': Data successfully saved at {}.'.format(filename))
pickle.dump(
{
'dialog': self.dialog,
'mtbeam_log': self.mtbeam_log,
'dialog_list': self.dialog_list,
'other_first': self.other_first,
'total_time': time.time() - self.start_time,
'workers': [ag.worker_id for ag in self.agents],
'hit_id': [ag.hit_id for ag in self.agents],
'assignment_id': [ag.assignment_id for ag in self.agents],
'human_personas': [ag.personas for ag in self.agents],
'model_personas': self.model_personas,
'bad_workers': bad_workers,
'n_turn': self.n_turn,
'engagingness': self.engagingness_scores,
'fluency': self.fluency_scores,
'consistency': self.consistency_scores,
'persona': self.persona_scores
}, open(filename, 'wb'))
def load_records():
global worker_record
global worker_bans
try:
with open('records/worker_record.dill', 'rb') as infile:
worker_record = dill.load(infile)
print("Loaded worker records.")
except FileNotFoundError:
pass
try:
with open('records/worker_bans.json', 'r') as infile:
worker_bans = json.load(infile)
except FileNotFoundError:
pass
# Exclude players who are blocked by completing the maximum number of games in a previous HIT
for worker_id in worker_bans:
print("Excluded banned worker {}".format(worker_id))
mturk_utils.give_worker_qualification(worker_id,
qualification_id)
def update_worker_record(worker, partner, played_game_id):
"""
Updates the record for a specific worker - partner pairing with a given game ID
:param worker: worker agent
:param partner: partner agent
:param played_game_id: assigned game ID
:return: Nothing.
"""
global worker_record
global worker_bans
player_id = worker.worker_id
partner_id = partner.worker_id
worker_record[player_id]["games"].append(played_game_id)
worker_record[player_id]["partners"].append(partner_id)
if len(worker_record[player_id]["games"]) == 5:
mturk_utils.give_worker_qualification(player_id,
qualification_id)
worker_bans.append(player_id)
worker.block_worker(
"Reached the maxiumum of 5 games in the DMG Pilot")
