def _setup_data(self, opt): counts = { 'partner': { gend_utils.UNKNOWN: 0, gend_utils.FEM: 0, gend_utils.MASC: 0 }, 'self': { gend_utils.UNKNOWN: 0, gend_utils.FEM: 0, gend_utils.MASC: 0 }, } dt = opt['datatype'].split(':')[0] if dt == 'test': warn_once('No test set; switching to valid') dt = 'valid' # build data print('[ Building data ... ]') new_eps = [] orig_teacher = OrigConvai2Teacher(opt) total_exs = orig_teacher.num_examples() num_exs = 0 while num_exs < total_exs: current_episode = [] episode_done = False while not episode_done: # TODO: eventually all teachers should return Messages, so # we should assert this action = Message(orig_teacher.act()) current_episode.append(action) episode_done = action.get('episode_done', False) num_exs += 1 # now we have the entire episode,... do something first_ex = current_episode[0] first_ex_text = [] partner_persona = [] your_persona = [] for line in first_ex['text'].split('\n'): # NOTE: we flip "your" and "partner" here since we are taking the 'text' # field instead of the 'label' if 'partner\'s persona: ' in line: your_persona.append(line.split('partner\'s persona: ')[1]) elif 'your persona: ' in line: partner_persona.append(line.split('your persona: ')[1]) else: first_ex_text.append(line) your, your_prob, partner, partner_prob = self.get_genders( your_persona, partner_persona) for i, ex in enumerate(current_episode): counts['self'][your] += 1 counts['partner'][partner] += 1 if i == 0: text = '\n'.join(first_ex_text) else: text = ex['text'] new_ex = { 'text': text, 'episode_done': True, 'your_persona': '\n'.join(your_persona), 'partner_persona': '\n'.join(partner_persona), 'id': 'ConvAI2 Gender', } if not self.use_probably: new_ex['partner_prob'] = partner_prob new_ex['your_prob'] = your_prob if your is not None and self.labels_to_use != 'partner': # Get the your task labels = [f'SELF:{your}'] your_ex = deepcopy(new_ex) your_ex['labels'] = labels your_ex['class_type'] = 'self' new_eps.append(your_ex) if partner is not None and self.labels_to_use != 'self': # Get the partner task labels = [f'PARTNER:{partner}'] partner_ex = deepcopy(new_ex) partner_ex['labels'] = labels partner_ex['class_type'] = 'partner' new_eps.append(partner_ex) if self.labels_to_use == 'all' and self.add_unknown_classes: # load about data all_about_data = gend_utils.get_inferred_about_data( self.opt['task'], self.opt) sample_rate = self.opt['unknown_temp'] if sample_rate < 1.0: to_samp = int(sample_rate * len(all_about_data)) sampled = random.sample(all_about_data, to_samp) new_eps += sampled else: new_eps += all_about_data if self.is_train: random.shuffle(new_eps) self.data = new_eps print(f'Missing cnt: {self.missing_cnt} / {len(self.data) * 2}') for x in ['self', 'partner']: print(f'Totals for {x}:') subtot = sum(counts[x].values()) for k, v in counts[x].items(): print(f'\t{k}: {v} ({v / subtot})')
def act(self): reply = Message() reply['id'] = self.getID() reply['text'] = self.searchInput return reply
def setup_data(self, datafile): for _ in range(3): for j in range(1, 4): yield Message({'text': str(j), 'label': str(j * 2)}), j == 1
def test_base_task(self): with testing_utils.tempdir() as tmpdir: # Paths expected_states_folder = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'expected_states') expected_chat_data_path = os.path.join( expected_states_folder, 'final_chat_data__image_chat.json') expected_state_path = os.path.join(expected_states_folder, 'state__image_chat.json') parlai_data_folder = os.path.join(tmpdir, 'parlai_data') chat_data_folder = os.path.join(tmpdir, 'final_chat_data') sample_image_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'test_image_stack', 'sample_image.jpg', ) image_context_path = os.path.join(tmpdir, 'image_contexts') stack_folder = os.path.join(tmpdir, 'image_stack') # Save image context: instead of downloading images, just save a pickle # file with all of the image act image_context = [{ 'image_act': Message({ 'text': 'Obsessive', 'image_id': '2923e28b6f588aff2d469ab2cccfac57', 'episode_done': False, 'label_candidates': [ "I must learn that bird's name!", "My, aren't you a pretty bird?", ], 'image': Image.open(sample_image_path), 'id': 'image_chat', 'eval_labels': ["I must learn that bird's name!"], }) }] with open(image_context_path, 'wb') as f: pickle.dump(image_context, f) # Download the Transresnet Multimodal model download_transresnet(parlai_data_folder) # Set up the config and database num_convos = 1 args = ModelImageChatBlueprintArgs() overrides = [ f'+mephisto.blueprint.{key}={val}' for key, val in args.__dict__.items() if key in [ 'evals_per_image_model_combo', 'max_resp_time', 'override_opt', 'random_seed', 'world_file', ] ] + [ 'mephisto.blueprint.annotations_config_path=""', f'mephisto.blueprint.chat_data_folder={chat_data_folder}', f'+mephisto.blueprint.image_context_path={image_context_path}', '+mephisto.blueprint.left_pane_text_path=${task_dir}/task_config/left_pane_text.html', '+mephisto.blueprint.max_concurrent_responses=1', '+mephisto.blueprint.model_opt_path=${task_dir}/task_config/image_model_opts.yaml', f'+mephisto.blueprint.num_conversations={num_convos:d}', f'+mephisto.blueprint.stack_folder={stack_folder}', '+mephisto.blueprint.task_description_file=${task_dir}/task_config/task_description.html', 'mephisto.blueprint.task_model_parallel=False', ] # TODO: remove all of these params once Hydra 1.1 is released with # support for recursive defaults self._set_up_config( blueprint_type=IMAGE_CHAT_BLUEPRINT_TYPE, task_directory=TASK_DIRECTORY, overrides=overrides, ) # Set up the operator and server shared_state = SharedModelImageChatTaskState( world_module=world_module) self._set_up_server(shared_state=shared_state) # Check that the agent states are as they should be self._get_channel_info( ).job.task_runner.task_run.get_blueprint().use_onboarding = ( False) # Don't require onboarding for this test agent with open(expected_state_path) as f: expected_state = json.load(f) self._test_agent_states( num_agents=1, agent_display_ids=AGENT_DISPLAY_IDS, agent_messages=AGENT_MESSAGES, form_messages=FORM_MESSAGES, form_task_data=FORM_TASK_DATA, expected_states=(expected_state, ), ) # Check that the contents of the chat data file are as expected with open(expected_chat_data_path) as f: expected_chat_data = json.load(f) results_path = list( glob.glob( os.path.join(chat_data_folder, '*_*_*_sandbox.json')))[0] with open(results_path) as f: actual_chat_data = json.load(f) self._check_final_chat_data(actual_value=actual_chat_data, expected_value=expected_chat_data)
def update_parley(self, k, Dk_tr, Dk_val): teacher, student = self.agents observations_tr = [] observations_val = [] for i in range(len(Dk_tr)): # todo later? batchify all the turns from all the dialogs in the meta-batch together. for a in teacher.get_episode(Dk_tr[i]): observations_tr.extend([student.observe(a)]) student.self_observe(observations_tr[-1]) for a in teacher.get_episode(Dk_val[i]): observations_val.extend([student.observe(a)]) student.self_observe(observations_val[-1]) # logging.info('%s obs in Dk_tr' % len(observations_tr)) batch_reply = [ Message({ 'id': self.getID(), 'episode_done': False }) for _ in observations_tr ] # batch_reply_val = [ # Message({'id': self.getID(), 'episode_done': False}) for _ in observations_val # ] # check if there are any labels available, if so we will train on them self.is_training = True #any('labels' in obs for obs in observations) # print('Train obs to batchify: ', observations_tr); sys.exit() # create a batch from the vectors batch = student.batchify(observations_tr) batch_val = student.batchify(observations_val) # logging.info('Tr batch size: %s' % batch.batchsize) # logging.info('Val batch size: %s' % batch_val.batchsize) student._init_cuda_buffer(self.opt['batchsize'], student.label_truncate or 256) student.model.train() student.zero_grad() init_model_params = copy.deepcopy(student.model.state_dict()) param_names = list(init_model_params.keys()) # import torch; torch.set_printoptions(precision=6) # print('Init: ', student.model.state_dict()[param_names[3]][0,:10]) student._control_local_metrics( disabled=True) # turn off local metric computation # print(student.__local_metrics_enabled); sys.exit() loss = student.compute_loss(batch) student.backward(loss) student.update_params() # print('grad: ', list(student.model.parameters())[0].grad) # print('Updated: ', student.model.state_dict()[param_names[3]][0,:10]) loss = student.compute_loss(batch_val) student.backward(loss) # print('second eval:', student.model.state_dict()[param_names[3]][0,:10]) student.model.load_state_dict(init_model_params) # print('back to init: ', student.model.state_dict()[param_names[3]][0,:10]) student.update_params() # print('final updated: ', student.model.state_dict()[param_names[3]][0,:10]) # else: # with torch.no_grad(): # # save memory and compute by disabling autograd. # # use `with torch.enable_grad()` to gain back gradients. # output = self.eval_step(batch) # # if output is not None: # # local metrics are automatically matched up # self.match_batch(batch_reply, batch.valid_indices, output) # # # broadcast the metrics back # for k, values in self._local_metrics.items(): # if len(values) != len(batch.valid_indices): # raise IndexError( # f"Batchsize mismatch on metric {k} (got {len(values)}, " # f"expected {len(batch.valid_indices)}" # ) # for i, value in zip(batch.valid_indices, values): # if 'metrics' not in batch_reply[i]: # batch_reply[i]['metrics'] = {} # batch_reply[i]['metrics'][k] = value # # # Make sure we push all the metrics to main thread in hogwild/workers # self.global_metrics.flush() return batch_reply
def parley(self): self.turn_idx += 1 print(self.world_tag + ' is at turn {}...'.format(self.turn_idx)) """If at first turn, we need to give each agent their persona""" if self.turn_idx == 1: for idx, agent in enumerate(self.agents): persona_text = '' for s in self.personas[idx]: persona_text += ('<b><span style="color:blue">' '{}\n</span></b>'.format(s.strip())) control_msg = self.get_control_msg() control_msg['persona_text'] = persona_text control_msg['text'] = self.get_instruction(tag='start', agent_id=agent.id) # TODO: check that get instruction actually exists? agent.observe(validate(control_msg)) if idx == 0: time.sleep(3) """If we get to the min turns, inform turker that they can end if they want. """ if self.turn_idx == self.n_turn + 1: for idx, agent in enumerate(self.agents): control_msg = self.get_control_msg() control_msg['text'] = self.get_instruction( idx, tag='exceed_min_turns') control_msg['exceed_min_turns'] = True agent.observe(validate(control_msg)) """Otherwise, we proceed accordingly.""" # Other agent first if self.other_first and self.turn_idx == 1: if self.model_agent is not None: # Model must observe its persona persona_act = { 'text': '\n'.join([self.model_persona_text, '__SILENCE__']), 'episode_done': False, } self.model_agent.observe(persona_act) self.bot_seen_persona = True model_act = copy.deepcopy(self.model_agent.act()) model_act.force_set('text', self.format_model_reply(model_act['text'])) model_act.force_set('id', 'PERSON_2') self.dialog.append((1, model_act.get('text'))) _random_delay() self.eval_agent.observe(_strip_tensors(model_act)) else: act = self.get_human_agent_act(self.other_agent) timeout = self.check_timeout(act) if timeout: # eval agent early disconnect control_msg = self.get_control_msg() control_msg['text'] = UNEXPECTED_DISCONNECTION_MSG self.eval_agent.observe(validate(control_msg)) return else: self.dialog.append((1, act.get('text'))) act = copy.deepcopy(act) act.force_set('text', self.format_model_reply(act['text'])) self.eval_agent.observe(act) # Eval agent turn act = Message(self.get_human_agent_act(self.eval_agent)) timeout = self.check_timeout(act) if timeout: if self.model_agent is None: control_msg = self.get_control_msg() control_msg['text'] = UNEXPECTED_DISCONNECTION_MSG self.other_agent.observe(validate(control_msg)) return if act['episode_done']: if self.turn_idx >= self.n_turn: if not self.other_first: self.dialog_list = [ '\n'.join([self.dialog[i][1], self.dialog[i + 1][1]]) for i in range(0, len(self.dialog), 2) ] else: self.dialog_list = [' \n' + self.dialog[0][1]] + [ '\n'.join([self.dialog[i][1], self.dialog[i + 1][1]]) for i in range(1, len(self.dialog) - 1, 2) ] self.parallel_eval_mode() self.chat_done = True for ag in self.agents: control_msg = self.get_control_msg() control_msg['text'] = CHAT_ENDED_MSG ag.observe(validate(control_msg)) return self.dialog.append((0, act['text'])) if not self.bot_seen_persona and self.model_agent is not None: # Add persona for model to observe act.force_set('text', '\n'.join([self.model_persona_text, act['text']])) self.bot_seen_persona = True if self.model_agent is not None: self.model_agent.observe(act) else: act = copy.deepcopy(act) act.force_set('text', self.format_model_reply(act['text'])) self.other_agent.observe(act) # Model_agent turn if not self.other_first or self.turn_idx < self.n_turn: if self.model_agent is not None: _random_delay() act = _strip_tensors(copy.deepcopy(self.model_agent.act())) act.force_set('text', self.format_model_reply(act['text'])) act.force_set('id', 'PERSON_2') # NOTE: your model may or may not need to observe itself here # If it does, call model_observes_itself or some other specialized # function else: act = self.get_human_agent_act(self.other_agent) timeout = self.check_timeout(act) if timeout: # eval agent early disconnect control_msg = self.get_control_msg() control_msg['text'] = UNEXPECTED_DISCONNECTION_MSG self.eval_agent.observe(validate(control_msg)) return self.dialog.append((1, act.get('text'))) act = copy.deepcopy(act) act.force_set('text', self.format_model_reply(act['text'])) self.eval_agent.observe(act)
def _build_rpa_episodes(self, ep: List[Message]) -> List[Message]: """ Construct new episodes from old, LIGHT ones. enumerate over all possible start and label positions :param ep: episode to explode and build into new eps :return episodes: return a list of episodes after enumerating over possible labels. """ episodes = [] context, characters, utterances = self._explode_episode( ep, self.exclude_from_context, self.use_speech_prefix) candidates = (self.candidates if self.inline_candidate_type == 'all' else list(characters.values())) # determine initial start and end indices num_utts = self.num_utterances if num_utts < 0: num_utts = len(utterances) - 1 start_idx, end_idx = (0, num_utts - 1) # Enumerate over all possible start, end positions while end_idx < len(utterances) - 1: # Step 0: (maybe) annotate the prior utterances of dialogue prev_utts = [ maybe_annotate( *utt[:-1], self.annotate_speaker, self.speaker_separator, self.speaker_annotation_position, ) for utt in utterances[start_idx:end_idx] ] if self.include_light_context: prev_utts = [context] + prev_utts # Step 1: enumerate over each successive utterance for speaker, label, listener in utterances[end_idx:]: # Step 2: determine the label control / task type if self.classifier_label_type == 'character': speaker_label = (speaker if self.speaker_label_type == 'speaker' else listener) label_control = (WHO_AM_I if self.speaker_label_type == 'speaker' else WHO_ARE_YOU) else: speaker_label = (SELF if speaker == characters['_self_name'] else PARTNER) label_control = WHO_IS_THIS # Step 3: Determine what label candidates to use if (self.num_train_inline_candidates > 0 and DatatypeHelper.is_training(self.datatype) and self.inline_candidate_type == 'all'): label_cands = [speaker, listener] while speaker in label_cands and listener in label_cands: label_cands = random.sample( candidates, self.num_train_inline_candidates - 2) label_cands += [speaker, listener] random.shuffle(label_cands) else: label_cands = candidates # Step 4: Build the Message if self.left_to_right: label_words = label.split(' ') for i in range(1, len(label_words) + 1): if self.delimit_label_control: text = self.delimiter.join( prev_utts + [label_control, ' '.join(label_words[:i])]) else: text = self.delimiter.join( prev_utts[:-1] + [f"{prev_utts[-1]} {label_control}"] + [' '.join(label_words[:i])]) message = Message({ 'text': text, 'labels': [speaker_label], 'label_candidates': label_cands, 'episode_done': True, }) episodes.append([message]) else: if self.delimit_label_control: text = self.delimiter.join(prev_utts + [label_control, label]) else: text = self.delimiter.join( prev_utts[:-1] + [f"{prev_utts[-1]} {label_control}"] + [label]) message = Message({ 'text': text, 'labels': [speaker_label], 'label_candidates': label_cands, 'episode_done': True, }) episodes.append([message]) if start_idx == end_idx: # edge case where num_utterances == 1 break else: start_idx += 1 end_idx += 1 return episodes
def act(self): return Message({ 'id': self.getID(), 'text': self.fixed_response, 'episode_done': False })
def get(self, episode_idx, entry_idx=0): d = self.data[episode_idx] episode_done = entry_idx == (self.len_episode(episode_idx) - 1) wizard_first = 'Wizard' in d['dialog'][0]['speaker'] idx = entry_idx * 2 if wizard_first else (entry_idx * 2) + 1 # first, get knowledge apprentice_ret_passages = wizard_ret_passages = {} if not wizard_first or idx != 0: apprentice_entry = d['dialog'][idx - 1] apprentice_ret_passages = apprentice_entry['retrieved_passages'] if idx - 2 >= 0: wizard_prev_entry = d['dialog'][idx - 2] wizard_ret_passages = wizard_prev_entry['retrieved_passages'] chosen_topic = d.get('chosen_topic', '') chosen_topic_passages = d['chosen_topic_passage'] chosen_topic = d.get('chosen_topic', '') knowledge_dict = {chosen_topic: chosen_topic_passages} for ret_passes in [apprentice_ret_passages, wizard_ret_passages]: for passage in ret_passes: for k, v in passage.items(): if k not in knowledge_dict.keys(): knowledge_dict[k] = v # then, get text if idx == 0: # first message - only have the chosen topic text = chosen_topic elif idx == 1: # first response - only have the first message text = ( f"{chosen_topic}{self.chosen_topic_delimiter}{apprentice_entry['text']}" ) else: text = '' if self.label_type == 'chosen_sent': # if chosen_sent, add wizard response to dialog history text += '{}\n'.format(wizard_prev_entry['text']) text += apprentice_entry['text'] # next, get label wizard_entry = d['dialog'][idx] if self.label_type == 'response': labels = [wizard_entry['text']] else: title, sentence = _get_chosen_title_and_sent( wizard_entry, knowledge_dict) if self.knowledge_separator and title != TOKEN_NOCHOSEN: labels = ['{} {} {}'.format(title, TOKEN_KNOWLEDGE, sentence)] else: labels = ['{} {}'.format(title, sentence)] # finally, get label_candidates label_cands = ['{} {}'.format(TOKEN_NOCHOSEN, TOKEN_NOCHOSEN)] knowledge_str = '' for title, passage in knowledge_dict.items(): for p in passage: if self.knowledge_separator: cand = '{} {} {}'.format(title, TOKEN_KNOWLEDGE, p) else: cand = '{} {}'.format(title, p) knowledge_str += cand + '\n' label_cands.append(cand) if self.label_type == 'response': if 'train' in self.datatype: label_cands = [] else: label_cands = wizard_entry.get('candidate_responses', []) action = Message({ 'id': 'WizardDialogKnowledgeTeacher', 'text': text, 'labels': labels, 'chosen_topic': chosen_topic, 'episode_done': episode_done, 'label_candidates': label_cands, }) if self.include_knowledge: action['knowledge'] = knowledge_str if self.include_checked_sentence: title, sentence = _get_chosen_title_and_sent( wizard_entry, knowledge_dict) action['title'] = title action['checked_sentence'] = sentence return action
def test_custom_eval(self): """ Test whether custom evaluation works. """ with testing_utils.capture_output(): parser = setup_args() opt = parser.parse_args( [ '--task', 'wizard_of_wikipedia', '--datatype', 'valid', '--label-type', 'chosen_sent', ] ) teacher = create_task_agent_from_taskname(opt)[0] title = 'Gardening' cands = list('four') text = "Gardening\nI like Gardening, even when I've only been doing it for a short time." response = 'I live on a farm, we garden all year long, it is very relaxing.' checked_sent = ( 'Gardening is considered by many people to be a relaxing activity.' ) checked_sent_label = f'{title}{TOKEN_KNOWLEDGE}{checked_sent}' retrieval_metric_keys = ['passage_r@1', 'passage_r@5', 'title_r@1', 'title_r@5'] chosen_sent_teacher_action = Message( { 'text': text, 'labels': [checked_sent_label], 'title': [title], 'checked_sentence': [checked_sent], } ) correct_chosen_sent_response = Message( { 'text': checked_sent_label, 'title_candidates': [title] + cands, 'text_candidates': [checked_sent_label] + cands, } ) top5_chosen_sent_response = Message( { 'text': f'hello{TOKEN_KNOWLEDGE}goodbye', 'title_candidates': cands + [title], 'text_candidates': cands + [checked_sent_label], } ) incorrect_chosen_sent_response = Message( { 'text': f'hello{TOKEN_KNOWLEDGE}goodbye', 'title_candidates': cands, 'text_candidates': cands, } ) response_teacher_action = Message( {'text': text, 'labels': [response], 'checked_sentence': checked_sent} ) high_f1_response = Message({'text': checked_sent}) low_f1_response = Message({'text': 'incorrect'}) # 1) Test with correct top sentence teacher.reset_metrics() teacher.custom_evaluation( chosen_sent_teacher_action, [checked_sent_label], correct_chosen_sent_response, ) report = teacher.report() for k in retrieval_metric_keys: assert k in report assert report[k] == AverageMetric(1) # 2) Test with top sentence in top 5 teacher.reset_metrics() teacher.custom_evaluation( chosen_sent_teacher_action, [checked_sent_label], top5_chosen_sent_response ) report = teacher.report() for k in retrieval_metric_keys: assert k in report assert report[k] == AverageMetric(1) if '5' in k else AverageMetric(0) # 3) Test with no top sentences teacher.reset_metrics() teacher.custom_evaluation( chosen_sent_teacher_action, [checked_sent_label], incorrect_chosen_sent_response, ) report = teacher.report() for k in retrieval_metric_keys: assert k in report assert report[k] == AverageMetric(0) # 4) Test knowledge f1 with high f1 teacher.label_type = 'response' teacher.reset_metrics() teacher.custom_evaluation(response_teacher_action, [response], high_f1_response) report = teacher.report() assert 'knowledge_f1' in report assert report['knowledge_f1'] == F1Metric(1) # 5) Test knowledge f1 with low f1 teacher.reset_metrics() teacher.custom_evaluation(response_teacher_action, [response], low_f1_response) report = teacher.report() assert 'knowledge_f1' in report assert report['knowledge_f1'] == F1Metric(0)
def _setup_data(self, opt: Opt) -> List[List[Message]]: """ Flatten and classify the normal task data. Save/load where applicable. :param opt: options dict. """ # create save directory, if it does not already exist self.original_task_name = ':'.join(opt['task'].split(':')[2:]) self.save_dir = self._get_save_path(opt['datapath'], str(datetime.datetime.today())) os.makedirs(self.save_dir, exist_ok=True) fname = f"{opt['datatype'].split(':')[0]}.json" self.save_path = os.path.join(self.save_dir, fname) data = self.load_data(opt, fname) if data is not None: # successfully load data return data # build the original teacher original_task_module = get_original_task_module(opt) teacher_opt = deepcopy(opt) teacher_opt['task'] = self.original_task_name teacher = original_task_module(teacher_opt) total_exs = teacher.num_examples() if self.opt['max_examples'] > 0: total_exs = min(self.opt['max_examples'], total_exs) progress_bar = tqdm(total=total_exs, unit='ex', unit_scale=True, desc='Building flattened data') all_episodes = [] num_exs = 0 while num_exs < total_exs: current_episode = [] episode_done = False while not episode_done: action = Message(teacher.act()) current_episode.append(action) episode_done = action.get('episode_done', False) num_exs += 1 # flatten the episode into 1-example episodes with context flattened_ep = flatten_and_classify( current_episode, opt['flatten_max_context_length'], include_labels=opt['flatten_include_labels'], delimiter=opt['flatten_delimiter'], word_lists=self.word_lists, ) all_episodes += flattened_ep progress_bar.update(len(flattened_ep)) # save data for future use self.save_data(all_episodes) return all_episodes
def _setup_data_with_context(self): yield Message(self._add_context(EXAMPLE1)) yield Message(EXAMPLE2) yield Message(self._add_context(EXAMPLE3)) yield Message(EXAMPLE4)
def _setup_data(self): yield Message(EXAMPLE1) yield Message(EXAMPLE2) yield Message(EXAMPLE3) yield Message(EXAMPLE4)
def get(self, episode_idx, entry_idx=0): ep = self.data[episode_idx] ep['label_candidates'] = self.label_candidates[ep['class_type']] ep['id'] = 'Yelp Gender' ep['episode_done'] = True return Message(ep)
def _to_tuple(self, msg: Message) -> Tuple: # turned into an indexable object keys = ['text', 'labels', 'eval_labels'] return tuple(self._val(msg.get(k)) for k in keys)
def parley(self): print( f'{self.__class__.__name__}:{self.tag}: is at turn {self.task_turn_idx}, with {self.num_turns} pairs of turns needed...' ) if self.task_turn_idx == 0: self._run_initial_turn() self.task_turn_idx += 1 return """Otherwise, we proceed accordingly""" print( f'{self.__class__.__name__}:{self.tag}: About to act with task turn idx: {self.task_turn_idx}' ) acts = [None, None] for idx, agent in enumerate([self.agent, self.bot]): if not self.chat_done: acts[idx] = agent.act(timeout=self.max_resp_time) acts[idx] = Message(Compatibility.maybe_fix_act( acts[idx])).json_safe_payload() print( f'Got act for agent idx {idx}, act was: {acts[idx]} and self.task_turn_idx: {self.task_turn_idx}.' ) if acts[idx].get('task_data', {}).get('final_rating') is not None: self.chat_done = True # agent ends chat after exceeding minimum number of turns # Human has just responded. Any problem data received now will be # regarding the bot's prior utterance turn_idx = -1 # Attach the problem data and final rating to the last utterance, since # the human hasn't said anything since then p = acts[idx]['task_data'].get( 'problem_data_for_prior_message') if p is not None: self.__add_problem_data_to_utterance(p, turn_idx=turn_idx) self.dialog[turn_idx]['final_rating'] = acts[idx]['task_data'][ 'final_rating'] # Save the final chat data date_folder = time.strftime('%Y_%m_%d') time_string = time.strftime('%Y%m%d_%H%M%S') chat_data_subfolder = os.path.join( self.opt['chat_data_folder'], date_folder) os.makedirs(chat_data_subfolder, exist_ok=True) chat_data_path = os.path.join( chat_data_subfolder, f'{time_string}_{np.random.randint(0, 1000)}_{self.task_type}.json', ) final_chat_data = self.get_final_chat_data() self.agent.mephisto_agent.state.messages.append( {'final_chat_data': final_chat_data}) # Append the chat data directly to the agent state's message list in # order to prevent the worker from seeing a new text response in the UI with open(chat_data_path, 'w+') as f_json: data_str = json.dumps(final_chat_data) f_json.write(data_str) print(f'{self.__class__.__name__}:{self.tag}: Data saved at ' f'{chat_data_path} for model: {self.bot.worker_id}.') # Soft-block the worker if there were acceptability violations acceptability_violations = final_chat_data[ 'acceptability_violations'][0] if (acceptability_violations is not None and acceptability_violations != ''): print( f'**NOTE** Acceptability violations detected: {acceptability_violations}' ) # Grant the failed qualification self.agent.mephisto_agent.get_worker().grant_qualification( self.block_qualification, 1) return else: utterance_data = { 'agent_idx': idx, # Get rid of annotations HTML if it's the bot response 'text': acts[idx]['text'].split('<br>')[0], 'id': acts[idx].get('id', 'NULL_ID'), # In case model doesn't set id } self.dialog.append(utterance_data) if idx == 0: # Human has just responded. Any problem data received now will be # regarding the bot's prior utterance p = acts[idx]['task_data'].get( 'problem_data_for_prior_message') if p is not None: turn_idx = -2 # Attach the problem data to the second-to-last utterance, since # the last utterance is what the human just said self.__add_problem_data_to_utterance(p, turn_idx=turn_idx) self._postprocess_acts(acts=acts, agent_idx=idx) for other_agent in [self.agent, self.bot]: if other_agent != agent: other_agent.observe(validate(acts[idx])) print( f'[agent {idx}] self.task_turn_idx: {self.task_turn_idx}, self.dialog is: {self.dialog}' ) self.task_turn_idx += 1
def batch_act(self, observations): self._padding_counter.update( ['val' for o in observations if o.is_padding()]) self._counter.update( [self._to_tuple(o) for o in observations if not o.is_padding()]) return [Message() for o in observations]
def get(self, episode_idx, entry_idx): return Message(self.data[episode_idx])
def setup_data(self, datafile): print('loading: ' + datafile) if self.datatype.startswith('train'): path_to_open = os.path.join(datafile, 'train.txt') elif self.datatype.startswith('valid'): path_to_open = os.path.join(datafile, 'valid.txt') else: path_to_open = os.path.join(datafile, 'test.txt') with PathManager.open(path_to_open) as f: raw_data = [json.loads(line.strip()) for line in f] data = [] label_speaker_id_range = {} predicted_summary_dict = {} if self.use_predicted_summary: is_session_level = not ('utt_' in self.previous_persona_type) predsum_path = get_predicted_summary_path(self.msc_dpath, is_session_level) logger.warning(f"use the predicted summary from {predsum_path}") with PathManager.open(predsum_path) as jsonfile: predicted_summary_dict = json.load(jsonfile) def _get_time_gap(time_num, time_unit, time_token=""): time_gap = str(time_num) + ' ' + time_unit return f'{time_token} {time_gap}' if len(time_token) > 0 else time_gap def _compile_persona_dialog_input( dialog, personas, previous_dialogs, label_speaker_id ): new_dialog = copy.deepcopy(dialog) new_previous_dialogs = copy.deepcopy(previous_dialogs) your_persona = "" partner_persona = "" if label_speaker_id == 'self': your_persona = '\n'.join([f'your persona: {x}' for x in personas[1]]) partner_persona = '\n'.join( [f"partner's persona: {x}" for x in personas[0]] ) elif label_speaker_id == 'their': your_persona = '\n'.join([f'your persona: {x}' for x in personas[0]]) partner_persona = '\n'.join( [f"partner's persona: {x}" for x in personas[1]] ) for prev_dialog in new_previous_dialogs: prev_dialog['dialog'].insert(0, {"text": DUMMY_TEXT}) if len(prev_dialog['dialog']) % 2 == 1 and ( self.history_person_tokens is None ): prev_dialog['dialog'].append({"text": DUMMY_TEXT}) new_dialog.insert(0, {"text": DUMMY_TEXT}) return your_persona, partner_persona, new_dialog, new_previous_dialogs for dialog_dict in raw_data: initial_data_id = dialog_dict['metadata']['initial_data_id'] if self.label_speaker_id == 'both': label_speaker_id_range = ['their', 'self'] else: label_speaker_id_range = [self.label_speaker_id] for label_speaker_id in label_speaker_id_range: if self.use_predicted_summary: personas_to_complie = predicted_summary_dict[ str(self.session_id - 1) ][initial_data_id] elif self.previous_persona_type.startswith('init'): personas_to_complie = dialog_dict['init_personas'] else: personas_to_complie = dialog_dict['personas'] ( your_persona, partner_persona, new_dialog, new_previous_dialogs, ) = _compile_persona_dialog_input( dialog_dict['dialog'], personas_to_complie, dialog_dict['previous_dialogs'], label_speaker_id, ) previous_sessions_msgs = [] if self.previous_persona_type == 'raw_history': for d_id in range(len(new_previous_dialogs)): previous_dialog_msg = [ x['text'] for x in new_previous_dialogs[d_id]['dialog'] ] if self.history_person_tokens: previous_dialog_msg = [ self.history_person_tokens[i % 2] + ' ' + text for i, text in enumerate(previous_dialog_msg) if text != DUMMY_TEXT ] if self.history_time_gaps_token: time_gap_i = _get_time_gap( new_previous_dialogs[d_id]['time_num'], new_previous_dialogs[d_id]['time_unit'], time_token=self.history_time_gaps_token, ) previous_sessions_msgs.append( '\n'.join(previous_dialog_msg + [time_gap_i]) ) else: previous_sessions_msgs.append( '\n'.join(previous_dialog_msg) ) if self.previous_session_delimiter is not None: previous_sessions_msgs = [ val for pair in zip( previous_sessions_msgs, [self.previous_session_delimiter] * len(previous_sessions_msgs), ) for val in pair ] previous_sessions_msgs = '\n'.join(previous_sessions_msgs) episode = [] for i in range(0, len(new_dialog) - 1, 2): text = new_dialog[i]['text'] partner_persona_one_line = partner_persona.replace('\n', '').split( "partner's persona: " ) your_persona_one_line = your_persona.replace('\n', '').split( "your persona: " ) action = { 'id': self.id, 'text': self.normalize_replies(text), 'labels': [self.normalize_replies(new_dialog[i + 1]['text'])], 'session_id': self.session_id, 'initial_data_id': initial_data_id, 'personas': f'{partner_persona}\n{your_persona}', 'personas_one_line': f"partner's persona: {' '.join(partner_persona_one_line)}\nyour persona: {' '.join(your_persona_one_line)}", } episode.append(action) if self.session_openning: break persona_context_str = "" if 'self' in self.previous_persona_type: persona_context_str = your_persona elif 'their' in self.previous_persona_type: persona_context_str = partner_persona elif 'both' in self.previous_persona_type: if self.your_persona_first: persona_context_str = ( (your_persona + '\n') if len(your_persona) > 0 else "" ) + partner_persona else: persona_context_str = ( (partner_persona + '\n') if len(partner_persona) > 0 else "" ) + your_persona elif self.previous_persona_type == 'raw_history': persona_context_str = previous_sessions_msgs if self.include_last_time_gap: time_gap = _get_time_gap( dialog_dict['previous_dialogs'][-1]['time_num'], dialog_dict['previous_dialogs'][-1]['time_unit'], ) persona_context_str = ( (persona_context_str + '\n') if len(persona_context_str) > 0 else "" ) + f'[{time_gap}]' if persona_context_str and len(persona_context_str) > 0: episode[0]['text'] = persona_context_str + '\n' + episode[0]['text'] data.append(episode) for episode in data: start_idx = 0 for i, turn in enumerate(episode): yield Message(turn), i == start_idx
def parley(self): """ Agent 0 goes first. Alternate between the two agents. """ if self.turn_cnt == 0: self.p1, self.p2 = self.get_contexts() acts = self.acts print("acts", acts) agents = self.agents if self.turn_cnt == 0 and self.p1 != '': # add the context on to the first message to agent 0 context_act = Message({ 'id': 'context', 'text': self.p1, 'episode_done': True }) agents[0].observe(validate(context_act)) try: act = deepcopy(agents[0].act()) except StopIteration: self.reset() self.finalize_episode() self.turn_cnt = 0 return acts[0] = act if self.turn_cnt == 0 and self.p2 != '': # add the context on to the first message to agent 1 context_act = Message({ 'id': 'context', 'text': self.p2, 'episode_done': False }) agents[1].observe(validate(context_act)) try: try: cache = dict() # caching to save the user_id def get_user_id(user_id): print("getting user id") if user_id not in cache: agents_clone_human = deepcopy(agents[0].clone()) agents_clone_bot = deepcopy(agents[1].clone()) agents[0] = agents_clone_human agents[1] = agents_clone_bot print("new agent created") cache[user_id] = user_id return cache[user_id] file = open("input.txt", "r").read() text = file.split("_") text = text[1] print("user_id_value", text) userid = get_user_id(text) except: print("None") agents[1].observe(validate(act)) if os.path.isfile("input.txt"): os.remove("input.txt") print("client:", act) acts[1] = agents[1].act() print("agent:", acts[1]) print("text only:", acts[1]['text']) file = open("output.txt", "w").write(str(acts[1]['text'])) agents[0].observe(validate(acts[1])) self.update_counters() self.turn_cnt += 1 if act['episode_done']: self.finalize_episode() self.turn_cnt = 0 except: #agents[1].observe(validate(act)) if os.path.isfile("input.txt"): os.remove("input.txt") print("client:", act) acts[1] = { 'id': 'ImageSeq2seq', 'episode_done': False, 'text': "I didn't get that" } print("agent:", acts[1]) print("text only:", acts[1]['text']) agents[0].observe(validate(acts[1])) self.update_counters() self.turn_cnt += 1 act = { 'id': 'localHuman', 'episode_done': False, 'label_candidates': None, 'text': 'hi' } if act['episode_done']: self.finalize_episode() self.turn_cnt = 0
def _strip_tensors(act): """ Remove all tensor objects from an act to ensure we don't try to serialize them. """ return Message({k: v for k, v in act.items() if not torch.is_tensor(v)})