def interactive_rank(opt, print_parser=None): # Create model and assign it to the specified task human = create_agent(opt) task = create_task_agent_from_taskname(opt)[0] metrics = Metrics(opt) episodes = 0 def print_metrics(): report = metrics.report() report['episodes'] = episodes print(report) # Show some example dialogs: try: while not task.epoch_done(): msg = task.act() print('[{id}]: {text}'.format(id=task.getID(), text=msg.get('text', ''))) cands = list(msg.get('label_candidates', [])) random.shuffle(cands) for i, c in enumerate(cands): print(' [{i}]: {c}'.format(i=i, c=c)) print('[ Please choose a response from the list. ]') choice = None while choice is None: choice = human.act().get('text') try: choice = int(choice) if choice >= 0 and choice < len(cands): choice = cands[choice] else: print('[ Try again: you selected {i} but the ' 'candidates are indexed from 0 to {j}. ]' ''.format(i=choice, j=len(cands) - 1)) choice = None except (TypeError, ValueError): print('[ Try again: you did not enter a valid index. ]') choice = None print('[ You chose ]: {}'.format(choice)) reply = {'text_candidates': [choice]} labels = msg.get('eval_labels', msg.get('labels')) metrics.update(reply, labels) if msg.get('episode_done'): episodes += 1 print_metrics() print('------------------------------') print('[ True reply ]: {}'.format(labels[0])) if msg.get('episode_done'): print('******************************') except KeyboardInterrupt: pass print() print_metrics()
def _run_through(self, task, mutators): pp = ParlaiParser(True, False) opt = pp.parse_kwargs(task=task, mutators=mutators) teacher = create_task_agent_from_taskname(opt)[0] outputs = [] for _ in range(5): outputs.append(teacher.act()) return outputs
def test_iter(self): with testing_utils.tempdir() as tmpdir: teacher = create_task_agent_from_taskname({ 'task': 'integration_tests', 'datatype': 'valid', 'datapath': tmpdir })[0] # twice to assert we reset iterators correctly assert len(list(teacher)) == 100 assert len(list(teacher)) == 100
def get_teacher_act(defaults, teacher_processed=False, agent_to=None): parser = train_setup_args() parser.set_defaults(**defaults) opt = parser.parse_args([]) build_dict(opt) teacher = create_task_agent_from_taskname(opt)[0] agent = create_agent(opt) act = teacher.act() if teacher_processed: return act, agent return agent.observe(act), agent
def __init__(self, opt: Opt, shared=None): if ',' in opt['task']: raise ValueError( 'AbstractWrapperTeacher cannot be used with multiple tasks!') self.id = opt['task'] self.opt = opt if shared: self.task = create_agent_from_shared(shared['task']) else: opt_singletask = copy.deepcopy(opt) opt_singletask['task'] = opt['wrapper_task'] self.task = create_task_agent_from_taskname(opt_singletask)[0]
def __init__(self, opt: Opt, shared=None): if ',' in opt['task']: raise ValueError( 'LabelToTextTeacher cannot be used with multiple tasks!') self.id = opt['task'] self.opt = opt if shared and 'task' in shared: self.task = create_agent_from_shared(shared['task']) else: opt_singletask = copy.deepcopy(opt) opt_singletask['task'] = opt['label_to_text_task'] self.task = create_task_agent_from_taskname(opt_singletask)[0]
def test_pytd_teacher(self): """ Test that the pytorch teacher works with given Pytorch Datasets as well. """ defaults = integration_test_parser_defaults.copy() defaults['datatype'] = 'train:stream' defaults['image_mode'] = 'ascii' # Get processed act from agent parser = display_setup_args() defaults['pytorch_teacher_dataset'] = 'integration_tests' del defaults['pytorch_teacher_task'] parser.set_defaults(**defaults) opt = parser.parse_args([]) teacher = create_task_agent_from_taskname(opt)[0] pytorch_teacher_act = teacher.act() parser = display_setup_args() defaults['task'] = 'integration_tests' del defaults['pytorch_teacher_dataset'] parser.set_defaults(**defaults) opt = parser.parse_args([]) teacher = create_task_agent_from_taskname(opt)[0] regular_teacher_act = teacher.act() keys = set(pytorch_teacher_act.keys()).intersection( set(regular_teacher_act.keys())) self.assertTrue(len(keys) != 0) for key in keys: self.assertTrue( pytorch_teacher_act[key] == regular_teacher_act[key], 'PytorchDataTeacher does not have the same value ' 'as regular teacher for act key: {}. ' 'Values: {}; {}'.format(key, pytorch_teacher_act[key], regular_teacher_act[key]), )
def test_image_loader(self): """ Test that model correctly handles text task. """ opt = ParlaiParser().parse_args([]) opt.update(BASE_IMAGE_ARGS) for image_mode, dim in IMAGE_MODE_TO_DIM.items(): opt["image_mode"] = image_mode teacher = create_task_agent_from_taskname(opt)[0] teacher_act = teacher.get(0) self.assertEquals( teacher_act["image"].size(), dim, f"dim mismatch for image mode {image_mode}", )
def _split_type_teacher( split_type: str, ) -> CMUDocumentGroundedConversationsTeacher: kwargs = { 'task': 'cmu_dog', 'datatype': 'valid', 'cmu_dog_split_type': split_type, 'datapath': data_path, } parser = setup_args() parser.set_defaults(**kwargs) opt = parser.parse_args([]) agents = create_task_agent_from_taskname(opt) assert isinstance(agents, List) task = agents[0] assert isinstance(task, CMUDocumentGroundedConversationsTeacher) return task
def test_not_sticky(self): pp = ParlaiParser(True, False) opt = pp.parse_kwargs( task='integration_tests:multiturn', mutators='flatten', datatype='train:ordered', ) teacher = create_task_agent_from_taskname(opt)[0] first_epoch = [] second_epoch = [] for _ in range(teacher.num_examples()): first_epoch.append(teacher.act()) teacher.reset() for _ in range(teacher.num_examples()): second_epoch.append(teacher.act()) assert all(f == s for f, s in zip(first_epoch, second_epoch))
def setup_title_to_passage(opt): print('[ Setting up Title to Passage Dict ]') saved_dp = os.path.join(os.getcwd() + '/data/', 'title_to_passage.pkl') if os.path.exists(saved_dp): print('[ Loading from saved location, {} ]'.format(saved_dp)) with open(saved_dp, 'rb') as f: title_to_passage = pickle.load(f) return title_to_passage topics_path = '{}/personas_with_wiki_links.txt'.format(os.getcwd()) topics = [] with open(topics_path) as f: text = f.read() personas = text.split('\n\n') for persona in personas: persona = persona.split('\n') for i in range(1, len(persona)): p_i = persona[i] if 'https' in p_i: topic = unquote(p_i[p_i.rfind('/') + 1:]).replace('_', ' ') topics.append(topic) ordered_opt = opt.copy() ordered_opt['datatype'] = 'train:ordered:stream' ordered_opt['batchsize'] = 1 ordered_opt['numthreads'] = 1 ordered_opt['task'] = 'wikipedia:full:key-value' teacher = create_task_agent_from_taskname(ordered_opt)[0] title_to_passage = {} i = 0 length = teacher.num_episodes() pbar = tqdm.tqdm(total=length) while not teacher.epoch_done(): pbar.update(1) i += 1 action = teacher.act() title = action['text'] if title in topics: text = action['labels'][0] title_to_passage[title] = text pbar.close() print('[ Finished Building Title to Passage dict; saving now]') with open(saved_dp, 'wb') as f: pickle.dump(title_to_passage, f) return title_to_passage
def _base_test_loader(self, image_mode_partial: str, no_cuda: bool = False): """ Test for given partial image mode. """ opt = ParlaiParser().parse_args([]) opt.update(BASE_IMAGE_ARGS) opt['no_cuda'] = no_cuda for image_mode, dim in IMAGE_MODE_TO_DIM.items(): if image_mode_partial not in image_mode: continue opt["image_mode"] = image_mode teacher = create_task_agent_from_taskname(opt)[0] teacher_act = teacher.get(0) self.assertEquals( teacher_act["image"].size(), dim, f"dim mismatch for image mode {image_mode}", ) torch.cuda.empty_cache()
def _create_task_agents(opt: Opt): """ Create task agent(s) for the given task name. It does this by calling the create_agent function in agents.py of the given task. If create_agents function does not exist, it just looks for the teacher (agent) class defined by the task name directly. (This saves the task creator bothering to define the create_agents function when it is not needed.) """ my_module = load_task_module(opt['task']) try: # Tries to call the create_agent function in agents.py task_agents = my_module.create_agents(opt) # type: ignore except AttributeError: # Create_agent not found, so try to create the teacher directly. return create_task_agent_from_taskname(opt) if type(task_agents) != list: task_agents = [task_agents] return task_agents
def load_classifier(self, gpu_num): if self.has_classifier: if self.classifier is None: opt_overrides = {} self.classifier_gpu_num = gpu_num opt_overrides['gpu'] = gpu_num opt_overrides['datatype'] = 'test' # opt_overrides['inference'] = 'nucleus' opt_overrides['skip_generation'] = False self.classifier = create_agent_from_model_file( self.classifier_checkpoint, opt_overrides=opt_overrides) teacher_for_classifier_opt = deepcopy(self.classifier.opt) teacher_for_classifier_opt.update({"build_data_or_not": False}) self.teacher_for_classifier = create_task_agent_from_taskname( teacher_for_classifier_opt)[0] logging.info("load classifier from:{}".format( self.classifier_checkpoint)) logging.info("allocate classifier to gpu_{}".format(gpu_num)) else: self.classifier = None
def _create_task_agents(opt: Opt): """ Create task agent(s) for the given task name. It does this by calling the create_agent function in agents.py of the given task. If create_agents function does not exist, it just looks for the teacher (agent) class defined by the task name directly. (This saves the task creator bothering to define the create_agents function when it is not needed.) """ if opt.get('interactive_task', False) or opt.get('selfchat_task', False): # do not need task agents in interactive or self chat settings return [] try: # Tries to call the create_agent function in agents.py my_module = load_task_module(opt['task']) task_agents = my_module.create_agents(opt) # type: ignore except (ModuleNotFoundError, AttributeError): # Create_agent not found, so try to create the teacher directly. return create_task_agent_from_taskname(opt) if type(task_agents) != list: task_agents = [task_agents] return task_agents
def test_shuffle(self): """ Simple test to ensure that dataloader is initialized with correct data sampler. """ dts = ['train', 'valid', 'test'] exts = ['', ':stream', ':ordered', ':stream:ordered'] shuffle_opts = [False, True] task = 'babi:task1k:1' for dt in dts: for ext in exts: datatype = dt + ext for shuffle in shuffle_opts: opt_defaults = { 'pytorch_teacher_task': task, 'datatype': datatype, 'shuffle': shuffle, } parser = display_setup_args() parser.set_defaults(**opt_defaults) opt = parser.parse_args([]) teacher = create_task_agent_from_taskname(opt)[0] if ('ordered' in datatype or ('stream' in datatype and not opt.get('shuffle')) or 'train' not in datatype): self.assertIsInstance( teacher.pytorch_dataloader.sampler, Sequential, 'PytorchDataTeacher failed with args: {}'.format( opt), ) else: self.assertIsInstance( teacher.pytorch_dataloader.sampler, RandomSampler, 'PytorchDataTeacher failed with args: {}'.format( opt), )
def store_contents(opt, task, save_path, context_length=-1, include_labels=True): """ Preprocess and store a corpus of documents in sqlite. Args: task: ParlAI tasks of text (and possibly values) to store. save_path: Path to output sqlite db. num_workers: Number of parallel processes to use when reading docs. """ if os.path.isfile(save_path): raise RuntimeError('%s already exists! Not overwriting.' % save_path) logger.info('Reading into database...') conn = sqlite3.connect(save_path) c = conn.cursor() c.execute('CREATE TABLE documents (id INTEGER PRIMARY KEY, text, value);') if not task: logger.info('No data to initialize table: just creating table.') logger.info('Add more data by passing observations to the agent.') logger.info('Committing...') conn.commit() conn.close() return ordered_opt = opt.copy() dt = opt.get('datatype', '').split(':') ordered_opt['datatype'] = ':'.join([dt[0], 'ordered'] + dt[1:]) ordered_opt['batchsize'] = 1 ordered_opt['numthreads'] = 1 ordered_opt['task'] = task teacher = create_task_agent_from_taskname(ordered_opt)[0] episode_done = False current = [] triples = [] context_length = context_length if context_length >= 0 else None context = deque(maxlen=context_length) with tqdm(total=teacher.num_episodes()) as pbar: while not teacher.epoch_done(): # collect examples in episode while not episode_done: action = teacher.act() current.append(action) episode_done = action['episode_done'] for ex in current: if 'text' in ex: text = ex['text'] context.append(text) if len(context) > 1: text = '\n'.join(context) # add labels to context labels = ex.get('labels', ex.get('eval_labels')) label = None if labels is not None: label = random.choice(labels) if include_labels: context.append(label) # use None for ID to auto-assign doc ids--we don't need to # ever reverse-lookup them triples.append((None, text, label)) c.executemany('INSERT OR IGNORE INTO documents VALUES (?,?,?)', triples) pbar.update() # reset flags and content episode_done = False triples.clear() current.clear() context.clear() logger.info('Read %d examples from %d episodes.' % (teacher.num_examples(), teacher.num_episodes())) logger.info('Committing...') conn.commit() conn.close()
def set_classifier(self, classifier): self.classifier = classifier teacher_for_classifier_opt = deepcopy(self.classifier.opt) teacher_for_classifier_opt.update({"build_data_or_not": False}) self.teacher_for_classifier = create_task_agent_from_taskname( teacher_for_classifier_opt)[0]
def create_agents(opt): if not opt.get('interactive_task', False): return create_task_agent_from_taskname(opt) else: # interactive task has no task agents (they are attached as user agents) return []
def test_custom_eval(self): """ Test whether custom evaluation works. """ with testing_utils.capture_output(): parser = setup_args() opt = parser.parse_args([ '--task', 'wizard_of_wikipedia', '--datatype', 'valid', '--label-type', 'chosen_sent', ]) teacher = create_task_agent_from_taskname(opt)[0] title = 'Gardening' cands = list('four') text = "Gardening\nI like Gardening, even when I've only been doing it for a short time." response = 'I live on a farm, we garden all year long, it is very relaxing.' checked_sent = ( 'Gardening is considered by many people to be a relaxing activity.' ) checked_sent_label = f'{title}{TOKEN_KNOWLEDGE}{checked_sent}' retrieval_metric_keys = [ 'passage_r@1', 'passage_r@5', 'title_r@1', 'title_r@5' ] chosen_sent_teacher_action = Message({ 'text': text, 'labels': [checked_sent_label], 'title': [title], 'checked_sentence': [checked_sent], }) correct_chosen_sent_response = Message({ 'text': checked_sent_label, 'title_candidates': [title] + cands, 'text_candidates': [checked_sent_label] + cands, }) top5_chosen_sent_response = Message({ 'text': f'hello{TOKEN_KNOWLEDGE}goodbye', 'title_candidates': cands + [title], 'text_candidates': cands + [checked_sent_label], }) incorrect_chosen_sent_response = Message({ 'text': f'hello{TOKEN_KNOWLEDGE}goodbye', 'title_candidates': cands, 'text_candidates': cands, }) response_teacher_action = Message({ 'text': text, 'labels': [response], 'checked_sentence': checked_sent }) high_f1_response = Message({'text': checked_sent}) low_f1_response = Message({'text': 'incorrect'}) # 1) Test with correct top sentence teacher.reset_metrics() teacher.custom_evaluation( chosen_sent_teacher_action, [checked_sent_label], correct_chosen_sent_response, ) report = teacher.report() for k in retrieval_metric_keys: assert k in report assert report[k] == AverageMetric(1) # 2) Test with top sentence in top 5 teacher.reset_metrics() teacher.custom_evaluation(chosen_sent_teacher_action, [checked_sent_label], top5_chosen_sent_response) report = teacher.report() for k in retrieval_metric_keys: assert k in report assert report[k] == AverageMetric( 1) if '5' in k else AverageMetric(0) # 3) Test with no top sentences teacher.reset_metrics() teacher.custom_evaluation( chosen_sent_teacher_action, [checked_sent_label], incorrect_chosen_sent_response, ) report = teacher.report() for k in retrieval_metric_keys: assert k in report assert report[k] == AverageMetric(0) # 4) Test knowledge f1 with high f1 teacher.label_type = 'response' teacher.reset_metrics() teacher.custom_evaluation(response_teacher_action, [response], high_f1_response) report = teacher.report() assert 'knowledge_f1' in report assert report['knowledge_f1'] == F1Metric(1) # 5) Test knowledge f1 with low f1 teacher.reset_metrics() teacher.custom_evaluation(response_teacher_action, [response], low_f1_response) report = teacher.report() assert 'knowledge_f1' in report assert report['knowledge_f1'] == F1Metric(0)