def setup_args(parser=None): if parser is None: parser = ParlaiParser(True, True, 'Evaluate a model') # Get command line arguments parser.add_argument( '-rf', '--report-filename', type=str, default='', help='Saves a json file of the evaluation report either as an ' 'extension to the model-file (if begins with a ".") or a whole ' 'file path. Set to the empty string to not save at all.', ) parser.add_argument( '--save-world-logs', type='bool', default=False, help='Saves a jsonl file containing all of the task examples and ' 'model replies. Must also specify --report-filename.', ) parser.add_argument( '--save-format', type=str, default='conversations', choices=['conversations', 'parlai'], ) parser.add_argument('-ne', '--num-examples', type=int, default=-1) parser.add_argument('-d', '--display-examples', type='bool', default=False) parser.add_argument('-ltim', '--log-every-n-secs', type=float, default=10) parser.add_argument( '-mcs', '--metrics', type=str, default='default', help='list of metrics to show/compute, e.g. all, default,' 'or give a list split by , like ' 'ppl,f1,accuracy,hits@1,rouge,bleu' 'the rouge metrics will be computed as rouge-1, rouge-2 and rouge-l', ) parser.add_argument( '-micro', '--aggregate-micro', type='bool', default=False, help='Report micro-averaged metrics instead of macro averaged metrics.', recommended=False, ) WorldLogger.add_cmdline_args(parser) TensorboardLogger.add_cmdline_args(parser) parser.set_params(datatype='valid') return parser
def get_context_generator( override_opt: Optional[Dict[str, Any]] = None) -> ContextGenerator: """ Return an object to return BlendedSkillTalk-style context info (personas, etc.). """ argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() if override_opt is not None: argparser.set_params(**override_opt) opt = argparser.parse_args([]) context_generator = ContextGenerator(opt, datatype='test', seed=0) # We pull from the test set so that the model can't regurgitate # memorized conversations return context_generator
def _create_safety_model(self, custom_model_file, device): from parlai.core.params import ParlaiParser parser = ParlaiParser(False, False) TransformerClassifierAgent.add_cmdline_args(parser, partial_opt=None) parser.set_params( model='transformer/classifier', model_file=custom_model_file, print_scores=True, data_parallel=False, ) safety_opt = parser.parse_args([]) safety_opt["override"]["no_cuda"] = False if "cuda" in device else True return create_agent(safety_opt, requireModelExists=True)
def get_agent(**kwargs): r""" Return opt-initialized agent. :param kwargs: any kwargs you want to set using parser.set_params(\*\*kwargs) """ if 'no_cuda' not in kwargs: kwargs['no_cuda'] = True from parlai.core.params import ParlaiParser parser = ParlaiParser() MockTorchAgent.add_cmdline_args(parser) parser.set_params(**kwargs) opt = parser.parse_args([], print_args=False) return MockTorchAgent(opt)
def test_maintain_dialog_history(self): try: from parlai.core.torch_agent import TorchAgent except ImportError as e: if 'pytorch' in e.msg: print( 'Skipping TestTorchAgent.test_maintain_dialog_history, no pytorch.' ) return from parlai.core.params import ParlaiParser parser = ParlaiParser() TorchAgent.add_cmdline_args(parser) parser.set_params(no_cuda=True, truncate=5) opt = parser.parse_args(print_args=False) mdict = MockDict() shared = {'opt': opt, 'dict': mdict} agent = TorchAgent(opt, shared) observation = { "text": "What is a painting?", "labels": ["Paint on a canvas."], "episode_done": False } agent.maintain_dialog_history(observation) self.assertTrue('dialog' in agent.history, "Failed initializing self.history.") self.assertTrue('episode_done' in agent.history, "Failed initializing self.history.") self.assertTrue('labels' in agent.history, "Failed initializing self.history.") self.assertTrue( list(agent.history['dialog']) == [7, 8, 9], "Failed adding vectorized text to dialog.") self.assertTrue(not agent.history['episode_done'], "Failed to properly store episode_done field.") self.assertTrue(agent.history['labels'] == observation['labels'], "Failed saving labels.") observation['text_vec'] = agent.maintain_dialog_history(observation) print(agent.history['dialog']) self.assertTrue( list(agent.history['dialog']) == [8, 9, 7, 8, 9], "Failed adding vectorized text to dialog.")
def _set_up_knowledge_agent(self, add_token_knowledge: bool = False, shared=None) -> None: """ Set up knowledge agent for knowledge retrieval generated from WoW project. """ parser = ParlaiParser(False, False) KnowledgeRetrieverAgent.add_cmdline_args(parser) parser.set_params( model='projects:wizard_of_wikipedia:knowledge_retriever', add_token_knowledge=add_token_knowledge, ) knowledge_opt = parser.parse_args([]) if shared: self.knowledge_agent = KnowledgeRetrieverAgent( knowledge_opt, shared.get('knowledge_retriever', None)) else: self.knowledge_agent = KnowledgeRetrieverAgent(knowledge_opt)
def setup_args(parser=None): if parser is None: parser = ParlaiParser(True, True, 'compute statistics from model predictions') DictionaryAgent.add_cmdline_args(parser) # These defaults can be overriden by both .opt file and user's command line flags parser.add_argument('-ne', '--num-examples', type=int, default=-1) parser.add_argument('-ltim', '--log-every-n-secs', type=float, default=2) parser.add_argument( '-ed', '--external-dict', type=str, default=None, help='External dictionary for stat computation', ) parser.add_argument( '-fb', '--freq-bins', type=str, default='0,100,1000,10000', help='Bins boundaries for rare words stat', ) parser.add_argument( '-gr', '--gold-response', type=bool, default=False, help='Compute stats for gold response', ) # These settings override .opt file but not user's command line flags parser.set_params( datatype='valid', task='projects.controllable_dialogue.tasks.agents', model= 'projects.controllable_dialogue.controllable_seq2seq.controllable_seq2seq:ControllableSeq2seqAgent', # noqa: E501 batchsize=64, beam_size=20, beam_min_n_best=10, use_reply='model', ) TensorboardLogger.add_cmdline_args(parser) return parser
def get_context_generator( override_opt: Optional[Dict[str, Any]] = None, task: Optional[str] = 'blended_skill_talk', **kwargs, ) -> ContextGenerator: """ Return an object to return BlendedSkillTalk-style context info (personas, etc.). """ argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() if override_opt is not None: argparser.set_params(**override_opt) opt = argparser.parse_args([]) task_module = load_task_module(task) context_generator_class = getattr(task_module, 'ContextGenerator', None) context_generator = context_generator_class(opt, datatype='test', seed=0, **kwargs) # We pull from the test set so that the model can't regurgitate # memorized conversations return context_generator
def test_resize_embeddings(self): # train original model with testing_utils.tempdir() as tmpdir: model_file = os.path.join(tmpdir, 'model_file') _, _ = testing_utils.train_model( dict( model='transformer/generator', task='integration_tests:short_fixed', n_layers=1, n_encoder_layers=2, n_decoder_layers=4, num_epochs=1, dict_tokenizer='bytelevelbpe', bpe_vocab=DEFAULT_BYTELEVEL_BPE_VOCAB, bpe_merge=DEFAULT_BYTELEVEL_BPE_MERGE, bpe_add_prefix_space=False, model_file=model_file, save_after_valid=True, ) ) # now create agent with special tokens parser = ParlaiParser() parser.set_params( model='transformer/generator', task='integration_tests:short_fixed', n_layers=1, n_encoder_layers=2, n_decoder_layers=4, dict_tokenizer='bytelevelbpe', bpe_vocab=DEFAULT_BYTELEVEL_BPE_VOCAB, bpe_merge=DEFAULT_BYTELEVEL_BPE_MERGE, bpe_add_prefix_space=False, model_file=model_file, save_after_valid=True, special_tok_lst='PARTY,PARROT', ) opt = parser.parse_args([]) agent = create_agent(opt) # assert that the embeddings were resized assert agent.resized_embeddings # assert model has special tokens self.assertEqual(agent.special_toks, ['PARTY', 'PARROT'])
def get_bot_agents(args: DictConfig, model_opts: Dict[str, str], no_cuda=False) -> Dict[str, dict]: """ Return shared bot agents. Pass in model opts with the `model_opts` arg, where `model_opts` is a dictionary whose keys are model names and whose values are strings that specify model params (i.e. `--model image_seq2seq`). """ # Set up overrides model_overrides = { 'model_parallel': args.blueprint.task_model_parallel } if no_cuda: # If we load many models at once, we have to keep it on CPU model_overrides['no_cuda'] = no_cuda else: logging.warning( 'WARNING: MTurk task has no_cuda FALSE. Models will run on GPU. Will ' 'not work if loading many models at once.') # Convert opt strings to Opt objects parser = ParlaiParser(True, True) parser.set_params(**model_overrides) processed_opts = {} for name, opt_string in model_opts.items(): processed_opts[name] = parser.parse_args(opt_string.split()) # Load and share all model agents logging.info( f'Got {len(list(processed_opts.keys()))} models: {processed_opts.keys()}.' ) shared_bot_agents = {} for model_name, model_opt in processed_opts.items(): logging.info('\n\n--------------------------------') logging.info(f'model_name: {model_name}, opt_dict: {model_opt}') model_agent = create_agent(model_opt, requireModelExists=True) shared_bot_agents[model_name] = model_agent.share() return shared_bot_agents
def add_cmdline_args(cls, parser: ParlaiParser, partial_opt: Optional[Opt] = None) -> ParlaiParser: super().add_cmdline_args(parser, partial_opt) group = parser.add_argument_group( 'Gender Multiclass Interactive World') group.add_argument( '--self-threshold', type=float, default=0.52, help='Threshold for choosing unknown for self', ) group.add_argument( '--partner-threshold', type=float, default=0.52, help='Threshold for choosing unknown for self', ) parser.set_params( single_turn=True, # this is a single turn task currently eval_candidates='inline', return_cand_scores=True, ) return parser
def test_knowledge_retriever(self): from parlai.core.params import ParlaiParser parser = ParlaiParser(False, False) KnowledgeRetrieverAgent.add_cmdline_args(parser) parser.set_params( model='projects:wizard_of_wikipedia:knowledge_retriever', add_token_knowledge=True, ) knowledge_opt = parser.parse_args([], print_args=False) knowledge_agent = create_agent(knowledge_opt) knowledge_agent.observe( { 'text': 'what do you think of mountain dew?', 'chosen_topic': 'Mountain Dew', 'episode_done': False, } ) knowledge_act = knowledge_agent.act() title = knowledge_act['title'] self.assertEqual(title, 'Mountain Dew', 'Did not save chosen topic correctly') knowledge = knowledge_act['text'] self.assertIn( TOKEN_KNOWLEDGE, knowledge, 'Knowledge token was not inserted correctly' ) checked_sentence = knowledge_act['checked_sentence'] self.assertEqual( checked_sentence, 'Mountain Dew (stylized as Mtn Dew) is a carbonated soft drink brand produced and owned by PepsiCo.', 'Did not correctly choose the checked sentence', )
# All rights reserved. # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. """Interact with a pre-trained model. This seq2seq model was trained on convai2:self. """ from parlai.core.build_data import download_models from parlai.core.params import ParlaiParser from parlai.scripts.interactive import interactive if __name__ == '__main__': parser = ParlaiParser(add_model_args=True) parser.set_params( model='legacy:seq2seq:0', model_file='models:convai2/seq2seq/convai2_self_seq2seq_model', dict_file='models:convai2/seq2seq/convai2_self_seq2seq_model.dict', dict_lower=True, batchsize=1, ) opt = parser.parse_args() if (opt.get('model_file', '') .find('convai2/seq2seq/convai2_self_seq2seq_model') != -1): opt['model_type'] = 'seq2seq' fnames = ['convai2_self_seq2seq_model.tgz', 'convai2_self_seq2seq_model.dict', 'convai2_self_seq2seq_model.opt'] download_models(opt, fnames, 'convai2', version='v3.0') interactive(opt)
Results on unseen test set (run with flag `-t wizard_of_wikipedia:WizardDialogKnowledge:topic_split`): Hits@1/100: 68.96 """ if __name__ == '__main__': parser = ParlaiParser(add_model_args=True) parser.add_argument('-n', '--num-examples', default=100000000) parser.add_argument('-d', '--display-examples', type='bool', default=False) parser.add_argument('-ltim', '--log-every-n-secs', type=float, default=2) WizardTransformerRankerAgent.add_cmdline_args(parser, partial_opt=None) parser.set_params( task='wizard_of_wikipedia', model='projects:wizard_of_wikipedia:wizard_transformer_ranker', model_file= 'models:wizard_of_wikipedia/full_dialogue_retrieval_model/model', datatype='test', n_heads=6, ffn_size=1200, embeddings_scale=False, delimiter=' __SOC__ ', n_positions=1000, legacy=True, ) opt = parser.parse_args() download(opt['datapath']) # download pretrained retrieval model eval_model(opt)
def setup_args(parser=None): if parser is None: parser = ParlaiParser(True, True, 'Evaluate a model') # Get command line arguments parser.add_argument( '-rf', '--report-filename', type=str, default='', help='Saves a json file of the evaluation report either as an ' 'extension to the model-file (if begins with a ".") or a whole ' 'file path. Set to the empty string to not save at all.', ) parser.add_argument( '--world-logs', type=str, default='', help='Saves a jsonl file of the world logs.' 'Set to the empty string to not save at all.', ) parser.add_argument( '--save-format', type=str, default='conversations', choices=['conversations', 'parlai'], ) parser.add_argument( '--area-under-curve-digits', '-auc', type=int, default=-1, help= 'a positive number indicates to calculate the area under the roc curve and it also determines how many decimal digits of the predictions to keep (higher numbers->more precise); also used to determine whether or not to calculate the AUC metric', ) parser.add_argument( '--area-under-curve-class', '-auclass', type=str, default=None, nargs='*', help='the name(s) of the class to calculate the auc for', ) parser.add_argument('-ne', '--num-examples', type=int, default=-1) parser.add_argument('-d', '--display-examples', type='bool', default=False) parser.add_argument('-ltim', '--log-every-n-secs', type=float, default=10) parser.add_argument( '-mcs', '--metrics', type=str, default='default', help='list of metrics to show/compute, e.g. all, default,' 'or give a list split by , like ' 'ppl,f1,accuracy,hits@1,rouge,bleu' 'the rouge metrics will be computed as rouge-1, rouge-2 and rouge-l', ) parser.add_argument( '-micro', '--aggregate-micro', type='bool', default=False, help='Report micro-averaged metrics instead of macro averaged metrics.', recommended=False, ) WorldLogger.add_cmdline_args(parser, partial_opt=None) TensorboardLogger.add_cmdline_args(parser, partial_opt=None) parser.set_params(datatype='valid') return parser
def setup_args(parser=None): if parser is None: parser = ParlaiParser() parser.set_params( model='parlai.agents.local_human.local_human:LocalHumanAgent') return parser
def add_cmdline_args(cls, parser: ParlaiParser, partial_opt=None) -> ParlaiParser: super().add_cmdline_args(parser, partial_opt) parser.set_params(prepend_gold_knowledge=True) return parser
from parlai.scripts.interactive import interactive from parlai.agents.language_model.language_model import LanguageModelAgent '''Interact with pre-trained model Language model trained on Opensubtitles 2018 dataset Run from ParlAI directory ''' if __name__ == '__main__': parser = ParlaiParser(add_model_args=True) parser.add_argument('-d', '--display-examples', type='bool', default=False) LanguageModelAgent.add_cmdline_args(parser) parser.set_params( dict_file= 'models:personachat/language_model/languagemodel_esz512_hid1024_nl2.pt.dict', sampling_mode=True, task='parlai.agents.local_human.local_human:LocalHumanAgent', model='language_model', model_file= 'models:personachat/language_model/languagemodel_esz512_hid1024_nl2.pt' ) opt = parser.parse_args() opt['model_type'] = 'language_model' # for builder # build all profile memory models fnames = [ 'languagemodel_esz512_hid1024_nl2.pt', 'languagemodel_esz512_hid1024_nl2.pt.opt', 'languagemodel_esz512_hid1024_nl2.pt.dict' ] download_models(opt, fnames, 'personachat', version='v3.0')
#!/usr/bin/env python3 # Copyright (c) Facebook, Inc. and its affiliates. # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. """ Interact with a pre-trained model. Key-Value Memory Net model trained on personachat using persona 'self'. [Note: no persona in this example code is actually given to the model.] """ from parlai.core.build_data import download_models from parlai.core.params import ParlaiParser from parlai.scripts.interactive import interactive if __name__ == '__main__': parser = ParlaiParser(add_model_args=True) parser.add_argument('-d', '--display-examples', type='bool', default=False) parser.set_params( model='projects.personachat.kvmemnn.kvmemnn:KvmemnnAgent', model_file='models:convai2/kvmemnn/model', interactive_mode=True, ) opt = parser.parse_args() # build all profile memory models fnames = ['kvmemnn.tgz'] opt['model_type'] = 'kvmemnn' # for builder download_models(opt, fnames, 'convai2') interactive(opt)
# Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. from parlai.core.build_data import download_models from parlai.core.params import ParlaiParser from parlai.scripts.interactive import interactive '''Interact with pre-trained model Key-Value Memory Net model trained on personachat using persona 'self' [Note: no persona in this example code is actually given to the model.] ''' if __name__ == '__main__': parser = ParlaiParser(add_model_args=True) parser.add_argument('-d', '--display-examples', type='bool', default=False) parser.set_params( task='parlai.agents.local_human.local_human:LocalHumanAgent', model='projects.personachat.kvmemnn.kvmemnn:KvmemnnAgent', model_file= 'models:personachat/kvmemnn/kvmemnn/persona-self_rephraseTrn-True_rephraseTst-False_lr-0.1_esz-500_margin-0.1_tfidf-False_shareEmb-True_hops1_lins0_model', interactive_mode=True, ) opt = parser.parse_args() # build all profile memory models fnames = ['kvmemnn.tgz'] opt['model_type'] = 'kvmemnn' # for builder download_models(opt, fnames, 'personachat') interactive(opt)
def add_cmdline_args(cls, parser: ParlaiParser, partial_opt: Optional[Opt] = None) -> ParlaiParser: """ Add command line arguments. """ Transformer.add_common_cmdline_args(parser) agent = parser.add_argument_group('TransresnetModel arguments') agent.add_argument( '--truncate', type=int, default=32, help='Max amount of tokens allowed in a text sequence', ) agent.add_argument( '--image-features-dim', type=int, default=2048, help='dimensionality of image features', ) agent.add_argument( '--embedding-type', type=str, default=None, choices=[None, 'fasttext_cc'], help='Specify if using pretrained embeddings', ) agent.add_argument( '--load-encoder-from', type=str, default=None, help='Specify if using a pretrained transformer encoder', ) agent.add_argument( '--hidden-dim', type=int, default=300, help='Hidden dimesionality of personality and image encoder', ) agent.add_argument( '--num-layers-all', type=int, default=-1, help='If >= 1, number of layers for both the text ' 'and image encoders.', ) agent.add_argument( '--num-layers-text-encoder', type=int, default=1, help='Number of layers for the text encoder', ) agent.add_argument( '--num-layers-image-encoder', type=int, default=1, help='Number of layers for the image encoder', ) agent.add_argument('--no-cuda', dest='no_cuda', action='store_true', help='If True, perform ops on CPU only') agent.add_argument( '--learningrate', type=float, default=0.0005, help='learning rate for optimizer', ) agent.add_argument( '--additional-layer-dropout', type=float, default=0.2, help='dropout for additional linear layer', ) parser.set_params(ffn_size=1200, attention_dropout=0.2, relu_dropout=0.2, n_positions=1000) return parser
def run_task(override_opt: Optional[dict] = None): """ This task consists of an MTurk worker talking to a model and MTurker also evaluates each utterance of the bot for various buckets (see constants). """ config_folder = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'task_config') argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() default_task_folder = os.path.join(argparser.parlai_home, 'data', 'turn_annotations') argparser.add_mturk_args() argparser.add_argument('-num_t', '--num_turns', default=6, type=int, help='minimum number of turns') argparser.add_argument( '--conversations-needed', dest='conversations_needed_string', default=None, type=str, help= 'Number of convos needed for each model. For example: "modelA:50,modelB:20"', ) argparser.add_argument( '--task-model-parallel', default=True, type=bool, help='Whether to load models to be used with model_parallel True.', ) argparser.add_argument( '--auto-approve-delay', dest='auto_approve_delay', type=int, default=3600 * 24 * 5, help='how long to wait for auto approval', ) argparser.add_argument( '--max-resp-time', type=int, default=180, help='time limit for entering a dialog message', ) argparser.add_argument( '--max-onboard-time', type=int, default=300, help='time limit accepting onboarding', ) argparser.add_argument( '--base-save-folder', default=default_task_folder, type=str, help='base folder for saving all crowdsourcing results', ) argparser.add_argument( '--base-model-folder', default=None, type=str, help='base folder for loading model files from', ) argparser.add_argument( '--onboard-worker-answer-folder', default=os.path.join(default_task_folder, 'onboard_answers'), type=str, help= 'base folder for saving all worker answer results during onboarding', ) argparser.add_argument( '--worker-blocklist-paths', default=None, type=str, help= 'Path(s) to a list of IDs of workers to soft-block, separated by newlines. Use commas to indicate multiple lists', ) argparser.add_argument( '--check-acceptability', default=False, type=bool, help= "Check worker's responses against several metrics of acceptability", ) argparser.add_argument('--include-persona', default=False, type=bool, help="Show persona to the bot") argparser.add_argument( '--conversation-start-mode', default='hi', type=str, choices=['hi', 'bst'], help= 'Whether to show "Hi!" or two previous utterances (as in BlendedSkillTalk) at the beginning of the conversation', ) argparser.add_argument( '--context-seed', default=None, type=int, help="Set seed for pulling the context info (for testing)", ) argparser.add_argument( '--hit-config-path', default=os.path.join(config_folder, 'hit_config.json'), type=str, help= 'Path to file of parameters describing how MTurk will describe the HIT to the workers', ) argparser.add_argument( '--task-description-path', default=os.path.join(config_folder, 'task_description.html'), type=str, help='Path to file of HTML to show on the task-description page', ) argparser.add_argument( '--left-pane-text-path', default=os.path.join(config_folder, 'left_pane_text.html'), type=str, help= 'Path to file of HTML to show on the left-hand pane of the chat window', ) argparser.add_argument( '--annotations-intro', default= 'Does this comment from your partner have any of the following attributes? (Check all that apply)', type=str, help='Text shown to worker before they fill out annotation form', ) argparser.add_argument( '--annotations-config-path', default=os.path.join(config_folder, 'annotations_config.json'), type=str, help='Path to JSON of annotation categories', ) argparser.add_argument( '--onboard-task-data-path', default=os.path.join(config_folder, 'onboard_task_data.json'), type=str, help='Path to JSON containing settings for running onboarding', ) argparser.add_argument( '--final-rating-question', default='Please rate your partner on a scale of 1-5.', type=str, help='Text to show when asking worker to make their final rating', ) # NOTE: you have to set all three of these opts to enforce the MTurk core # param max_hits_per_worker. # - Without unique_qual_name, MTurkManager creates different qualification # for each run (so a worker could do N hits per run) Also, the # worker has to get to N HITs in at least one run or they won't be given # the qualification. # - allowed_conversations is like max concurrent conversations # allowed_conversations needs to be 1 or the actual max would be N + # allowed_conversations. Worker gets notified via frontend message that # they aren't eligible (second description screen), UNLESS the frontend # overwrites that functionality. # There's also still a race condition where the worker might be able to open # 1 extra task argparser.set_defaults( unique_qual_name='turn_annotations_max_submissions', max_hits_per_worker=10, allowed_conversations=3, ) if override_opt is not None: argparser.set_params(**override_opt) opt = argparser.parse_args([]) else: opt = argparser.parse_args() directory_path = os.path.dirname(os.path.abspath(__file__)) opt['task'] = os.path.basename(directory_path) # Set the number of conversations needed if opt.get('conversations_needed_string') is not None: parts = opt['conversations_needed_string'].split(',') conversations_needed = {} for part in parts: model_name, num_string = part.split(':') conversations_needed[model_name] = int(num_string) opt['conversations_needed'] = conversations_needed # Read in workers to soft-block if opt.get('worker_blocklist_paths') is not None: blocklist_paths = opt['worker_blocklist_paths'].split(',') worker_blocklist = set() for path in blocklist_paths: with open(path) as f: worker_blocklist |= set(f.read().strip().split('\n')) opt['worker_blocklist'] = worker_blocklist # Read in and define text shown to users if opt.get('hit_config') is None: with open(opt['hit_config_path']) as f: opt['hit_config'] = json.load(f) opt.update(opt['hit_config']) # Add all of the settings in hit_config into the base opt if opt.get('task_description') is None: with open(opt['task_description_path']) as f: opt['task_description'] = f.readlines() if opt.get('left_pane_text') is None: with open(opt['left_pane_text_path']) as f: opt['left_pane_text'] = f.readlines() if opt.get('annotations_config') is None: with open(opt['annotations_config_path']) as f: opt['annotations_config'] = json.load(f) if opt.get('onboard_task_data') is None: with open(opt['onboard_task_data_path']) as f: opt['onboard_task_data'] = json.load(f) # Limits the number of models that can generate at once max_concurrent_responses = 1 semaphore = threading.Semaphore(max_concurrent_responses) run_statistics = copy.deepcopy(opt['conversations_needed']) run_statistics = {r: 0 for (r, v) in run_statistics.items()} onboard_statistics = {} save_folder = 'sandbox' if opt['is_sandbox'] else 'live' opt['save_folder'] = os.path.join(opt['base_save_folder'], save_folder, time.strftime("%Y_%m_%d")) os.makedirs(opt['save_folder'], exist_ok=True) print( f'Going to start collecting {opt["num_conversations"]} conversations, max_hits_per_worker: {opt["max_hits_per_worker"]}, reward: {opt["reward"]}, is_sandbox: {opt["is_sandbox"]}.' ) # Create the models before it launches Heroku backend b/c takes a while models_needed = list(opt['conversations_needed'].keys()) active_models = [ m for m in models_needed if opt['conversations_needed'][m] > 0 ] shared_bot_agents = TurkLikeAgent.get_bot_agents(opt, active_models) mturk_agent_ids = [AGENT_0] mturk_manager = MTurkManager(opt=opt, mturk_agent_ids=mturk_agent_ids) mturk_manager.setup_server(task_directory_path=directory_path) if opt['include_persona'] or opt['conversation_start_mode'] == 'bst': context_generator = ContextGenerator(opt, datatype='test', seed=0) # We pull from the test set so that the model can't regurgitate # memorized conversations else: context_generator = None try: mturk_manager.start_new_run() mturk_manager.create_hits() if not opt['is_sandbox']: # Soft-block all chosen workers if len(opt['worker_blocklist']) > 0: print( f"About to soft-block {len(opt['worker_blocklist'])} workers." ) for w in set(opt['worker_blocklist']): try: print('Soft Blocking {}\n'.format(w)) mturk_manager.soft_block_worker(w) except Exception as e: print(f'Did not soft block worker {w}: {e}') time.sleep(0.1) else: print( 'WARNING: We are in live mode, but a list of workers to soft-block ' 'has not been passed in.') def run_onboard(worker): world = TurnAnnotationsOnboardWorld(opt, worker) status = world.parley() if status not in onboard_statistics: onboard_statistics[status] = 0 onboard_statistics[status] += 1 print( f'After onboard world parley. About to shutdown onboard world for {worker.worker_id}, status was: {status}. Total onboard statistics for this run are: {onboard_statistics}.' ) world.shutdown() mturk_manager.set_onboard_function(onboard_function=run_onboard) mturk_manager.ready_to_accept_workers() def check_worker_eligibility(worker): return True def assign_worker_roles(workers): workers[0].id = mturk_agent_ids[0] def run_conversation(mturk_manager, opt, workers): remaining_counts_needed = [ (m, c - run_statistics[m]) for (m, c) in opt['conversations_needed'].items() ] remaining_counts_needed.sort(reverse=True, key=lambda x: x[1]) model_name = remaining_counts_needed[0][0] print( f'Remaining conversation counts needed: {remaining_counts_needed}' ) # Get a bot and add it to the list of "workers" print(f'Choosing the "{model_name}" model for the bot.') agent = create_agent_from_shared(shared_bot_agents[model_name]) bot_worker = TurkLikeAgent( opt, model_name=model_name, model_agent=agent, num_turns=opt['num_turns'], semaphore=semaphore, ) workers_including_bot = workers + [bot_worker] assert len(workers_including_bot) == 2 # Get context: personas, previous utterances, etc. if context_generator is not None: context_info = context_generator.get_context() else: context_info = None conv_idx = mturk_manager.conversation_index world = TurnAnnotationsChatWorld( opt=opt, agents=workers_including_bot, num_turns=opt['num_turns'], max_resp_time=opt['max_resp_time'], tag='conversation t_{}'.format(conv_idx), context_info=context_info, ) while not world.episode_done(): print('About to parley') world.parley() model_nickname, worker_is_unacceptable, convo_finished = world.save_data( ) if worker_is_unacceptable: print(f'Soft-blocking worker {workers[0].worker_id}') mturk_manager.soft_block_worker(workers[0].worker_id) time.sleep(0.1) if not worker_is_unacceptable and convo_finished: run_statistics[model_nickname] += 1 world.shutdown() world.review_work() mturk_manager.start_task( eligibility_function=check_worker_eligibility, assign_role_function=assign_worker_roles, task_function=run_conversation, ) except BaseException: raise finally: mturk_manager.expire_all_unassigned_hits() mturk_manager.shutdown()
# Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. """Interact with a pre-trained model. Key-Value Memory Net model trained on personachat using persona 'self' [Note: no persona in this example code is actually given to the model.] """ from parlai.core.build_data import download_models from parlai.core.params import ParlaiParser from parlai.scripts.interactive import interactive if __name__ == '__main__': parser = ParlaiParser(add_model_args=True) parser.add_argument('-d', '--display-examples', type='bool', default=False) parser.set_params( model='projects.personachat.kvmemnn.arms:ArmsAgent', model_file='/tmp/persona_self_original.checkpoint', #model='projects.personachat.kvmemnn.kvmemnn:KvmemnnAgent', #model_file='models:convai2/kvmemnn/model', interactive_mode=True, ) opt = parser.parse_args() # build all profile memory models fnames = ['kvmemnn.tgz'] opt['model_type'] = 'kvmemnn' # for builder download_models(opt, fnames, 'convai2') interactive(opt)
# LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. """Interact with a pre-trained model. This transformer model was trained on convai2:self. """ from parlai.core.build_data import download_models, modelzoo_path from parlai.core.params import ParlaiParser from parlai.scripts.interactive import interactive if __name__ == '__main__': parser = ParlaiParser(add_model_args=True) parser.set_params( model='transformer', model_file='models:convai2/transformer/convai2_self_transformer_model', dict_file= 'models:convai2/transformer/convai2_self_transformer_model.dict', dict_lower=True, ) opt = parser.parse_args() if opt.get('model_file', '').startswith( modelzoo_path(opt.get('datapath'), "models:convai2")): opt['model_type'] = 'transformer' fnames = [ 'convai2_self_transformer_model.tgz', 'convai2_self_transformer_model.dict', 'convai2_self_transformer_model.opt' ] download_models(opt, fnames, 'convai2', version='v3.0') interactive(opt)
def get_parlai_opt(self) -> Opt: """ Parser for converting fairseq argument to ParlAI opt. :return opt: opt parsed by ParlAI Parser """ # assume encoder/decoder symetrical except for number of layers state = self.state fairseq_args = state['args'].__dict__ transformer_common_config = {} # 1. Map transformer params for each in TRANSFORMER_PARAMETER_MAPPING: transformer_common_config[TRANSFORMER_PARAMETER_MAPPING[ each]] = fairseq_args[f'encoder_{each}'] # 2. Map dropout for each in TRANSFORMER_DROPOUT: transformer_common_config[each] = fairseq_args[each] if 'activation_dropout' in fairseq_args: transformer_common_config['relu_dropout'] = fairseq_args[ 'activation_dropout'] else: transformer_common_config['relu_dropout'] = fairseq_args[ 'relu_dropout'] # 3. Map other options transformer_common_config.update({ 'model': self.opt['model'], # number of layers 'n_encoder_layers': fairseq_args['encoder_layers'], 'n_decoder_layers': fairseq_args['decoder_layers'], # tokenization args 'dict_tokenizer': self.opt['tokenizer'], 'bpe_vocab': self.opt['vocab'], 'bpe_merge': self.opt['merge'], 'n_positions': fairseq_args['max_source_positions'], }) # 4. Embedding scale if 'encoder_embed_scale' in fairseq_args: transformer_common_config['embeddings_scale'] = ( fairseq_args['encoder_embed_scale'] != 1.0) else: transformer_common_config[ 'embeddings_scale'] = not fairseq_args['no_scale_embedding'] # 5. Determine variant if fairseq_args['encoder_normalize_before']: transformer_common_config['variant'] = 'prelayernorm' elif fairseq_args['layernorm_embedding']: transformer_common_config['variant'] = 'bart' else: transformer_common_config['variant'] = 'aiayn' if self.opt['add_prefix_space']: transformer_common_config['bpe_add_prefix_space'] = True parser = ParlaiParser() parser.set_params(**transformer_common_config) opt = parser.parse_args([]) # 6. Augment opt with additional ParlAI options opt['fp16'] = self.opt['fp16'] opt['activation'] = self.opt['activation'] opt['delimiter'] = self.opt['delimiter'] opt['history_add_global_end_token'] = self.opt[ 'history_add_global_end_token'] # Makes model fp16 ready for fine-tuning, means 4 extra padding tokens. opt['force_fp16_tokens'] = True opt['converting'] = True return opt
def test_vectorize(self): """ Make sure that the vectorize function is actually adding a new field. """ try: from parlai.core.torch_agent import TorchAgent except ImportError as e: if 'pytorch' in e.msg: print('Skipping TestTorchAgent.test_vectorize, no pytorch.') return from parlai.core.params import ParlaiParser parser = ParlaiParser() TorchAgent.add_cmdline_args(parser) parser.set_params(no_cuda=True) opt = parser.parse_args(print_args=False) mdict = MockDict() shared = {'opt': opt, 'dict': mdict} agent = TorchAgent(opt, shared) observation = {} observation["text"] = "What does the dog do?" observation["labels"] = ["The dog jumps over the cat."] # add start and end obs_vec = agent.vectorize(observation, add_start=True, add_end=True) self.assertTrue( 'text_vec' in obs_vec, "Field 'text_vec' missing from vectorized observation") self.assertTrue(obs_vec['text_vec'].numpy().tolist() == [7, 8, 9], "Vectorized text is incorrect.") self.assertTrue( 'labels_vec' in obs_vec, "Field 'labels_vec' missing from vectorized observation") self.assertTrue( obs_vec['labels_vec'].numpy().tolist() == [ mdict.START_IDX, 7, 8, 9, mdict.END_IDX ], "Vectorized label is incorrect.") # no start, add end obs_vec = agent.vectorize(observation, add_start=False, add_end=True) self.assertTrue( obs_vec['labels_vec'].numpy().tolist() == [7, 8, 9, mdict.END_IDX], "Vectorized label is incorrect.") # add start, no end obs_vec = agent.vectorize(observation, add_start=True, add_end=False) self.assertTrue( obs_vec['labels_vec'].numpy().tolist() == [ mdict.START_IDX, 7, 8, 9 ], "Vectorized label is incorrect.") # no start, no end obs_vec = agent.vectorize(observation, add_start=False, add_end=False) self.assertTrue(obs_vec['labels_vec'].numpy().tolist() == [7, 8, 9], "Vectorized label is incorrect.") observation = {} observation["text"] = "What does the dog do?" observation["eval_labels"] = ["The dog jumps over the cat."] # eval_labels obs_vec = agent.vectorize(observation) self.assertTrue( 'eval_labels_vec' in obs_vec, "Field \'eval_labels_vec\' missing from vectorized observation") self.assertTrue( obs_vec['eval_labels_vec'].numpy().tolist() == [ mdict.START_IDX, 7, 8, 9, mdict.END_IDX ], "Vectorized label is incorrect.") # truncate obs_vec = agent.vectorize(observation, truncate=2) self.assertTrue( 'eval_labels_vec' in obs_vec, "Field \'eval_labels_vec\' missing from vectorized observation") self.assertTrue( obs_vec['eval_labels_vec'].numpy().tolist() == [ mdict.START_IDX, 7 ], "Vectorized label is incorrect: " + str(obs_vec['eval_labels_vec'])) # truncate obs_vec = agent.vectorize(observation, truncate=10) self.assertTrue( 'eval_labels_vec' in obs_vec, "Field \'eval_labels_vec\' missing from vectorized observation") self.assertTrue( obs_vec['eval_labels_vec'].numpy().tolist() == [ mdict.START_IDX, 7, 8, 9, mdict.END_IDX ], "Vectorized label is incorrect.")
# Copyright (c) 2017-present, Facebook, Inc. # All rights reserved. # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. An additional grant # of patent rights can be found in the PATENTS file in the same directory. """Interact with a pre-trained model. This seq2seq model was trained on convai2:self. """ from parlai.core.build_data import download_models from parlai.core.params import ParlaiParser from parlai.scripts.interactive import interactive if __name__ == '__main__': parser = ParlaiParser(add_model_args=True) parser.set_params( model='seq2seq', model_file='models:convai2/seq2seq/convai2_self_seq2seq_model', dict_file='models:convai2/seq2seq/convai2_self_seq2seq_model.dict', dict_lower=True, ) opt = parser.parse_args() if opt.get('model_file', '').startswith('models:convai2'): opt['model_type'] = 'seq2seq' fnames = [ 'convai2_self_seq2seq_model.tgz', 'convai2_self_seq2seq_model.dict', 'convai2_self_seq2seq_model.opt' ] download_models(opt, fnames, 'convai2', version='v3.0') interactive(opt)
# of patent rights can be found in the PATENTS file in the same directory. """Interact with a pre-trained model. This transformer model was trained on convai2:self. """ from parlai.core.build_data import download_models, modelzoo_path from parlai.core.params import ParlaiParser from parlai.scripts.interactive import interactive if __name__ == '__main__': parser = ParlaiParser(add_model_args=True) parser.set_params( model='transformer', #model_file='models:convai2/transformer/convai2_self_transformer_model', #dict_file='models:convai2/transformer/convai2_self_transformer_model.dict', model_file= './checkpoints/convai2_transformer_volta_[l=4,h=2,dw=256,dm=256,di=2048,dk=64,dv=64,src_tgt_share=False,tgt_prj=False,smooth=False]', dict_file= './checkpoints/convai2_transformer_volta_[l=4,h=2,dw=256,dm=256,di=2048,dk=64,dv=64,src_tgt_share=False,tgt_prj=False,smooth=False].dict', dict_lower=True, ) opt = parser.parse_args() if opt.get('model_file', '').startswith( modelzoo_path(opt.get('datapath'), "models:convai2")): opt['model_type'] = 'transformer' #fnames = ['convai2_self_transformer_model.tgz', # 'convai2_self_transformer_model.dict', # 'convai2_self_transformer_model.opt'] fnames = [ 'convai2_transformer_volta_[l=4,h=2,dw=256,dm=256,di=2048,dk=64,dv=64,src_tgt_share=False,tgt_prj=False,smooth=False].tgz', 'convai2_transformer_volta_[l=4,h=2,dw=256,dm=256,di=2048,dk=64,dv=64,src_tgt_share=False,tgt_prj=False,smooth=False].dict' 'convai2_transformer_volta_[l=4,h=2,dw=256,dm=256,di=2048,dk=64,dv=64,src_tgt_share=False,tgt_prj=False,smooth=False].opt'
def __init__(self, task_run: "TaskRun", args: "DictConfig", shared_state: "SharedTaskState"): # Set the number of conversations needed conversations_needed_string = args.blueprint.conversations_needed_string conversations_needed = {} parts = conversations_needed_string.split(',') for part in parts: model_name, num_string = part.split(':') conversations_needed[model_name] = int(num_string) self.conversations_needed = conversations_needed shared_state.conversations_needed = conversations_needed args.blueprint.num_conversations = sum(conversations_needed.values()) # Default conversation initialization super().__init__(task_run, args=args, shared_state=shared_state) random.seed(self.args.blueprint.random_seed) np.random.seed(self.args.blueprint.random_seed) # Load task configuration data beyond the task desscription, as the super does that left_pane_path = os.path.expanduser(args.blueprint.left_pane_text_path) with open(left_pane_path, "r") as left_pane_file: self.left_pane_text = left_pane_file.read() annotations_config_path = os.path.expanduser( args.blueprint.annotations_config_path) with open(annotations_config_path, "r") as annotations_config_file: self.annotations_config = annotations_config_file.read() onboard_task_data_path = os.path.expanduser( args.blueprint.onboard_task_data_path) with open(onboard_task_data_path, "r") as onboard_task_data_file: self.onboard_task_data = json.load(onboard_task_data_file) run_statistics = {r: 0 for (r, v) in self.conversations_needed.items()} shared_state.run_statistics = run_statistics # Initialize models models_needed = list(conversations_needed.keys()) active_models = [ m for m in models_needed if conversations_needed[m] > 0 ] shared_bot_agents = TurkLikeAgent.get_bot_agents(args, active_models) shared_state.shared_models = shared_bot_agents # Context need parlai options argparser = ParlaiParser(False, False) argparser.add_parlai_data_path() if len(args.blueprint.override_opt) > 0: argparser.set_params(**args.blueprint.override_opt) opt = argparser.parse_args([]) if (args.blueprint.include_persona or args.blueprint.conversation_start_mode == 'bst'): context_generator = ContextGenerator(opt, datatype='test', seed=0) # We pull from the test set so that the model can't regurgitate # memorized conversations else: context_generator = None shared_state.context_generator = context_generator # Limits the number of models that can generate at once max_concurrent_responses = 1 semaphore = Semaphore(max_concurrent_responses) # Lock for editing run statistics between threads statistics_condition = Condition() # Move shared state into the world and onboarding opts, such that these # can be used by the worlds shared_state.onboarding_world_opt.update({ 'onboard_statistics': shared_state.onboard_statistics, 'statistics_condition': statistics_condition, 'max_onboard_time': args.blueprint.max_onboard_time, 'onboard_task_data': self.onboard_task_data, 'onboarding_qualification': args.blueprint.onboarding_qualification, }) shared_state.world_opt.update({ 'annotations_config': self.annotations_config, 'block_qualification': args.blueprint.block_qualification, 'conversations_needed': conversations_needed, 'run_statistics': shared_state.run_statistics, 'context_generator': context_generator, 'semaphore': semaphore, 'shared_bot_agents': shared_bot_agents, 'num_turns': args.blueprint.num_turns, 'max_resp_time': args.blueprint.max_resp_time, 'is_sandbox': args.provider.requester_name == 'MOCK_REQUESTER', 'statistics_condition': statistics_condition, 'check_acceptability': args.blueprint.check_acceptability, 'include_persona': args.blueprint.include_persona, 'conversation_start_mode': args.blueprint.conversation_start_mode, 'chat_data_folder': args.blueprint.chat_data_folder, })
def test_map_unmap(self): try: from parlai.core.torch_agent import TorchAgent, Output except ImportError as e: if 'pytorch' in e.msg: print('Skipping TestTorchAgent.test_map_unmap, no pytorch.') return observations = [] observations.append({ "text": "What is a painting?", "labels": ["Paint on a canvas."] }) observations.append({}) observations.append({}) observations.append({ "text": "What is a painting?", "labels": ["Paint on a canvas."] }) observations.append({}) observations.append({}) from parlai.core.params import ParlaiParser parser = ParlaiParser() TorchAgent.add_cmdline_args(parser) parser.set_params(no_cuda=True) opt = parser.parse_args(print_args=False) mdict = MockDict() shared = {'opt': opt, 'dict': mdict} agent = TorchAgent(opt, shared) vec_observations = [agent.vectorize(obs) for obs in observations] batch = agent.batchify(vec_observations) self.assertTrue(batch.text_vec is not None, "Missing 'text_vecs' field.") self.assertTrue( batch.text_vec.numpy().tolist() == [[7, 8, 9], [7, 8, 9]], "Incorrectly vectorized text field of obs_batch.") self.assertTrue(batch.label_vec is not None, "Missing 'label_vec' field.") self.assertTrue( batch.label_vec.numpy().tolist() == [[ mdict.START_IDX, 7, 8, 9, mdict.END_IDX ], [mdict.START_IDX, 7, 8, 9, mdict.END_IDX]], "Incorrectly vectorized text field of obs_batch.") self.assertTrue( batch.labels == ["Paint on a canvas.", "Paint on a canvas."], "Doesn't return correct labels: " + str(batch.labels)) true_i = [0, 3] self.assertTrue( all(batch.valid_indices[i] == true_i[i] for i in range(2)), "Returns incorrect indices of valid observations.") observations = [] observations.append({ "text": "What is a painting?", "eval_labels": ["Paint on a canvas."] }) observations.append({}) observations.append({}) observations.append({ "text": "What is a painting?", "eval_labels": ["Paint on a canvas."] }) observations.append({}) observations.append({}) vec_observations = [agent.vectorize(obs) for obs in observations] batch = agent.batchify(vec_observations) self.assertTrue(batch.label_vec is not None, "Missing \'eval_label_vec\' field.") self.assertTrue( batch.label_vec.numpy().tolist() == [[ mdict.START_IDX, 7, 8, 9, mdict.END_IDX ], [mdict.START_IDX, 7, 8, 9, mdict.END_IDX]], "Incorrectly vectorized text field of obs_batch.") batch_reply = [{} for i in range(6)] predictions = ["Oil on a canvas.", "Oil on a canvas."] output = Output(predictions, None) expected_unmapped = batch_reply.copy() expected_unmapped[0]["text"] = "Oil on a canvas." expected_unmapped[3]["text"] = "Oil on a canvas." self.assertTrue( agent.match_batch(batch_reply, batch.valid_indices, output) == expected_unmapped, "Unmapped predictions do not match expected results.")