def setup_args(parser=None): if parser is None: parser = ParlaiParser(True, True) train = parser.add_argument_group('Training Loop Arguments') train.add_argument('-et', '--evaltask', help=('task to use for valid/test (defaults to the ' 'one used for training if not set)')) train.add_argument('--display-examples', type='bool', default=False) train.add_argument('-eps', '--num-epochs', type=float, default=-1) train.add_argument('-ttim', '--max-train-time', type=float, default=-1) train.add_argument('-ltim', '--log-every-n-secs', type=float, default=2) train.add_argument('-vtim', '--validation-every-n-secs', type=float, default=-1, help='Validate every n seconds. Whenever the the best ' 'validation metric is found, saves the model to ' 'the model_file path if set.') train.add_argument('-stim', '--save-every-n-secs', type=float, default=-1, help='Saves the model to model_file.checkpoint after ' 'every n seconds (default -1, never).') train.add_argument('-sval', '--save-after-valid', type='bool', default=False, help='Saves the model to model_file.checkpoint after ' 'every validation (default True).') train.add_argument('-vme', '--validation-max-exs', type=int, default=-1, help='max examples to use during validation (default ' '-1 uses all)') train.add_argument('-vp', '--validation-patience', type=int, default=10, help=('number of iterations of validation where result' ' does not improve before we stop training')) train.add_argument('-vmt', '--validation-metric', default='accuracy', help='key into report table for selecting best ' 'validation') train.add_argument('-vmm', '--validation-metric-mode', default='max', type=str, choices=['max', 'min'], help='how to optimize validation metric (max or min)') train.add_argument('-vcut', '--validation-cutoff', type=float, default=1.0, help='value at which training will stop if exceeded by ' 'training metric') train.add_argument('-dbf', '--dict-build-first', type='bool', default=True, help='build dictionary first before training agent') train.add_argument('-lfc', '--load-from-checkpoint', type='bool', default=False, help='load model from checkpoint if available') TensorboardLogger.add_cmdline_args(parser) parser = setup_dict_args(parser) return parser
def setup_args(parser=None) -> ParlaiParser: """ Build the ParlAI parser, adding command line args if necessary. :param ParlaiParser parser: Preexisting parser to append options to. Will be created if needed. :returns: the ParlaiParser with CLI options added. """ if parser is None: parser = ParlaiParser(True, True, 'Train a model') train = parser.add_argument_group('Training Loop Arguments') train.add_argument( '-et', '--evaltask', help='task to use for valid/test (defaults to the one used for training)', ) train.add_argument( '--final-extra-opt', type=str, default='', help="A '.opt' file that is used for final eval. Useful for setting skip-generation to false. 'datatype' must be included as part of the opt.", ) train.add_argument( '--eval-batchsize', type=int, hidden=True, help='Eval time batch size (defaults to same as -bs)', ) train.add_argument( '--eval-dynamic-batching', # FIXME: see https://github.com/facebookresearch/ParlAI/issues/3367 default=None, type='nonestr', choices={None, 'off', 'full', 'batchsort'}, help=( 'Set dynamic batching at evaluation time. Set to off for ' 'train-only dynamic batching. Set to none (default) to use same ' 'setting as --dynamic-batching.' ), ) train.add_argument( '--num-workers', default=0, type=int, help='Number of background workers (training only)', ) train.add_argument('--display-examples', type='bool', default=False, hidden=True) train.add_argument('-eps', '--num-epochs', type=float, default=-1) train.add_argument('-ttim', '--max-train-time', type=float, default=-1) train.add_argument( '-tstep', '--max-train-steps', '--max-lr-steps', type=int, default=-1, help='End training after n model updates', ) train.add_argument('-ltim', '--log-every-n-secs', type=float, default=-1) train.add_argument( '-lstep', '--log-every-n-steps', type=int, default=50, help='Log every n training steps', ) train.add_argument( '-vtim', '--validation-every-n-secs', type=float, default=-1, help='Validate every n seconds. Saves model to model_file ' '(if set) whenever best val metric is found', ) train.add_argument( '-vstep', '--validation-every-n-steps', type=int, default=-1, help='Validate every n training steps. Saves model to model_file ' '(if set) whenever best val metric is found', ) train.add_argument( '-stim', '--save-every-n-secs', type=float, default=-1, help='Saves the model to model_file.checkpoint after ' 'every n seconds (default -1, never).', ) train.add_argument( '-sval', '--save-after-valid', type='bool', default=False, help='Saves the model to model_file.checkpoint after ' 'every validation (default %(default)s).', ) train.add_argument( '-veps', '--validation-every-n-epochs', type=float, default=-1, help='Validate every n epochs. Saves model to model_file ' '(if set) whenever best val metric is found', ) train.add_argument( '-vme', '--validation-max-exs', type=int, default=-1, hidden=True, help='max examples to use during validation (default -1 uses all)', ) train.add_argument( '--short-final-eval', default=False, hidden=True, type='bool', help='If true, obeys --validation-max-exs in the final ' 'validation and test evaluations.', ) train.add_argument( '-vp', '--validation-patience', type=int, default=10, help=( 'number of iterations of validation where result' ' does not improve before we stop training' ), ) train.add_argument( '-vmt', '--validation-metric', default='accuracy', help='key into report table for selecting best validation', ) train.add_argument( '-vmm', '--validation-metric-mode', type=str, choices=['max', 'min'], help='the direction in which to optimize the validation metric, i.e. maximize or minimize', ) train.add_argument( '-vcut', '--validation-cutoff', type=float, default=1.0, hidden=True, help='value at which training will stop if exceeded by metric', ) train.add_argument( '-lfc', '--load-from-checkpoint', type='bool', default=True, hidden=True, help='load model from checkpoint if available', ) train.add_argument( '-vshare', '--validation-share-agent', default=False, hidden=True, help='use a shared copy of the agent for validation. ' 'this will eventually default to True, but ' 'currently defaults to False.', ) train.add_argument( '-mcs', '--metrics', type=str, default='default', help='list of metrics to show/compute, e.g. all, default,' 'or give a list split by , like ' 'ppl,f1,accuracy,hits@1,rouge,bleu' 'the rouge metrics will be computed as rouge-1, rouge-2 and rouge-l', ) train.add_argument( '-micro', '--aggregate-micro', type='bool', default=False, help='Report micro-averaged metrics instead of macro averaged metrics.', recommended=False, ) train.add_argument( '--world-logs', type=str, default='', help='Saves a jsonl file of the world logs.' 'Set to the empty string to not save at all.', ) train.add_argument( '--save-format', type=str, default='conversations', choices=['conversations', 'parlai'], ) WorldLogger.add_cmdline_args(parser, partial_opt=None) TensorboardLogger.add_cmdline_args(parser, partial_opt=None) WandbLogger.add_cmdline_args(parser, partial_opt=None) parser = setup_dict_args(parser) return parser
def setup_args(parser=None) -> ParlaiParser: """ Build the ParlAI parser, adding command line args if necessary. :param ParlaiParser parser: Preexisting parser to append options to. Will be created if needed. :returns: the ParlaiParser with CLI options added. """ if parser is None: parser = ParlaiParser(True, True, 'Train a model') train = parser.add_argument_group('Training Loop Arguments') train.add_argument( '-et', '--evaltask', help= 'task to use for valid/test (defaults to the one used for training)', ) train.add_argument( '--eval-batchsize', type=int, hidden=True, help='Eval time batch size (defaults to same as -bs)', ) train.add_argument('--display-examples', type='bool', default=False, hidden=True) train.add_argument('-eps', '--num-epochs', type=float, default=-1) train.add_argument('-ttim', '--max-train-time', type=float, default=-1) train.add_argument('-ltim', '--log-every-n-secs', type=float, default=10) train.add_argument( '-vtim', '--validation-every-n-secs', type=float, default=-1, help='Validate every n seconds. Saves model to model_file ' '(if set) whenever best val metric is found', ) train.add_argument( '-stim', '--save-every-n-secs', type=float, default=-1, help='Saves the model to model_file.checkpoint after ' 'every n seconds (default -1, never).', ) train.add_argument( '-sval', '--save-after-valid', type='bool', default=False, help='Saves the model to model_file.checkpoint after ' 'every validation (default %(default)s).', ) train.add_argument( '-veps', '--validation-every-n-epochs', type=float, default=-1, help='Validate every n epochs. Saves model to model_file ' '(if set) whenever best val metric is found', ) train.add_argument( '-vme', '--validation-max-exs', type=int, default=-1, hidden=True, help='max examples to use during validation (default -1 uses all)', ) train.add_argument( '--short-final-eval', default=False, hidden=True, type='bool', help='If true, obeys --validation-max-exs in the final ' 'validation and test evaluations.', ) train.add_argument( '-vp', '--validation-patience', type=int, default=10, help=('number of iterations of validation where result' ' does not improve before we stop training'), ) train.add_argument( '-vmt', '--validation-metric', default='accuracy', help='key into report table for selecting best validation', ) train.add_argument( '-vmm', '--validation-metric-mode', type=str, choices=['max', 'min'], help='how to optimize validation metric (max or min)', ) train.add_argument( '-vcut', '--validation-cutoff', type=float, default=1.0, hidden=True, help='value at which training will stop if exceeded by metric', ) train.add_argument( '-lfc', '--load-from-checkpoint', type='bool', default=True, hidden=True, help='load model from checkpoint if available', ) train.add_argument( '-vshare', '--validation-share-agent', default=False, hidden=True, help='use a shared copy of the agent for validation. ' 'this will eventually default to True, but ' 'currently defaults to False.', ) train.add_argument( '-mcs', '--metrics', type=str, default='default', help='list of metrics to show/compute, e.g. all, default,' 'or give a list split by , like ' 'ppl,f1,accuracy,hits@1,rouge,bleu' 'the rouge metrics will be computed as rouge-1, rouge-2 and rouge-l', ) train.add_argument( '-micro', '--aggregate-micro', type='bool', default=False, help='Report micro-averaged metrics instead of macro averaged metrics.', recommended=False, ) TensorboardLogger.add_cmdline_args(parser) parser = setup_dict_args(parser) return parser
def setup_args(parser=None): if parser is None: parser = ParlaiParser(True, True, 'Train a model') parser.add_pytorch_datateacher_args() train = parser.add_argument_group('Training Loop Arguments') train.add_argument('-et', '--evaltask', help=('task to use for valid/test (defaults to the ' 'one used for training if not set)')) train.add_argument('--eval-batchsize', type=int, hidden=True, help='Eval time batch size (defaults to same as -bs)') train.add_argument('--display-examples', type='bool', default=False, hidden=True) train.add_argument('-eps', '--num-epochs', type=float, default=-1) train.add_argument('-ttim', '--max-train-time', type=float, default=-1) train.add_argument('-ltim', '--log-every-n-secs', type=float, default=2) train.add_argument( '-vtim', '--validation-every-n-secs', type=float, default=-1, help='Validate every n seconds. Saves model to model_file ' '(if set) whenever best val metric is found') train.add_argument('-stim', '--save-every-n-secs', type=float, default=-1, help='Saves the model to model_file.checkpoint after ' 'every n seconds (default -1, never).') train.add_argument('-sval', '--save-after-valid', type='bool', default=False, help='Saves the model to model_file.checkpoint after ' 'every validation (default %(default)s).') train.add_argument( '-veps', '--validation-every-n-epochs', type=float, default=-1, help='Validate every n epochs. Saves model to model_file ' '(if set) whenever best val metric is found') train.add_argument('-vme', '--validation-max-exs', type=int, default=-1, hidden=True, help='max examples to use during validation (default ' '-1 uses all)') train.add_argument('--short-final-eval', default=False, hidden=True, type='bool', help='If true, obeys --validation-max-exs in the final ' 'validation and test evaluations.') train.add_argument('-vp', '--validation-patience', type=int, default=10, help=('number of iterations of validation where result' ' does not improve before we stop training')) train.add_argument('-vmt', '--validation-metric', default='accuracy', help='key into report table for selecting best ' 'validation') train.add_argument('-vmm', '--validation-metric-mode', type=str, choices=['max', 'min'], help='how to optimize validation metric (max or min)') train.add_argument('-vcut', '--validation-cutoff', type=float, default=1.0, hidden=True, help='value at which training will stop if exceeded by ' 'training metric') train.add_argument('-dbf', '--dict-build-first', hidden=True, type='bool', default=True, help='build dictionary first before training agent') train.add_argument('-lfc', '--load-from-checkpoint', type='bool', default=False, hidden=True, help='load model from checkpoint if available') train.add_argument('-vshare', '--validation-share-agent', default=False, hidden=True, help='use a shared copy of the agent for validation. ' 'this will eventually default to True, but ' 'currently defaults to False.') TensorboardLogger.add_cmdline_args(parser) parser = setup_dict_args(parser) return parser
def setup_args(): """ Use create test env setting :return: opt """ parser = setup_dict_args() # 'dot', 'bilinear', 'cos' score_method = 'dot' # 'scale', 'sigmoid', 'none', 'scalesigmoid' score_normalize = 'scale' # 'sum', 'inc', 'soft' score_train = 'soft' batch_size = 2 validation_batch_size = 8 candidates_num = 2 val_candidates_num = 32 sent_hidden_size = -1 dialog_hidden_size = -1 weight_decay = 1e-5 learning_rate = 1e-4 bert_learning_rate = 6.25e-6 dropout = 0.1 input_dropout = 0 bidirectional = False sparse = 1e-2 # {'none', 'general'} attention = 'general' # {'cross_entropy', 'margin'} criterion = 'margin' marginloss_margin = 0.5 optim_method = 'adam' validation_every_n_secs = 2400 log_every_n_secs = 20 num_epoch = 50 validation_exs = 5000 lr_impatience = 5 numlayers = 1 cri = criterion if criterion == 'margin': cri += str(marginloss_margin) exp_name = VERSION task_name = setup_task() parser.set_defaults( exp=exp_name, # name for experiment task=task_name, batchsize=batch_size, validation_batch_size=validation_batch_size, dict_include_valid=True, dict_tokenizer='split', dict_nulltoken=SpecialToken.pad, dict_starttoken=SpecialToken.start, dict_endtoken=SpecialToken.end, dict_unktoken=SpecialToken.unk, datatype='train', # model configuration model='agents.receiver.receiver:ReceiverAgent', model_file=os.path.join(os.path.abspath(RECEIVER_DIR), '{}.model'.format(exp_name)), init_transmitter=os.path.join(os.path.abspath(RECEIVER_DIR), '{}.model'.format(exp_name)), # validation configuration validation_every_n_secs=validation_every_n_secs, # in default, this can be chosen from hit@k, f1, bleu, accuracy # here only accuracy and hit@k make sense validation_metric='hits@1', validation_metric_mode='max', # Stop training when the metrics cannot meet the best result for # validation_patience times. validation_patience=15, log_every_n_secs=log_every_n_secs, # device configuration gpu=0, tensorboard_log=True, tensorboard_tag='exp', tensorboard_metrics='loss,hits@1,hits@5,hits@10,lr', # teacher negative sampling num negative_sample_num=candidates_num - 1, valid_negative_sample_num=val_candidates_num - 1, # can be set as all, which will show all # the metrics in training metrics='loss,hits@1,lr', tensorboard_comment='', # add to the tensorboard output num_epochs=num_epoch, # total number of epochs max_train_time=60000, save_every_n_secs=1200, validation_every_n_epochs=1, # enable this when debugging display_examples=False, # limitation of the number of examples if exs > 0 validation_max_exs=validation_exs, # when the accuracy meet this value, stop training validation_cutoff=0.999, no_cuda=False, # {'random', 'glove', 'glove_fixed'} embedding_type="glove_fixed", embeddingsize=300, sent_hiddensize=sent_hidden_size, dialog_embedding_size=dialog_hidden_size, learningrate=learning_rate, weight_decay=weight_decay, bert_learning_rate=bert_learning_rate, bidirectional=bidirectional, dropout=dropout, input_dropout=input_dropout, attention=attention, criterion=criterion, lr_impatience=lr_impatience, marginloss_margin=marginloss_margin, optimizer=optim_method, numlayers=numlayers, sparse=sparse, score_train=score_train, score_normalize=score_normalize, score_method=score_method) opt = parser.parse_args([]) # Override the setting which is saved in the .opt file opt['override'] = dict( # Whether load model from the latest checkpoint load_from_checkpoint=True) return opt