def setup_args(parser=None):
if parser is None:
parser = ParlaiParser(True, True)
train = parser.add_argument_group('Training Loop Arguments')
train.add_argument('-et', '--evaltask',
help=('task to use for valid/test (defaults to the '
'one used for training if not set)'))
train.add_argument('--display-examples', type='bool', default=False)
train.add_argument('-eps', '--num-epochs', type=float, default=-1)
train.add_argument('-ttim', '--max-train-time',
type=float, default=-1)
train.add_argument('-ltim', '--log-every-n-secs',
type=float, default=2)
train.add_argument('-vtim', '--validation-every-n-secs',
type=float, default=-1,
help='Validate every n seconds. Whenever the the best '
'validation metric is found, saves the model to '
'the model_file path if set.')
train.add_argument('-stim', '--save-every-n-secs',
type=float, default=-1,
help='Saves the model to model_file.checkpoint after '
'every n seconds (default -1, never).')
train.add_argument('-sval', '--save-after-valid', type='bool',
default=False,
help='Saves the model to model_file.checkpoint after '
'every validation (default True).')
train.add_argument('-vme', '--validation-max-exs',
type=int, default=-1,
help='max examples to use during validation (default '
'-1 uses all)')
train.add_argument('-vp', '--validation-patience',
type=int, default=10,
help=('number of iterations of validation where result'
' does not improve before we stop training'))
train.add_argument('-vmt', '--validation-metric', default='accuracy',
help='key into report table for selecting best '
'validation')
train.add_argument('-vmm', '--validation-metric-mode', default='max',
type=str, choices=['max', 'min'],
help='how to optimize validation metric (max or min)')
train.add_argument('-vcut', '--validation-cutoff',
type=float, default=1.0,
help='value at which training will stop if exceeded by '
'training metric')
train.add_argument('-dbf', '--dict-build-first',
type='bool', default=True,
help='build dictionary first before training agent')
train.add_argument('-lfc', '--load-from-checkpoint',
type='bool', default=False,
help='load model from checkpoint if available')
TensorboardLogger.add_cmdline_args(parser)
parser = setup_dict_args(parser)
return parser
def setup_args(parser=None) -> ParlaiParser:
"""
Build the ParlAI parser, adding command line args if necessary.
:param ParlaiParser parser:
Preexisting parser to append options to. Will be created if needed.
:returns:
the ParlaiParser with CLI options added.
"""
if parser is None:
parser = ParlaiParser(True, True, 'Train a model')
train = parser.add_argument_group('Training Loop Arguments')
train.add_argument(
'-et',
'--evaltask',
help='task to use for valid/test (defaults to the one used for training)',
)
train.add_argument(
'--final-extra-opt',
type=str,
default='',
help="A '.opt' file that is used for final eval. Useful for setting skip-generation to false. 'datatype' must be included as part of the opt.",
)
train.add_argument(
'--eval-batchsize',
type=int,
hidden=True,
help='Eval time batch size (defaults to same as -bs)',
)
train.add_argument(
'--eval-dynamic-batching', # FIXME: see https://github.com/facebookresearch/ParlAI/issues/3367
default=None,
type='nonestr',
choices={None, 'off', 'full', 'batchsort'},
help=(
'Set dynamic batching at evaluation time. Set to off for '
'train-only dynamic batching. Set to none (default) to use same '
'setting as --dynamic-batching.'
),
)
train.add_argument(
'--num-workers',
default=0,
type=int,
help='Number of background workers (training only)',
)
train.add_argument('--display-examples', type='bool', default=False, hidden=True)
train.add_argument('-eps', '--num-epochs', type=float, default=-1)
train.add_argument('-ttim', '--max-train-time', type=float, default=-1)
train.add_argument(
'-tstep',
'--max-train-steps',
'--max-lr-steps',
type=int,
default=-1,
help='End training after n model updates',
)
train.add_argument('-ltim', '--log-every-n-secs', type=float, default=-1)
train.add_argument(
'-lstep',
'--log-every-n-steps',
type=int,
default=50,
help='Log every n training steps',
)
train.add_argument(
'-vtim',
'--validation-every-n-secs',
type=float,
default=-1,
help='Validate every n seconds. Saves model to model_file '
'(if set) whenever best val metric is found',
)
train.add_argument(
'-vstep',
'--validation-every-n-steps',
type=int,
default=-1,
help='Validate every n training steps. Saves model to model_file '
'(if set) whenever best val metric is found',
)
train.add_argument(
'-stim',
'--save-every-n-secs',
type=float,
default=-1,
help='Saves the model to model_file.checkpoint after '
'every n seconds (default -1, never).',
)
train.add_argument(
'-sval',
'--save-after-valid',
type='bool',
default=False,
help='Saves the model to model_file.checkpoint after '
'every validation (default %(default)s).',
)
train.add_argument(
'-veps',
'--validation-every-n-epochs',
type=float,
default=-1,
help='Validate every n epochs. Saves model to model_file '
'(if set) whenever best val metric is found',
)
train.add_argument(
'-vme',
'--validation-max-exs',
type=int,
default=-1,
hidden=True,
help='max examples to use during validation (default -1 uses all)',
)
train.add_argument(
'--short-final-eval',
default=False,
hidden=True,
type='bool',
help='If true, obeys --validation-max-exs in the final '
'validation and test evaluations.',
)
train.add_argument(
'-vp',
'--validation-patience',
type=int,
default=10,
help=(
'number of iterations of validation where result'
' does not improve before we stop training'
),
)
train.add_argument(
'-vmt',
'--validation-metric',
default='accuracy',
help='key into report table for selecting best validation',
)
train.add_argument(
'-vmm',
'--validation-metric-mode',
type=str,
choices=['max', 'min'],
help='the direction in which to optimize the validation metric, i.e. maximize or minimize',
)
train.add_argument(
'-vcut',
'--validation-cutoff',
type=float,
default=1.0,
hidden=True,
help='value at which training will stop if exceeded by metric',
)
train.add_argument(
'-lfc',
'--load-from-checkpoint',
type='bool',
default=True,
hidden=True,
help='load model from checkpoint if available',
)
train.add_argument(
'-vshare',
'--validation-share-agent',
default=False,
hidden=True,
help='use a shared copy of the agent for validation. '
'this will eventually default to True, but '
'currently defaults to False.',
)
train.add_argument(
'-mcs',
'--metrics',
type=str,
default='default',
help='list of metrics to show/compute, e.g. all, default,'
'or give a list split by , like '
'ppl,f1,accuracy,hits@1,rouge,bleu'
'the rouge metrics will be computed as rouge-1, rouge-2 and rouge-l',
)
train.add_argument(
'-micro',
'--aggregate-micro',
type='bool',
default=False,
help='Report micro-averaged metrics instead of macro averaged metrics.',
recommended=False,
)
train.add_argument(
'--world-logs',
type=str,
default='',
help='Saves a jsonl file of the world logs.'
'Set to the empty string to not save at all.',
)
train.add_argument(
'--save-format',
type=str,
default='conversations',
choices=['conversations', 'parlai'],
)
WorldLogger.add_cmdline_args(parser, partial_opt=None)
TensorboardLogger.add_cmdline_args(parser, partial_opt=None)
WandbLogger.add_cmdline_args(parser, partial_opt=None)
parser = setup_dict_args(parser)
return parser
def setup_args(parser=None) -> ParlaiParser:
"""
Build the ParlAI parser, adding command line args if necessary.
:param ParlaiParser parser:
Preexisting parser to append options to. Will be created if needed.
:returns:
the ParlaiParser with CLI options added.
"""
if parser is None:
parser = ParlaiParser(True, True, 'Train a model')
train = parser.add_argument_group('Training Loop Arguments')
train.add_argument(
'-et',
'--evaltask',
help=
'task to use for valid/test (defaults to the one used for training)',
)
train.add_argument(
'--eval-batchsize',
type=int,
hidden=True,
help='Eval time batch size (defaults to same as -bs)',
)
train.add_argument('--display-examples',
type='bool',
default=False,
hidden=True)
train.add_argument('-eps', '--num-epochs', type=float, default=-1)
train.add_argument('-ttim', '--max-train-time', type=float, default=-1)
train.add_argument('-ltim', '--log-every-n-secs', type=float, default=10)
train.add_argument(
'-vtim',
'--validation-every-n-secs',
type=float,
default=-1,
help='Validate every n seconds. Saves model to model_file '
'(if set) whenever best val metric is found',
)
train.add_argument(
'-stim',
'--save-every-n-secs',
type=float,
default=-1,
help='Saves the model to model_file.checkpoint after '
'every n seconds (default -1, never).',
)
train.add_argument(
'-sval',
'--save-after-valid',
type='bool',
default=False,
help='Saves the model to model_file.checkpoint after '
'every validation (default %(default)s).',
)
train.add_argument(
'-veps',
'--validation-every-n-epochs',
type=float,
default=-1,
help='Validate every n epochs. Saves model to model_file '
'(if set) whenever best val metric is found',
)
train.add_argument(
'-vme',
'--validation-max-exs',
type=int,
default=-1,
hidden=True,
help='max examples to use during validation (default -1 uses all)',
)
train.add_argument(
'--short-final-eval',
default=False,
hidden=True,
type='bool',
help='If true, obeys --validation-max-exs in the final '
'validation and test evaluations.',
)
train.add_argument(
'-vp',
'--validation-patience',
type=int,
default=10,
help=('number of iterations of validation where result'
' does not improve before we stop training'),
)
train.add_argument(
'-vmt',
'--validation-metric',
default='accuracy',
help='key into report table for selecting best validation',
)
train.add_argument(
'-vmm',
'--validation-metric-mode',
type=str,
choices=['max', 'min'],
help='how to optimize validation metric (max or min)',
)
train.add_argument(
'-vcut',
'--validation-cutoff',
type=float,
default=1.0,
hidden=True,
help='value at which training will stop if exceeded by metric',
)
train.add_argument(
'-lfc',
'--load-from-checkpoint',
type='bool',
default=True,
hidden=True,
help='load model from checkpoint if available',
)
train.add_argument(
'-vshare',
'--validation-share-agent',
default=False,
hidden=True,
help='use a shared copy of the agent for validation. '
'this will eventually default to True, but '
'currently defaults to False.',
)
train.add_argument(
'-mcs',
'--metrics',
type=str,
default='default',
help='list of metrics to show/compute, e.g. all, default,'
'or give a list split by , like '
'ppl,f1,accuracy,hits@1,rouge,bleu'
'the rouge metrics will be computed as rouge-1, rouge-2 and rouge-l',
)
train.add_argument(
'-micro',
'--aggregate-micro',
type='bool',
default=False,
help='Report micro-averaged metrics instead of macro averaged metrics.',
recommended=False,
)
TensorboardLogger.add_cmdline_args(parser)
parser = setup_dict_args(parser)
return parser
def setup_args(parser=None):
if parser is None:
parser = ParlaiParser(True, True, 'Train a model')
parser.add_pytorch_datateacher_args()
train = parser.add_argument_group('Training Loop Arguments')
train.add_argument('-et',
'--evaltask',
help=('task to use for valid/test (defaults to the '
'one used for training if not set)'))
train.add_argument('--eval-batchsize',
type=int,
hidden=True,
help='Eval time batch size (defaults to same as -bs)')
train.add_argument('--display-examples',
type='bool',
default=False,
hidden=True)
train.add_argument('-eps', '--num-epochs', type=float, default=-1)
train.add_argument('-ttim', '--max-train-time', type=float, default=-1)
train.add_argument('-ltim', '--log-every-n-secs', type=float, default=2)
train.add_argument(
'-vtim',
'--validation-every-n-secs',
type=float,
default=-1,
help='Validate every n seconds. Saves model to model_file '
'(if set) whenever best val metric is found')
train.add_argument('-stim',
'--save-every-n-secs',
type=float,
default=-1,
help='Saves the model to model_file.checkpoint after '
'every n seconds (default -1, never).')
train.add_argument('-sval',
'--save-after-valid',
type='bool',
default=False,
help='Saves the model to model_file.checkpoint after '
'every validation (default %(default)s).')
train.add_argument(
'-veps',
'--validation-every-n-epochs',
type=float,
default=-1,
help='Validate every n epochs. Saves model to model_file '
'(if set) whenever best val metric is found')
train.add_argument('-vme',
'--validation-max-exs',
type=int,
default=-1,
hidden=True,
help='max examples to use during validation (default '
'-1 uses all)')
train.add_argument('--short-final-eval',
default=False,
hidden=True,
type='bool',
help='If true, obeys --validation-max-exs in the final '
'validation and test evaluations.')
train.add_argument('-vp',
'--validation-patience',
type=int,
default=10,
help=('number of iterations of validation where result'
' does not improve before we stop training'))
train.add_argument('-vmt',
'--validation-metric',
default='accuracy',
help='key into report table for selecting best '
'validation')
train.add_argument('-vmm',
'--validation-metric-mode',
type=str,
choices=['max', 'min'],
help='how to optimize validation metric (max or min)')
train.add_argument('-vcut',
'--validation-cutoff',
type=float,
default=1.0,
hidden=True,
help='value at which training will stop if exceeded by '
'training metric')
train.add_argument('-dbf',
'--dict-build-first',
hidden=True,
type='bool',
default=True,
help='build dictionary first before training agent')
train.add_argument('-lfc',
'--load-from-checkpoint',
type='bool',
default=False,
hidden=True,
help='load model from checkpoint if available')
train.add_argument('-vshare',
'--validation-share-agent',
default=False,
hidden=True,
help='use a shared copy of the agent for validation. '
'this will eventually default to True, but '
'currently defaults to False.')
TensorboardLogger.add_cmdline_args(parser)
parser = setup_dict_args(parser)
return parser
def setup_args():
"""
Use create test env setting
:return: opt
"""
parser = setup_dict_args()
# 'dot', 'bilinear', 'cos'
score_method = 'dot'
# 'scale', 'sigmoid', 'none', 'scalesigmoid'
score_normalize = 'scale'
# 'sum', 'inc', 'soft'
score_train = 'soft'
batch_size = 2
validation_batch_size = 8
candidates_num = 2
val_candidates_num = 32
sent_hidden_size = -1
dialog_hidden_size = -1
weight_decay = 1e-5
learning_rate = 1e-4
bert_learning_rate = 6.25e-6
dropout = 0.1
input_dropout = 0
bidirectional = False
sparse = 1e-2
# {'none', 'general'}
attention = 'general'
# {'cross_entropy', 'margin'}
criterion = 'margin'
marginloss_margin = 0.5
optim_method = 'adam'
validation_every_n_secs = 2400
log_every_n_secs = 20
num_epoch = 50
validation_exs = 5000
lr_impatience = 5
numlayers = 1
cri = criterion
if criterion == 'margin':
cri += str(marginloss_margin)
exp_name = VERSION
task_name = setup_task()
parser.set_defaults(
exp=exp_name, # name for experiment
task=task_name,
batchsize=batch_size,
validation_batch_size=validation_batch_size,
dict_include_valid=True,
dict_tokenizer='split',
dict_nulltoken=SpecialToken.pad,
dict_starttoken=SpecialToken.start,
dict_endtoken=SpecialToken.end,
dict_unktoken=SpecialToken.unk,
datatype='train',
# model configuration
model='agents.receiver.receiver:ReceiverAgent',
model_file=os.path.join(os.path.abspath(RECEIVER_DIR),
'{}.model'.format(exp_name)),
init_transmitter=os.path.join(os.path.abspath(RECEIVER_DIR),
'{}.model'.format(exp_name)),
# validation configuration
validation_every_n_secs=validation_every_n_secs,
# in default, this can be chosen from hit@k, f1, bleu, accuracy
# here only accuracy and hit@k make sense
validation_metric='hits@1',
validation_metric_mode='max',
# Stop training when the metrics cannot meet the best result for
# validation_patience times.
validation_patience=15,
log_every_n_secs=log_every_n_secs,
# device configuration
gpu=0,
tensorboard_log=True,
tensorboard_tag='exp',
tensorboard_metrics='loss,hits@1,hits@5,hits@10,lr',
# teacher negative sampling num
negative_sample_num=candidates_num - 1,
valid_negative_sample_num=val_candidates_num - 1,
# can be set as all, which will show all
# the metrics in training
metrics='loss,hits@1,lr',
tensorboard_comment='', # add to the tensorboard output
num_epochs=num_epoch, # total number of epochs
max_train_time=60000,
save_every_n_secs=1200,
validation_every_n_epochs=1,
# enable this when debugging
display_examples=False,
# limitation of the number of examples if exs > 0
validation_max_exs=validation_exs,
# when the accuracy meet this value, stop training
validation_cutoff=0.999,
no_cuda=False,
# {'random', 'glove', 'glove_fixed'}
embedding_type="glove_fixed",
embeddingsize=300,
sent_hiddensize=sent_hidden_size,
dialog_embedding_size=dialog_hidden_size,
learningrate=learning_rate,
weight_decay=weight_decay,
bert_learning_rate=bert_learning_rate,
bidirectional=bidirectional,
dropout=dropout,
input_dropout=input_dropout,
attention=attention,
criterion=criterion,
lr_impatience=lr_impatience,
marginloss_margin=marginloss_margin,
optimizer=optim_method,
numlayers=numlayers,
sparse=sparse,
score_train=score_train,
score_normalize=score_normalize,
score_method=score_method)
opt = parser.parse_args([])
# Override the setting which is saved in the .opt file
opt['override'] = dict(
# Whether load model from the latest checkpoint
load_from_checkpoint=True)
return opt
