def setup_args():
parser = ParlaiParser(True, True)
train = parser.add_argument_group('Training Loop Arguments')
train.add_argument('-et', '--evaltask',
help=('task to use for valid/test (defaults to the '
'one used for training if not set)'))
train.add_argument('-d', '--display-examples',
type='bool', default=False)
train.add_argument('-e', '--num-epochs', type=float, default=-1)
train.add_argument('-ttim', '--max-train-time',
type=float, default=-1)
train.add_argument('-ltim', '--log-every-n-secs',
type=float, default=2)
train.add_argument('-vtim', '--validation-every-n-secs',
type=float, default=-1)
train.add_argument('-vme', '--validation-max-exs',
type=int, default=-1,
help='max examples to use during validation (default '
'-1 uses all)')
train.add_argument('-vp', '--validation-patience',
type=int, default=10,
help=('number of iterations of validation where result'
' does not improve before we stop training'))
train.add_argument('-vmt', '--validation-metric', default='accuracy',
help='key into report table for selecting best '
'validation')
train.add_argument('-dbf', '--dict-build-first',
type='bool', default=True,
help='build dictionary first before training agent')
return parser
def add_cmdline_args(
cls, parser: ParlaiParser, partial_opt: Optional[Opt] = None
) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt)
parser = parser.add_argument_group('WoW Interactive World Args')
parser.add_argument(
'--print-checked-sentence',
type='bool',
default=True,
help='Print sentence that the model checks.',
)
return parser
def add_cmdline_args(
cls, parser: ParlaiParser, partial_opt: Optional[Opt] = None
) -> ParlaiParser:
"""
Add command-line arguments specifically for this agent.
"""
agent = parser.add_argument_group('Transformer Arguments')
add_common_cmdline_args(agent)
cls.dictionary_class().add_cmdline_args(parser, partial_opt=partial_opt)
super().add_cmdline_args(parser, partial_opt=partial_opt)
return agent
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt)
agent = parser.add_argument_group('SaFeRDialogues options')
agent.add_argument(
'--recovery',
type=bool,
default=True,
help="Whether or not to include the recovery utterance",
)
return parser
def add_cmdline_args(
cls, parser: ParlaiParser, partial_opt: Optional[Opt] = None
) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt)
agent = parser.add_argument_group('Session Level NormalizedTeacher arguments')
agent.add_argument(
'--is-convai2-session-level',
type=bool,
default=False,
help="whether to flatten the persona lines into a single persona line per speaker",
)
return agent
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
agent = parser.add_argument_group('SQuAD Sentence Teacher Arguments')
agent.add_argument(
'--include-context',
type='bool',
default=False,
help='include context within text instead of as a '
'separate field',
)
return parser
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt)
group = parser.add_argument_group("Natural Questions retrieval")
group.add_argument(
"--normalize-everything",
default=False,
type=bool,
help="Noramlize text + label in training",
)
return parser
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
group = parser.add_argument_group('FixedResponse Arguments')
group.add_argument(
'-fr',
'--fixed-response',
type='nonestr',
default="I don't know.",
help='fixed response the agent always returns',
)
return parser
def setup_args():
"""
Set up args, specifically for the port number.
:return: A parser that parses the port from commandline arguments.
"""
parser = ParlaiParser(False, False)
parser_grp = parser.add_argument_group('Discord Chat')
parser_grp.add_argument(
'--port', default=35496, type=int, help='Port to run the discord chat server'
)
return parser.parse_args()
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt)
agent = parser.add_argument_group('Wizard arguments')
agent.add_argument(
'--ln',
type=str,
default='en',
help='language code. default: en',
)
return parser
def setup_args(parser=None):
if parser is None:
parser = ParlaiParser(True, True)
train = parser.add_argument_group('Training Loop Arguments')
train.add_argument('-et', '--evaltask',
help=('task to use for valid/test (defaults to the '
'one used for training if not set)'))
train.add_argument('--display-examples', type='bool', default=False)
train.add_argument('-eps', '--num-epochs', type=float, default=-1)
train.add_argument('-ttim', '--max-train-time',
type=float, default=-1)
train.add_argument('-ltim', '--log-every-n-secs',
type=float, default=2)
train.add_argument('-vtim', '--validation-every-n-secs',
type=float, default=-1,
help='Validate every n seconds. Whenever the the best '
'validation metric is found, saves the model to '
'the model_file path if set.')
train.add_argument('-stim', '--save-every-n-secs',
type=float, default=-1,
help='Saves the model to model_file.checkpoint after '
'every n seconds (default -1, never).')
train.add_argument('-sval', '--save-after-valid', type='bool',
default=False,
help='Saves the model to model_file.checkpoint after '
'every validation (default True).')
train.add_argument('-vme', '--validation-max-exs',
type=int, default=-1,
help='max examples to use during validation (default '
'-1 uses all)')
train.add_argument('-vp', '--validation-patience',
type=int, default=10,
help=('number of iterations of validation where result'
' does not improve before we stop training'))
train.add_argument('-vmt', '--validation-metric', default='accuracy',
help='key into report table for selecting best '
'validation')
train.add_argument('-vmm', '--validation-metric-mode', default='max',
type=str, choices=['max', 'min'],
help='how to optimize validation metric (max or min)')
train.add_argument('-vcut', '--validation-cutoff',
type=float, default=1.0,
help='value at which training will stop if exceeded by '
'training metric')
train.add_argument('-dbf', '--dict-build-first',
type='bool', default=True,
help='build dictionary first before training agent')
train.add_argument('-lfc', '--load-from-checkpoint',
type='bool', default=False,
help='load model from checkpoint if available')
TensorboardLogger.add_cmdline_args(parser)
parser = setup_dict_args(parser)
return parser
def add_cmdline_args(
cls, parser: ParlaiParser, partial_opt: Optional[Opt] = None
) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt)
parser = parser.add_argument_group('ConvAI2 Interactive World')
parser.add_argument(
'--display-partner-persona',
type='bool',
default=True,
help='Display your partner persona at the end of the chat',
)
return parser
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt)
parser = parser.add_argument_group('LIGHT Interactive World')
parser.add_argument(
'--add-task-string',
type='bool',
default=False,
help='Add _task_speech to text input to model or not',
)
return parser
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt=partial_opt)
agent = parser.add_argument_group('DistillTransformer arguments')
agent.add_argument(
'--copy-teacher-weights',
type='bool',
default=True,
help='Copy weights from the teacher model to the student model',
)
return agent
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
BasePolyencoderAgent.add_cmdline_args(parser, partial_opt)
group = parser.add_argument_group('Return Weights Poly Group')
group.add_argument(
'--top-k',
type=int,
default=100,
help='How many tokens to output when outputting relevant tokens',
)
return parser
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt)
group = parser.add_argument_group('ELI5 Knowledge arguments')
group.add_argument(
'--knowledge',
type='bool',
default=True,
help='Whether to include supporting document knowledge',
)
return parser
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
agent = parser.add_argument_group('World Logging')
agent.add_argument(
'--log-keep-fields',
type=str,
default=KEEP_ALL,
help=
'Fields to keep when logging. Should be a comma separated list',
)
return parser
def setup_args(parser=None):
if parser is None:
parser = ParlaiParser(True, True)
parser = train_args(parser)
profile = parser.add_argument_group('Profiler Arguments')
profile.add_argument('--torch', type='bool', default=False,
help='If true, use the torch profiler. Otherwise use cProfile.')
profile.add_argument('--torch-cuda', type='bool', default=False,
help='If true, use the torch cuda profiler. Otherwise use cProfile.')
profile.add_argument('--debug', type='bool', default=False,
help='If true, enter debugger at end of run.')
return parser
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt=partial_opt)
agent = parser.add_argument_group('IGCResponseOnly Arguments')
agent.add_argument(
'--igc-multi-ref',
type='bool',
default=False,
help='specify true to evaluate on multi-reference labels',
)
return parser
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt)
parser = parser.add_argument_group('Multiturn Safety Teacher Args')
parser.add_argument(
'--single-turn',
type='bool',
default=False,
help='only include the single turn data and not the context info',
)
return parser
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt)
agent = parser.add_argument_group('Corrupt-Example-Arguments')
agent.add_argument(
'--exclude-invalid-data',
type='bool',
default=True,
help='Whether to include corrupt examples in the data',
)
return parser
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt)
group = parser.add_argument_group("Google SGD Out Domain Parser")
group.add_argument(
"--filter-single-goal-episodes",
type=bool,
default=False,
help=
"Filter for only conversations where the original has single goals",
)
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt=None) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt)
arg_group = parser.add_argument_group(
'Jericho World Action Teacher Args')
arg_group.add_argument(
'--include-knowledge-graph',
type='bool',
default=True,
help='Whether to include the knowledge graph.',
)
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt=partial_opt)
agent = parser.add_argument_group('MyCustom Transformer Arguments')
parser.add_argument(
'--decoder-ffn-variants',
type=DecoderFeedForwardVariant,
default=DecoderFeedForwardVariant.ONE,
help='Some variants in the decoder FFN implementation',
)
return agent # type: ignore
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
ChunkTeacher.add_cmdline_args(parser, partial_opt)
agent = parser.add_argument_group('LM Task Arguments')
agent.add_argument(
'--root-dir',
type=str,
default=None,
help="Root directory with generated world logs.",
)
agent.add_argument(
'--validation-data-indices',
type=str,
default='',
help="comma separated list of validation data indices",
)
agent.add_argument(
'--shared-knowledge-entity',
type='bool',
help=
'If True, only include examples with an entity in the knowledge that is also in the label',
default=False,
)
agent.add_argument(
'--min-knowledge-length',
type=int,
help=
'minimum length of the knowledge required to include the example. Default -1 means no min length',
default=5,
)
agent.add_argument(
'--min-knowledge-overlap',
type=float,
help=
'minimum overlap between knowledge and target sentence. Default 0 means no overlap required.',
default=0,
)
agent.add_argument(
'--skip-empty-context',
type=bool,
help='whether to exclude examples with empty context.',
default=False,
)
agent.add_argument(
'--exclude-retrieved-docs',
type='bool',
default=False,
help=
'specify to not include retrieved docs in the episodes. hopefully reduces memory footprint.',
)
return parser
def test_bool(self):
"""
test add_argument(type=bool)
"""
parser = ParlaiParser(True, True)
parser.add_argument('--foo', type=bool)
opt = parser.parse_args(['--foo', 'true'])
assert opt['foo'] is True
opt = parser.parse_args(['--foo', 'False'])
assert opt['foo'] is False
opt = parser.parse_args(['--foo', '0'])
assert opt['foo'] is False
group = parser.add_argument_group('foo container')
group.add_argument('--bar', type=bool)
opt = parser.parse_args(['--bar', 'true'])
assert opt['bar'] is True
opt = parser.parse_args(['--bar', 'False'])
assert opt['bar'] is False
opt = parser.parse_args(['--bar', '0'])
assert opt['bar'] is False
parser = ParlaiParser(True, True)
parser.add_argument('--foo', type='bool')
opt = parser.parse_args(['--foo', 'true'])
assert opt['foo'] is True
opt = parser.parse_args(['--foo', 'False'])
assert opt['foo'] is False
opt = parser.parse_args(['--foo', '0'])
assert opt['foo'] is False
group = parser.add_argument_group('foo container')
group.add_argument('--bar', type='bool')
opt = parser.parse_args(['--bar', 'true'])
assert opt['bar'] is True
opt = parser.parse_args(['--bar', 'False'])
assert opt['bar'] is False
opt = parser.parse_args(['--bar', '0'])
assert opt['bar'] is False
def main():
# Get command line arguments
argparser = ParlaiParser(True, True)
build = argparser.add_argument_group('Data Building Args')
build.add_argument('--datafile',
help=('The file to be loaded, preprocessed, and saved'))
build.add_argument('--pytorch_buildteacher', type=str, default='',
help='Which teacher to use when building the pytorch data')
build.add_argument('--pytorch_preprocess', type='bool', default=True,
help='Whether the agent should preprocess the data while building'
'the pytorch data')
opt = argparser.parse_args()
build_data(opt)
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt=None) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt)
arg_group = parser.add_argument_group('Search Query Teacher')
arg_group.add_argument(
'--only-last-search-query',
type='bool',
default=CONST.ONLY_LAST_QUERY_DEFAULT,
help=
'Whether to include only the last search before sending the response.',
)
return parser
def add_cmdline_args(
cls, parser: ParlaiParser, partial_opt: Optional[Opt] = None
) -> ParlaiParser:
parser = gend_utils.add_common_args(parser)
agent = parser.add_argument_group('New data gender')
agent.add_argument(
'--labels-to-use',
type=str,
default='all',
choices=['all', 'self', 'partner', 'about'],
help='Which labels to use for this teacher',
)
return parser
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt: Optional[Opt] = None) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt)
parser = parser.add_argument_group('MSC Summary Teacher Args')
parser.add_argument(
'--include-last-session',
type=bool,
default=False,
help="whether to include session 4 for valid and test splits",
)
SessionBasePersonaSummaryTeacher.add_cmdline_args(parser, partial_opt)
return parser
def add_cmdline_args(cls,
parser: ParlaiParser,
partial_opt=None) -> ParlaiParser:
super().add_cmdline_args(parser, partial_opt)
arg_group = parser.add_argument_group(
'Wizard Dialog Knowledge arguments')
arg_group.add_argument(
'--prepend-gold-knowledge',
type='bool',
default=False,
help='If true, prepend text with checked sentences',
)
return parser
def main():
# Get command line arguments
argparser = ParlaiParser(True, True)
build = argparser.add_argument_group('Data Building Args')
build.add_argument('--datafile',
help=('The file to be loaded, preprocessed, and saved'))
build.add_argument('--pytorch-buildteacher', type=str, default='',
help='Which teacher to use when building the pytorch data')
build.add_argument('--pytorch-preprocess', type='bool', default=True,
help='Whether the agent should preprocess the data while building'
'the pytorch data')
opt = argparser.parse_args()
build_data(opt)
def main():
# Get command line arguments
parser = ParlaiParser(True, True)
train = parser.add_argument_group('Training Loop Arguments')
train.add_argument('-et', '--evaltask',
help=('task to use for valid/test (defaults to the '
'one used for training if not set)'))
train.add_argument('-d', '--display-examples',
type='bool', default=False)
train.add_argument('-e', '--num-epochs', type=float, default=-1)
train.add_argument('-ttim', '--max-train-time',
type=float, default=-1)
train.add_argument('-ltim', '--log-every-n-secs',
type=float, default=2)
train.add_argument('-vtim', '--validation-every-n-secs',
type=float, default=-1)
train.add_argument('-vme', '--validation-max-exs',
type=int, default=-1,
help='max examples to use during validation (default '
'-1 uses all)')
train.add_argument('-vp', '--validation-patience',
type=int, default=5,
help=('number of iterations of validation where result'
' does not improve before we stop training'))
train.add_argument('-vmt', '--validation-metric', default='accuracy',
help='key into report table for selecting best '
'validation')
train.add_argument('-dbf', '--dict-build-first',
type='bool', default=True,
help='build dictionary first before training agent')
opt = parser.parse_args()
# Possibly build a dictionary (not all models do this).
if opt['dict_build_first'] and 'dict_file' in opt:
if opt['dict_file'] is None and opt.get('model_file'):
opt['dict_file'] = opt['model_file'] + '.dict'
print("[ building dictionary first... ]")
build_dict.build_dict(opt)
# Create model and assign it to the specified task
agent = create_agent(opt)
world = create_task(opt, agent)
train_time = Timer()
validate_time = Timer()
log_time = Timer()
print('[ training... ]')
parleys = 0
total_exs = 0
max_exs = opt['num_epochs'] * len(world)
max_parleys = math.ceil(max_exs / opt['batchsize'])
best_valid = 0
impatience = 0
saved = False
valid_world = None
while True:
world.parley()
parleys += 1
if opt['num_epochs'] > 0 and parleys >= max_parleys:
print('[ num_epochs completed: {} ]'.format(opt['num_epochs']))
break
if opt['max_train_time'] > 0 and train_time.time() > opt['max_train_time']:
print('[ max_train_time elapsed: {} ]'.format(train_time.time()))
break
if opt['log_every_n_secs'] > 0 and log_time.time() > opt['log_every_n_secs']:
if opt['display_examples']:
print(world.display() + '\n~~')
logs = []
# time elapsed
logs.append('time:{}s'.format(math.floor(train_time.time())))
logs.append('parleys:{}'.format(parleys))
# get report and update total examples seen so far
if hasattr(agent, 'report'):
train_report = agent.report()
agent.reset_metrics()
else:
train_report = world.report()
world.reset_metrics()
if hasattr(train_report, 'get') and train_report.get('total'):
total_exs += train_report['total']
logs.append('total_exs:{}'.format(total_exs))
# check if we should log amount of time remaining
time_left = None
if opt['num_epochs'] > 0:
exs_per_sec = train_time.time() / total_exs
time_left = (max_exs - total_exs) * exs_per_sec
if opt['max_train_time'] > 0:
other_time_left = opt['max_train_time'] - train_time.time()
if time_left is not None:
time_left = min(time_left, other_time_left)
else:
time_left = other_time_left
if time_left is not None:
logs.append('time_left:{}s'.format(math.floor(time_left)))
# join log string and add full metrics report to end of log
log = '[ {} ] {}'.format(' '.join(logs), train_report)
print(log)
log_time.reset()
if (opt['validation_every_n_secs'] > 0 and
validate_time.time() > opt['validation_every_n_secs']):
valid_report, valid_world = run_eval(
agent, opt, 'valid', opt['validation_max_exs'],
valid_world=valid_world)
if valid_report[opt['validation_metric']] > best_valid:
best_valid = valid_report[opt['validation_metric']]
impatience = 0
print('[ new best {}: {} ]'.format(
opt['validation_metric'], best_valid))
world.save_agents()
saved = True
if opt['validation_metric'] == 'accuracy' and best_valid == 1:
print('[ task solved! stopping. ]')
break
else:
impatience += 1
print('[ did not beat best {}: {} impatience: {} ]'.format(
opt['validation_metric'], round(best_valid, 4),
impatience))
validate_time.reset()
if opt['validation_patience'] > 0 and impatience >= opt['validation_patience']:
print('[ ran out of patience! stopping training. ]')
break
world.shutdown()
if not saved:
world.save_agents()
else:
# reload best validation model
agent = create_agent(opt)
run_eval(agent, opt, 'valid', write_log=True)
run_eval(agent, opt, 'test', write_log=True)
