def add_cmdline_args(argparser):
arg_group = argparser.add_argument_group('MemNN Arguments')
arg_group.add_argument('--init-model',
type=str,
default=None,
help='load dict/model/opts from this path')
arg_group.add_argument('-esz',
'--embedding-size',
type=int,
default=128,
help='size of token embeddings')
arg_group.add_argument('-hops',
'--hops',
type=int,
default=3,
help='number of memory hops')
arg_group.add_argument('--memsize',
type=int,
default=32,
help='size of memory')
arg_group.add_argument('-tf',
'--time-features',
type='bool',
default=True,
help='use time features for memory embeddings')
arg_group.add_argument(
'-pe',
'--position-encoding',
type='bool',
default=False,
help='use position encoding instead of bag of words embedding')
TorchAgent.add_cmdline_args(argparser)
MemnnAgent.dictionary_class().add_cmdline_args(argparser)
return arg_group
def add_cmdline_args(argparser):
TorchAgent.add_cmdline_args(argparser)
agent = argparser.add_argument_group('TorchRankerAgent')
agent.add_argument(
'-cands',
'--candidates',
type=str,
default='inline',
choices=['batch', 'inline', 'fixed', 'vocab'],
help='The source of candidates during training '
'(see TorchRankerAgent._build_candidates() for details).')
agent.add_argument(
'-ecands',
'--eval-candidates',
type=str,
choices=['batch', 'inline', 'fixed', 'vocab'],
help=
'The source of candidates during evaluation (defaults to the same'
'value as --candidates if no flag is given)')
agent.add_argument(
'-fcp',
'--fixed-candidates-path',
type=str,
help='A text file of fixed candidates to use for all examples, one '
'candidate per line')
agent.add_argument(
'--fixed-candidate-vecs',
type=str,
default='reuse',
help="One of 'reuse', 'replace', or a path to a file with vectors "
"corresponding to the candidates at --fixed-candidates-path. "
"The default path is a /path/to/model-file.<cands_name>, where "
"<cands_name> is the name of the file (not the full path) passed by "
"the flag --fixed-candidates-path. By default, this file is created "
"once and reused. To replace it, use the 'replace' option.")
def add_cmdline_args(parser):
TorchAgent.add_cmdline_args(parser)
parser = parser.add_argument_group('Torch Classifier Arguments')
# class arguments
parser.add_argument('--classes', type=str, nargs='*', default=None,
help='the name of the classes.')
parser.add_argument('--class-weights', type=float, nargs='*', default=None,
help='weight of each of the classes for the softmax')
parser.add_argument('--ref-class', type=str, default=None, hidden=True,
help='the class that will be used to compute '
'precision and recall. By default the first '
'class.')
parser.add_argument('--threshold', type=float, default=0.5,
help='during evaluation, threshold for choosing '
'ref class; only applies to binary '
'classification')
# interactive mode
parser.add_argument('--print-scores', type='bool', default=False,
help='print probability of chosen class during '
'interactive mode')
# miscellaneous arguments
parser.add_argument('--data-parallel', type='bool', default=False,
help='uses nn.DataParallel for multi GPU')
parser.add_argument('--get-all-metrics', type='bool', default=True,
help='give prec/recall metrics for all classes')
def _init_embeddings(self, log=True):
"""
Copy embeddings from the pretrained embeddings to the lookuptable.
:param weight: weights of lookup table (nn.Embedding/nn.EmbeddingBag)
:param emb_type: pretrained embedding type
"""
weight = self.model.lt.weight
emb_type = self.opt.get('embedding_type', 'random')
if emb_type == 'random':
return
embs, name = TorchAgent._get_embtype(self, emb_type)
cnt = 0
for w, i in self.dict.tok2ind.items():
if w in embs.stoi:
vec = TorchAgent._project_vec(
self, embs.vectors[embs.stoi[w]], weight.size(1)
)
weight.data[i] = vec
cnt += 1
if log:
print(
'Initialized embeddings for {} tokens ({}%) from {}.'
''.format(cnt, round(cnt * 100 / len(self.dict), 1), name)
)
def add_cmdline_args(argparser):
"""Add command-line arguments specifically for this agent."""
TorchAgent.add_cmdline_args(argparser)
agent = argparser.add_argument_group('Image Caption Model Arguments')
agent.add_argument(
'--word_dim',
default=300,
type=int,
help='Dimensionality of the word embedding.',
)
agent.add_argument(
'--embed_size',
default=1024,
type=int,
help='Dimensionality of the joint embedding.',
)
agent.add_argument('--num_layers',
default=1,
type=int,
help='Number of GRU layers.')
agent.add_argument('--finetune',
type='bool',
default=False,
help='Finetune the image encoder')
agent.add_argument(
'--cnn_type',
default='resnet152',
help="""The CNN used for image encoder
(e.g. vgg19, resnet152)""",
)
agent.add_argument(
'--no_imgnorm',
type='bool',
default=False,
help='Do not normalize the image embeddings.',
)
agent.add_argument('--margin',
default=0.2,
type=float,
help='Rank loss margin.')
agent.add_argument(
'--max_violation',
type='bool',
default=True,
help='Use max instead of sum in the rank loss.',
)
agent.add_argument('-lr',
'--learning_rate',
type=float,
default=0.001,
help='learning rate')
VseppCaptionAgent.dictionary_class().add_cmdline_args(argparser)
def get_agent(**kwargs):
"""Return opt-initialized agent.
:param kwargs: any kwargs you want to set using parser.set_params(**kwargs)
"""
if 'no_cuda' not in kwargs:
kwargs['no_cuda'] = True
from parlai.core.params import ParlaiParser
parser = ParlaiParser()
TorchAgent.add_cmdline_args(parser)
parser.set_params(**kwargs)
opt = parser.parse_args(print_args=False)
return TorchAgent(opt)
def vectorize(self, *args, **kwargs):
"""
Override vectorize for generative models.
"""
kwargs['add_start'] = True # need start token for BART
kwargs['add_end'] = True
return TorchAgent.vectorize(self, *args, **kwargs)
def test_maintain_dialog_history(self):
try:
from parlai.core.torch_agent import TorchAgent
except ImportError as e:
if 'pytorch' in e.msg:
print(
'Skipping TestTorchAgent.test_maintain_dialog_history, no pytorch.'
)
return
opt = {}
opt['no_cuda'] = True
opt['truncate'] = 5
opt['history_dialog'] = 10
opt['history_replies'] = 'label_else_model'
mdict = MockDict()
shared = {'opt': opt, 'dict': mdict}
agent = TorchAgent(opt, shared)
observation = {
"text": "What is a painting?",
"labels": ["Paint on a canvas."],
"episode_done": False
}
agent.maintain_dialog_history(observation)
self.assertTrue('dialog' in agent.history,
"Failed initializing self.history.")
self.assertTrue('episode_done' in agent.history,
"Failed initializing self.history.")
self.assertTrue('labels' in agent.history,
"Failed initializing self.history.")
self.assertTrue(
list(agent.history['dialog']) == [7, 8, 9],
"Failed adding vectorized text to dialog.")
self.assertTrue(not agent.history['episode_done'],
"Failed to properly store episode_done field.")
self.assertTrue(agent.history['labels'] == observation['labels'],
"Failed saving labels.")
observation['text_vec'] = agent.maintain_dialog_history(observation)
self.assertTrue(
list(agent.history['dialog']) == [8, 9, 7, 8, 9],
"Failed adding vectorized text to dialog.")
def vectorize(self, *args, **kwargs):
"""
Override vectorize for T5.
T5 dict already adds the end token.
"""
kwargs['add_start'] = False # model does this in module code
kwargs['add_end'] = False # T5 tokenizer takes care of this
return TorchAgent.vectorize(self, *args, **kwargs)
def add_cmdline_args(argparser):
TorchAgent.add_cmdline_args(argparser)
agent = argparser.add_argument_group("EntNet Arguments")
agent.add_argument("-wt",
"--weight-tying",
type=str,
default="layer-wise",
help="Type of weight tying")
agent.add_argument("-nmh",
"--num-memory-hops",
type=int,
default=3,
help="Number of memory hops")
EntNetAgent.dictionary_class().add_cmdline_args(argparser)
return agent
def add_cmdline_args(argparser):
"""Add command-line arguments specifically for this agent."""
TorchAgent.add_cmdline_args(argparser)
agent = argparser.add_argument_group('Seq2Seq Arguments')
agent.add_argument('-hs', '--hiddensize', type=int, default=128,
help='size of the hidden layers')
agent.add_argument('-esz', '--embeddingsize', type=int, default=128,
help='size of the token embeddings')
agent.add_argument('-nl', '--numlayers', type=int, default=2,
help='number of hidden layers')
agent.add_argument('-lr', '--learningrate', type=float, default=1,
help='learning rate')
agent.add_argument('-dr', '--dropout', type=float, default=0.1,
help='dropout rate')
agent.add_argument('--gpu', type=int, default=-1,
help='which GPU device to use')
agent.add_argument('-rf', '--report-freq', type=float, default=0.001,
help='Report frequency of prediction during eval.')
ExampleSeq2seqAgent.dictionary_class().add_cmdline_args(argparser)
return agent
def test_vectorize(self):
opt = {}
opt['no_cuda'] = True
opt['history_tokens'] = 10000
opt['history_dialog'] = 10
opt['history_replies'] = 'label_else_model'
dict = MockDict()
shared = {'opt': opt, 'dict': dict}
agent = TorchAgent(opt, shared)
observation = {}
observation["text"] = "What does the dog do?"
observation["labels"] = ["The dog jumps over the cat."]
obs_vec = agent.vectorize(observation)
self.assertTrue(
'text_vec' in obs_vec,
"Field \'text_vec\' missing from vectorized observation")
self.assertTrue(obs_vec['text_vec'].numpy().tolist() == [1, 3, 5],
"Vectorized text is incorrect.")
self.assertTrue(
'labels_vec' in obs_vec,
"Field \'labels_vec\' missing from vectorized observation")
self.assertTrue(
obs_vec['labels_vec'][0].numpy().tolist() == [
1, 3, 5, dict.END_IDX
], "Vectorized label is incorrect.")
observation = {}
observation["text"] = "What does the dog do?"
observation["eval_labels"] = ["The dog jumps over the cat."]
obs_vec = agent.vectorize(observation)
self.assertTrue(
'eval_labels_vec' in obs_vec,
"Field \'eval_labels_vec\' missing from vectorized observation")
self.assertTrue(
obs_vec['eval_labels_vec'][0].numpy().tolist() == [
1, 3, 5, dict.END_IDX
], "Vectorized label is incorrect.")
def test_maintain_dialog_history(self):
opt = {}
opt['no_cuda'] = True
opt['history_tokens'] = 5
opt['history_dialog'] = 10
opt['history_replies'] = 'label_else_model'
dict = MockDict()
shared = {'opt': opt, 'dict': dict}
agent = TorchAgent(opt, shared)
observation = {
"text": "What is a painting?",
"labels": ["Paint on a canvas."],
"episode_done": False
}
agent.maintain_dialog_history(observation)
self.assertTrue('dialog' in agent.history,
"Failed initializing self.history.")
self.assertTrue('episode_done' in agent.history,
"Failed initializing self.history.")
self.assertTrue('labels' in agent.history,
"Failed initializing self.history.")
self.assertTrue(
list(agent.history['dialog']) == [1, 3, 5],
"Failed adding vectorized text to dialog.")
self.assertTrue(not agent.history['episode_done'],
"Failed to properly store episode_done field.")
self.assertTrue(agent.history['labels'] == observation['labels'],
"Failed saving labels.")
observation['text_vec'] = agent.maintain_dialog_history(observation)
self.assertTrue(
list(agent.history['dialog']) == [3, 5, 1, 3, 5],
"Failed adding vectorized text to dialog.")
def report(self):
"""
Report metrics from model's perspective.
"""
m = TorchAgent.report(self) # Skip TorchRankerAgent; totally redundant
examples = self.metrics['examples']
if examples > 0:
m['examples'] = examples
if 'dialog' in self.subtasks and self.metrics['dia_exs'] > 0:
m['dia_loss'] = self.metrics['dia_loss'] / self.metrics[
'dia_exs']
m['dia_rank'] = self.metrics['dia_rank'] / self.metrics[
'dia_exs']
m['dia_acc'] = self.metrics['dia_correct'] / self.metrics[
'dia_exs']
m['dia_exs'] = self.metrics['dia_exs']
if 'feedback' in self.subtasks and self.metrics['fee_exs'] > 0:
m['fee_loss'] = self.metrics['fee_loss'] / self.metrics[
'fee_exs']
m['fee_rank'] = self.metrics['fee_rank'] / self.metrics[
'fee_exs']
m['fee_acc'] = self.metrics['fee_correct'] / self.metrics[
'fee_exs']
m['fee_exs'] = self.metrics['fee_exs']
m['fee_exs'] = self.metrics['fee_exs']
if 'satisfaction' in self.subtasks and self.metrics['sat_exs'] > 0:
tp = self.metrics['sat_tp']
tn = self.metrics['sat_tn']
fp = self.metrics['sat_fp']
fn = self.metrics['sat_fn']
assert tp + tn + fp + fn == self.metrics['sat_exs']
m['sat_loss'] = self.metrics['sat_loss'] / self.metrics[
'sat_exs']
m['sat_pr'] = tp / (tp + fp + EPS)
m['sat_re'] = tp / (tp + fn + EPS)
pr = m['sat_pr']
re = m['sat_re']
m['sat_f1'] = (2 * pr * re) / (pr + re) if (pr and re) else 0.0
m['sat_acc'] = (tp + tn) / self.metrics['sat_exs']
m['sat_exs'] = self.metrics['sat_exs']
for k, v in m.items():
# clean up: rounds to sigfigs and converts tensors to floats
if isinstance(v, float):
m[k] = round_sigfigs(v, 4)
else:
m[k] = v
return m
def test_maintain_dialog_history(self):
try:
from parlai.core.torch_agent import TorchAgent
except ImportError as e:
if 'pytorch' in e.msg:
print(
'Skipping TestTorchAgent.test_maintain_dialog_history, no pytorch.'
)
return
from parlai.core.params import ParlaiParser
parser = ParlaiParser()
TorchAgent.add_cmdline_args(parser)
parser.set_params(no_cuda=True, truncate=5)
opt = parser.parse_args(print_args=False)
mdict = MockDict()
shared = {'opt': opt, 'dict': mdict}
agent = TorchAgent(opt, shared)
observation = {
"text": "What is a painting?",
"labels": ["Paint on a canvas."],
"episode_done": False
}
agent.maintain_dialog_history(observation)
self.assertTrue('dialog' in agent.history,
"Failed initializing self.history.")
self.assertTrue('episode_done' in agent.history,
"Failed initializing self.history.")
self.assertTrue('labels' in agent.history,
"Failed initializing self.history.")
self.assertTrue(
list(agent.history['dialog']) == [7, 8, 9],
"Failed adding vectorized text to dialog.")
self.assertTrue(not agent.history['episode_done'],
"Failed to properly store episode_done field.")
self.assertTrue(agent.history['labels'] == observation['labels'],
"Failed saving labels.")
observation['text_vec'] = agent.maintain_dialog_history(observation)
print(agent.history['dialog'])
self.assertTrue(
list(agent.history['dialog']) == [8, 9, 7, 8, 9],
"Failed adding vectorized text to dialog.")
def __init__(self, agent: TorchAgent):
super().__init__()
self.is_bart = isinstance(agent, BartAgent)
self.device = agent.model.encoder.embeddings.weight.device
# Dictionary/tokenization setup
for key, val in self.CAIRAOKE_DICT_PARAMS.items():
assert (
agent.opt.get(key, val) == val
), f'The only currently supported value of "{key}" is {val}!'
orig_dict: DictionaryAgent = agent.dict
orig_bpe: Gpt2BpeHelper = orig_dict.bpe
assert all(len(key) == 2 for key in orig_bpe.bpe_ranks.keys())
assert not any(
i for key in orig_bpe.bpe_ranks.keys() for i in key if "\n" in i
), "We need to temporarily merge the bpe_ranks dict's keys with a newline character in order to use it as a TorchScript arg, but at least one of the dict's keys contains a newline character already!"
fused_key_bpe_ranks = {
"\n".join(key): float(val)
for key, val in orig_bpe.bpe_ranks.items()
}
# Cast the values as floats to be able to compare to float('inf') when doing BPE
# splitting
self.dict = ScriptableDictionaryAgent(
null_token=orig_dict.null_token,
end_token=orig_dict.end_token,
unk_token=orig_dict.unk_token,
start_token=orig_dict.start_token,
freq=orig_dict.freq,
tok2ind=orig_dict.tok2ind,
ind2tok=orig_dict.ind2tok,
bpe_add_prefix_space=agent.opt["bpe_add_prefix_space"],
bpe_encoder=orig_bpe.encoder,
bpe_byte_encoder=orig_bpe.byte_encoder,
fused_key_bpe_ranks=fused_key_bpe_ranks,
special_tokens=agent._get_special_tokens(),
)
# History tracking and start/end tokens
self.delimiter_tok = agent.history.delimiter_tok
self.history_size = agent.opt["history_size"]
if agent.opt.get("history_add_global_end_token", None) is not None:
self.global_end_token = agent.dict[agent.dict.end_token]
else:
self.global_end_token = None
self.text_truncate = agent.opt.get(
"text_truncate") or agent.opt["truncate"]
self.text_truncate = self.text_truncate if self.text_truncate >= 0 else None
self.start_idx = agent.model.START_IDX
self.end_idx = agent.model.END_IDX
self.null_idx = agent.model.NULL_IDX
if self.is_bart:
self.initial_decoder_input = [self.end_idx, self.start_idx]
else:
self.initial_decoder_input = [self.start_idx]
agent.model.eval()
# Create versions of the model and decoder that will flatten the incremental
# state dict, as required by TorchScript
wrapped_decoder = DecoderIncrStateFlattener(agent.model.decoder)
wrapped_model = ModelIncrStateFlattener(agent.model)
# Create sample inputs for tracing
sample_tokens = torch.tensor([[1, 2, 3, 4, 5]],
dtype=torch.long,
device=self.device)
sample_tokens = sample_tokens.to(self.device)
encoder_states = agent.model.encoder(sample_tokens)
initial_generations = self._get_initial_decoder_input(sample_tokens)
latent, initial_incr_state = wrapped_decoder(initial_generations,
encoder_states)
logits = agent.model.output(latent[:, -1:, :])
_, preds = logits.max(dim=2)
incr_state = {k: torch.clone(v) for k, v in initial_incr_state.items()}
# Copy the initial incremental state, used when tracing the
# .reorder_decoder_incremental_state() method below, to avoid having it be
# mutated by the following line
incr_state = wrapped_model.reorder_decoder_incremental_state(
incr_state,
torch.tensor([0], dtype=torch.long, device=sample_tokens.device))
generations = torch.cat([initial_generations, preds], dim=1)
# Do tracing
self.encoder = torch.jit.trace(agent.model.encoder, sample_tokens)
self.decoder_first_pass = torch.jit.trace(
wrapped_decoder, (initial_generations, encoder_states),
strict=False)
# We do strict=False to avoid an error when passing a Dict out of
# decoder.forward()
self.partially_traced_model = torch.jit.trace_module(
wrapped_model,
{
"output": (latent[:, -1:, :]),
"reorder_decoder_incremental_state": (
initial_incr_state,
torch.tensor(
[0], dtype=torch.long, device=sample_tokens.device),
),
},
strict=False,
)
self.decoder_later_pass = torch.jit.trace(
wrapped_decoder, (generations, encoder_states, incr_state),
strict=False)
def add_cmdline_args(argparser):
"""Add command-line arguments specifically for this agent."""
# first we need to add the general torch agent operations
TorchAgent.add_cmdline_args(argparser)
agent = argparser.add_argument_group('Fairseq Arguments')
agent.add_argument(
'--seed',
default=1,
type=int,
metavar='N',
help='pseudo random number generator seed'
)
agent.add_argument(
'--skip-generation',
default=False,
type=bool,
metavar='BOOL',
help='Skips test time beam search. Much faster if you only need PPL',
)
# Dictionary construction stuff. Using the subclass in case we end up
# needing any fairseq specific things
_FairseqDictionary.add_cmdline_args(argparser)
# Optimization and learning rate schedule specific arguments
options.add_optimization_args(argparser)
known_args = argparser.parse_known_args(nohelp=True)[0]
if hasattr(known_args, "optimizer"):
optimizer = known_args.optimizer
opt_group = argparser.add_argument_group(
'{} optimizer arguments'.format(optimizer)
)
optim.OPTIMIZER_REGISTRY[optimizer].add_args(opt_group)
if hasattr(known_args, "lr_scheduler"):
lr_scheduler = known_args.lr_scheduler
lr_group = argparser.add_argument_group(
'{} scheduler arguments'.format(lr_scheduler)
)
optim.lr_scheduler.LR_SCHEDULER_REGISTRY[lr_scheduler].add_args(lr_group)
# Generation arguments
options.add_generation_args(argparser)
# We need to find out the fairseq model-specific options, so grab the
# architecture stuff and look up its options
arch_group = options.add_model_args(argparser)
# Fairseq marks the arch flag as required, but it may be specified
# by a saved model cache, so we do some weird stuff to undo that
for a in arch_group._actions:
if a.dest == "arch":
a.required = False
a.default = None
break
known_args = argparser.parse_known_args(nohelp=True)[0]
if hasattr(known_args, "arch") and known_args.arch is not None:
arch = known_args.arch
arch_group = argparser.add_argument_group(
"{} architecture arguments".format(arch)
)
models.ARCH_MODEL_REGISTRY[arch].add_args(arch_group)
# Override a few defaults from within fairseq to more sensible defaults
argparser.set_defaults(
clip_norm=0.1,
adam_betas="(0.9,0.98)"
)
def add_cmdline_args(cls, argparser):
super().add_cmdline_args(argparser)
agent = argparser.add_argument_group('SteroidSeq2seq args')
agent.add_argument('--search-type-during-eval',
type=str,
default='greedy',
choices=['human', 'greedy', 'beam', 'blockbeam'])
agent.add_argument('--howtorank',
type=str,
choices=['ranker', 'beam'],
default='ranker')
agent.add_argument(
'--cand-type',
type=str,
choices=['none', 'current_labels', 'history', 'all'],
default='none',
help=
'Candidates used to train ranker part, history assumes injected preds'
)
agent.add_argument('--margin',
type=float,
default=1.0,
help='Margin for RLoss')
agent.add_argument('--input-dropout',
type=float,
default=0.1,
help='Change '
'input token to UNK with probability given here')
agent.add_argument(
'--num-rank-cand',
default=1,
type=int,
help='This is only used when we do current-labels cand-type')
agent.add_argument('--lmweight',
default=1.0,
type=float,
help='weight for LM loss')
agent.add_argument('--rankweight',
default=1.0,
type=float,
help='weight for Rank loss')
agent.add_argument('--rankhiddensize',
default=512,
type=int,
help='Hidden size of all layers in ranker')
agent.add_argument('--ranknl',
default=2,
type=int,
help='Number of linear layers in ranker')
agent.add_argument('--ranklossreduce',
type=str,
choices=['sum', 'elementwise_mean'],
default='elementwise_mean',
help='reduce type for the loss')
agent.add_argument(
'--rankloss',
type=str,
default='margin',
choices=['ce', 'margin'],
help='The loss which we use in the optimization criterion')
agent.add_argument('--rank-activation',
type=str,
default='ReLU',
help='Ranker activation function, should be nn.*')
agent.add_argument('--strict-load', type='bool', default=True)
agent.add_argument('--iter-cand', type=int, default=1)
agent.add_argument('--min-hamming-dist', type=int, default=1)
agent.add_argument('--count-overlaps', type='bool', default=False)
# not used right now
agent.add_argument('--dump-all-preds', type='bool', default=False)
TorchAgent.add_cmdline_args(argparser)
SteroidSeq2seqAgent.dictionary_class().add_cmdline_args(argparser)
return agent
def add_cmdline_args(cls, argparser):
"""Add command-line arguments specifically for this agent."""
agent = argparser.add_argument_group('Seq2Seq Arguments')
agent.add_argument('--init-model',
type=str,
default=None,
help='load dict/model/opts from this path')
agent.add_argument('-hs',
'--hiddensize',
type=int,
default=128,
help='size of the hidden layers')
agent.add_argument('-esz',
'--embeddingsize',
type=int,
default=128,
help='size of the token embeddings')
agent.add_argument('-nl',
'--numlayers',
type=int,
default=2,
help='number of hidden layers')
agent.add_argument('-dr',
'--dropout',
type=float,
default=0.1,
help='dropout rate')
agent.add_argument('-bi',
'--bidirectional',
type='bool',
default=False,
help='whether to encode the context with a '
'bidirectional rnn')
agent.add_argument(
'-att',
'--attention',
default='none',
choices=['none', 'concat', 'general', 'dot', 'local'],
help='Choices: none, concat, general, local. '
'If set local, also set attention-length. '
'(see arxiv.org/abs/1508.04025)')
agent.add_argument('-attl',
'--attention-length',
default=48,
type=int,
help='Length of local attention.')
agent.add_argument('--attention-time',
default='post',
choices=['pre', 'post'],
help='Whether to apply attention before or after '
'decoding.')
agent.add_argument('-rnn',
'--rnn-class',
default='lstm',
choices=Seq2seq.RNN_OPTS.keys(),
help='Choose between different types of RNNs.')
agent.add_argument('-dec',
'--decoder',
default='same',
choices=['same', 'shared'],
help='Choose between different decoder modules. '
'Default "same" uses same class as encoder, '
'while "shared" also uses the same weights. '
'Note that shared disabled some encoder '
'options--in particular, bidirectionality.')
agent.add_argument('-lt',
'--lookuptable',
default='unique',
choices=['unique', 'enc_dec', 'dec_out', 'all'],
help='The encoder, decoder, and output modules can '
'share weights, or not. '
'Unique has independent embeddings for each. '
'Enc_dec shares the embedding for the encoder '
'and decoder. '
'Dec_out shares decoder embedding and output '
'weights. '
'All shares all three weights.')
agent.add_argument('-soft',
'--numsoftmax',
default=1,
type=int,
help='default 1, if greater then uses mixture of '
'softmax (see arxiv.org/abs/1711.03953).')
agent.add_argument('--beam-size',
type=int,
default=1,
help='Beam size, if 1 then greedy search')
agent.add_argument(
'--beam-dot-log',
type='bool',
default=False,
help='Dump beam trees as png dot images into /tmp folder')
agent.add_argument(
'--beam-min-n-best',
type=int,
default=3,
help='Minimum number of nbest candidates to achieve '
'during the beam search')
agent.add_argument(
'--beam-min-length',
type=int,
default=3,
help='Minimum length of prediction to be generated by '
'the beam search')
agent.add_argument('-idr',
'--input-dropout',
type=float,
default=0.0,
help='Each token from the input will be masked with'
' __unk__ token with this probability.')
agent.add_argument(
'--beam-block-ngram',
type=int,
default=0,
help='Block all repeating ngrams up to history length n-1')
TorchAgent.add_cmdline_args(argparser)
Seq2seqAgent.dictionary_class().add_cmdline_args(argparser)
return agent
def test_vectorize(self):
"""
Goal of this test is to make sure that the vectorize function is
actually adding a new field.
"""
try:
from parlai.core.torch_agent import TorchAgent
except ImportError as e:
if 'pytorch' in e.msg:
print('Skipping TestTorchAgent.test_vectorize, no pytorch.')
return
opt = {}
opt['no_cuda'] = True
opt['truncate'] = 10000
opt['history_dialog'] = 10
opt['history_replies'] = 'label_else_model'
mdict = MockDict()
shared = {'opt': opt, 'dict': mdict}
agent = TorchAgent(opt, shared)
observation = {}
observation["text"] = "What does the dog do?"
observation["labels"] = ["The dog jumps over the cat."]
# add start and end
obs_vec = agent.vectorize(observation, add_start=True, add_end=True)
self.assertTrue(
'text_vec' in obs_vec,
"Field 'text_vec' missing from vectorized observation")
self.assertTrue(obs_vec['text_vec'].numpy().tolist() == [7, 8, 9],
"Vectorized text is incorrect.")
self.assertTrue(
'labels_vec' in obs_vec,
"Field 'labels_vec' missing from vectorized observation")
self.assertTrue(
obs_vec['labels_vec'].numpy().tolist() == [
mdict.START_IDX, 7, 8, 9, mdict.END_IDX
], "Vectorized label is incorrect.")
# no start, add end
obs_vec = agent.vectorize(observation, add_start=False, add_end=True)
self.assertTrue(
obs_vec['labels_vec'].numpy().tolist() == [7, 8, 9, mdict.END_IDX],
"Vectorized label is incorrect.")
# add start, no end
obs_vec = agent.vectorize(observation, add_start=True, add_end=False)
self.assertTrue(
obs_vec['labels_vec'].numpy().tolist() == [
mdict.START_IDX, 7, 8, 9
], "Vectorized label is incorrect.")
# no start, no end
obs_vec = agent.vectorize(observation, add_start=False, add_end=False)
self.assertTrue(obs_vec['labels_vec'].numpy().tolist() == [7, 8, 9],
"Vectorized label is incorrect.")
observation = {}
observation["text"] = "What does the dog do?"
observation["eval_labels"] = ["The dog jumps over the cat."]
# eval_labels
obs_vec = agent.vectorize(observation)
self.assertTrue(
'eval_labels_vec' in obs_vec,
"Field \'eval_labels_vec\' missing from vectorized observation")
self.assertTrue(
obs_vec['eval_labels_vec'].numpy().tolist() == [
mdict.START_IDX, 7, 8, 9, mdict.END_IDX
], "Vectorized label is incorrect.")
# truncate
obs_vec = agent.vectorize(observation, truncate=3)
self.assertTrue(
'eval_labels_vec' in obs_vec,
"Field \'eval_labels_vec\' missing from vectorized observation")
self.assertTrue(
obs_vec['eval_labels_vec'].numpy().tolist() == [
8, 9, mdict.END_IDX
], "Vectorized label is incorrect.")
# truncate
obs_vec = agent.vectorize(observation, truncate=10)
self.assertTrue(
'eval_labels_vec' in obs_vec,
"Field \'eval_labels_vec\' missing from vectorized observation")
self.assertTrue(
obs_vec['eval_labels_vec'].numpy().tolist() == [
mdict.START_IDX, 7, 8, 9, mdict.END_IDX
], "Vectorized label is incorrect.")
def add_cmdline_args(cls, argparser):
"""Add command-line arguments specifically for this agent."""
# first we need to add the general torch agent operations
TorchAgent.add_cmdline_args(argparser)
# Dictionary construction stuff. Using the subclass in case we end up
# needing any fairseq specific things
cls.dictionary_class().add_cmdline_args(argparser)
# let's store any defaults that were overridden
old_defaults = argparser._defaults
if 'clip_norm' not in old_defaults:
# fairseq has a few awful defaults
old_defaults['clip_norm'] = 1.0
if 'optimizer' not in old_defaults:
old_defaults['optimizer'] = 'adam'
old_defaults['adam_betas'] = '(0.9,0.98)'
agent = argparser.add_argument_group('Fairseq Arguments')
agent.add_argument('--fp16',
default=False,
type='bool',
help='Use fp16 training')
agent.add_argument('--fp16-init-scale',
default=2**7,
type=int,
help='default FP16 loss scale')
agent.add_argument('--seed',
default=1,
type=int,
metavar='N',
help='pseudo random number generator seed')
agent.add_argument(
'--skip-generation',
default=False,
type='bool',
metavar='BOOL',
help=
'Skips test time beam search. Much faster if you only need PPL',
)
# Check subargs for generation, optimizers, criterions, archs, etc
options.add_generation_args(argparser)
options.add_optimization_args(argparser)
options.add_checkpoint_args(argparser)
# restore any user set defaults that fairseq possibly overrode
argparser.set_defaults(**old_defaults)
known_args = argparser.parse_known_args(nohelp=True)[0]
if hasattr(known_args, "optimizer"):
optimizer = known_args.optimizer
opt_group = argparser.add_argument_group(
'{} optimizer arguments'.format(optimizer))
optim.OPTIMIZER_REGISTRY[optimizer].add_args(opt_group)
if hasattr(known_args, "lr_scheduler"):
lr_scheduler = known_args.lr_scheduler
lr_group = argparser.add_argument_group(
'{} scheduler arguments'.format(lr_scheduler))
optim.lr_scheduler.LR_SCHEDULER_REGISTRY[lr_scheduler].add_args(
lr_group)
# We need to find out the fairseq model-specific options, so grab the
# architecture stuff and look up its options
arch_group = options.add_model_args(argparser)
# Fairseq marks the arch flag as required, but it may be specified
# by a saved model cache, so we do some weird stuff to undo that
for a in arch_group._actions:
if a.dest == "arch":
a.required = False
a.default = None
break
# once again restore any user-set defaults
argparser.set_defaults(**old_defaults)
known_args = argparser.parse_known_args(nohelp=True)[0]
if hasattr(known_args, "arch") and known_args.arch is not None:
arch = known_args.arch
arch_group = argparser.add_argument_group(
"{} architecture arguments".format(arch))
models.ARCH_MODEL_REGISTRY[arch].add_args(arch_group)
if hasattr(known_args, "criterion"):
crit_group = argparser.add_argument_group(
'{} criterion arguments'.format(known_args.criterion))
criterions.CRITERION_REGISTRY[known_args.criterion].add_args(
crit_group)
# one last time, restore any user set defaults
argparser.set_defaults(**old_defaults)
def test_map_unmap(self):
try:
from parlai.core.torch_agent import TorchAgent, Output
except ImportError as e:
if 'pytorch' in e.msg:
print('Skipping TestTorchAgent.test_map_unmap, no pytorch.')
return
observations = []
observations.append({
"text": "What is a painting?",
"labels": ["Paint on a canvas."]
})
observations.append({})
observations.append({})
observations.append({
"text": "What is a painting?",
"labels": ["Paint on a canvas."]
})
observations.append({})
observations.append({})
opt = {}
opt['no_cuda'] = True
opt['truncate'] = 10000
opt['history_dialog'] = 10
opt['history_replies'] = 'label_else_model'
mdict = MockDict()
shared = {'opt': opt, 'dict': mdict}
agent = TorchAgent(opt, shared)
vec_observations = [agent.vectorize(obs) for obs in observations]
batch = agent.batchify(vec_observations)
self.assertTrue(batch.text_vec is not None,
"Missing 'text_vecs' field.")
self.assertTrue(
batch.text_vec.numpy().tolist() == [[7, 8, 9], [7, 8, 9]],
"Incorrectly vectorized text field of obs_batch.")
self.assertTrue(batch.label_vec is not None,
"Missing 'label_vec' field.")
self.assertTrue(
batch.label_vec.numpy().tolist() == [[
mdict.START_IDX, 7, 8, 9, mdict.END_IDX
], [mdict.START_IDX, 7, 8, 9, mdict.END_IDX]],
"Incorrectly vectorized text field of obs_batch.")
self.assertTrue(
batch.labels == ["Paint on a canvas.", "Paint on a canvas."],
"Doesn't return correct labels: " + str(batch.labels))
true_i = [0, 3]
self.assertTrue(
all(batch.valid_indices[i] == true_i[i] for i in range(2)),
"Returns incorrect indices of valid observations.")
observations = []
observations.append({
"text": "What is a painting?",
"eval_labels": ["Paint on a canvas."]
})
observations.append({})
observations.append({})
observations.append({
"text": "What is a painting?",
"eval_labels": ["Paint on a canvas."]
})
observations.append({})
observations.append({})
vec_observations = [agent.vectorize(obs) for obs in observations]
batch = agent.batchify(vec_observations)
self.assertTrue(batch.label_vec is not None,
"Missing \'eval_label_vec\' field.")
self.assertTrue(
batch.label_vec.numpy().tolist() == [[
mdict.START_IDX, 7, 8, 9, mdict.END_IDX
], [mdict.START_IDX, 7, 8, 9, mdict.END_IDX]],
"Incorrectly vectorized text field of obs_batch.")
batch_reply = [{} for i in range(6)]
predictions = ["Oil on a canvas.", "Oil on a canvas."]
output = Output(predictions, None)
expected_unmapped = batch_reply.copy()
expected_unmapped[0]["text"] = "Oil on a canvas."
expected_unmapped[3]["text"] = "Oil on a canvas."
self.assertTrue(
agent.match_batch(batch_reply, batch.valid_indices,
output) == expected_unmapped,
"Unmapped predictions do not match expected results.")
def add_cmdline_args(parser):
"""
Add CLI args.
"""
TorchAgent.add_cmdline_args(parser)
parser = parser.add_argument_group('Torch Classifier Arguments')
# class arguments
parser.add_argument(
'--classes',
type=str,
nargs='*',
default=None,
help='the name of the classes.',
)
parser.add_argument(
'--class-weights',
type=float,
nargs='*',
default=None,
help='weight of each of the classes for the softmax',
)
parser.add_argument(
'--ref-class',
type=str,
default=None,
hidden=True,
help='the class that will be used to compute '
'precision and recall. By default the first '
'class.',
)
parser.add_argument(
'--threshold',
type=float,
default=0.5,
help='during evaluation, threshold for choosing '
'ref class; only applies to binary '
'classification',
)
# interactive mode
parser.add_argument(
'--print-scores',
type='bool',
default=False,
help='print probability of chosen class during ' 'interactive mode',
)
# miscellaneous arguments
parser.add_argument(
'--data-parallel',
type='bool',
default=False,
help='uses nn.DataParallel for multi GPU',
)
parser.add_argument(
'--classes-from-file',
type=str,
default=None,
help='loads the list of classes from a file',
)
parser.add_argument(
'--ignore-labels',
type='bool',
default=None,
help='Ignore labels provided to model',
)
def test_vectorize(self):
"""
Make sure that the vectorize function is actually adding a new field.
"""
try:
from parlai.core.torch_agent import TorchAgent
except ImportError as e:
if 'pytorch' in e.msg:
print('Skipping TestTorchAgent.test_vectorize, no pytorch.')
return
from parlai.core.params import ParlaiParser
parser = ParlaiParser()
TorchAgent.add_cmdline_args(parser)
parser.set_params(no_cuda=True)
opt = parser.parse_args(print_args=False)
mdict = MockDict()
shared = {'opt': opt, 'dict': mdict}
agent = TorchAgent(opt, shared)
observation = {}
observation["text"] = "What does the dog do?"
observation["labels"] = ["The dog jumps over the cat."]
# add start and end
obs_vec = agent.vectorize(observation, add_start=True, add_end=True)
self.assertTrue(
'text_vec' in obs_vec,
"Field 'text_vec' missing from vectorized observation")
self.assertTrue(obs_vec['text_vec'].numpy().tolist() == [7, 8, 9],
"Vectorized text is incorrect.")
self.assertTrue(
'labels_vec' in obs_vec,
"Field 'labels_vec' missing from vectorized observation")
self.assertTrue(
obs_vec['labels_vec'].numpy().tolist() == [
mdict.START_IDX, 7, 8, 9, mdict.END_IDX
], "Vectorized label is incorrect.")
# no start, add end
obs_vec = agent.vectorize(observation, add_start=False, add_end=True)
self.assertTrue(
obs_vec['labels_vec'].numpy().tolist() == [7, 8, 9, mdict.END_IDX],
"Vectorized label is incorrect.")
# add start, no end
obs_vec = agent.vectorize(observation, add_start=True, add_end=False)
self.assertTrue(
obs_vec['labels_vec'].numpy().tolist() == [
mdict.START_IDX, 7, 8, 9
], "Vectorized label is incorrect.")
# no start, no end
obs_vec = agent.vectorize(observation, add_start=False, add_end=False)
self.assertTrue(obs_vec['labels_vec'].numpy().tolist() == [7, 8, 9],
"Vectorized label is incorrect.")
observation = {}
observation["text"] = "What does the dog do?"
observation["eval_labels"] = ["The dog jumps over the cat."]
# eval_labels
obs_vec = agent.vectorize(observation)
self.assertTrue(
'eval_labels_vec' in obs_vec,
"Field \'eval_labels_vec\' missing from vectorized observation")
self.assertTrue(
obs_vec['eval_labels_vec'].numpy().tolist() == [
mdict.START_IDX, 7, 8, 9, mdict.END_IDX
], "Vectorized label is incorrect.")
# truncate
obs_vec = agent.vectorize(observation, truncate=2)
self.assertTrue(
'eval_labels_vec' in obs_vec,
"Field \'eval_labels_vec\' missing from vectorized observation")
self.assertTrue(
obs_vec['eval_labels_vec'].numpy().tolist() == [
mdict.START_IDX, 7
], "Vectorized label is incorrect: " +
str(obs_vec['eval_labels_vec']))
# truncate
obs_vec = agent.vectorize(observation, truncate=10)
self.assertTrue(
'eval_labels_vec' in obs_vec,
"Field \'eval_labels_vec\' missing from vectorized observation")
self.assertTrue(
obs_vec['eval_labels_vec'].numpy().tolist() == [
mdict.START_IDX, 7, 8, 9, mdict.END_IDX
], "Vectorized label is incorrect.")
def add_cmdline_args(cls, argparser):
"""Add command-line arguments specifically for this agent."""
agent = argparser.add_argument_group('Seq2Seq Arguments')
agent.add_argument('--init-model', type=str, default=None,
help='load dict/model/opts from this path')
agent.add_argument('-hs', '--hiddensize', type=int, default=128,
help='size of the hidden layers')
agent.add_argument('-esz', '--embeddingsize', type=int, default=128,
help='size of the token embeddings')
agent.add_argument('-nl', '--numlayers', type=int, default=2,
help='number of hidden layers')
agent.add_argument('-lr', '--learningrate', type=float, default=1,
help='learning rate')
agent.add_argument('-dr', '--dropout', type=float, default=0.1,
help='dropout rate')
agent.add_argument('-clip', '--gradient-clip', type=float, default=0.1,
help='gradient clipping using l2 norm')
agent.add_argument('-bi', '--bidirectional', type='bool',
default=False,
help='whether to encode the context with a '
'bidirectional rnn')
agent.add_argument('-att', '--attention', default='none',
choices=['none', 'concat', 'general', 'dot',
'local'],
help='Choices: none, concat, general, local. '
'If set local, also set attention-length. '
'(see arxiv.org/abs/1508.04025)')
agent.add_argument('-attl', '--attention-length', default=48, type=int,
help='Length of local attention.')
agent.add_argument('--attention-time', default='post',
choices=['pre', 'post'],
help='Whether to apply attention before or after '
'decoding.')
agent.add_argument('-rnn', '--rnn-class', default='lstm',
choices=Seq2seq.RNN_OPTS.keys(),
help='Choose between different types of RNNs.')
agent.add_argument('-dec', '--decoder', default='same',
choices=['same', 'shared'],
help='Choose between different decoder modules. '
'Default "same" uses same class as encoder, '
'while "shared" also uses the same weights. '
'Note that shared disabled some encoder '
'options--in particular, bidirectionality.')
agent.add_argument('-lt', '--lookuptable', default='unique',
choices=['unique', 'enc_dec', 'dec_out', 'all'],
help='The encoder, decoder, and output modules can '
'share weights, or not. '
'Unique has independent embeddings for each. '
'Enc_dec shares the embedding for the encoder '
'and decoder. '
'Dec_out shares decoder embedding and output '
'weights. '
'All shares all three weights.')
agent.add_argument('-soft', '--numsoftmax', default=1, type=int,
help='default 1, if greater then uses mixture of '
'softmax (see arxiv.org/abs/1711.03953).')
agent.add_argument('--beam-size', type=int, default=1,
help='Beam size, if 1 then greedy search')
agent.add_argument('--beam-log-freq', type=float, default=0.0,
help='The portion of beams to dump from minibatch '
'into model_name.beam_dump folder')
agent.add_argument('--topk', type=int, default=1,
help='Top k sampling from renormalized softmax in '
'test/valid time, default 1 means simple '
'greedy max output')
TorchAgent.add_cmdline_args(argparser)
Seq2seqAgent.dictionary_class().add_cmdline_args(argparser)
return agent
def add_cmdline_args(cls, argparser):
"""Add command-line arguments specifically for this agent."""
agent = argparser.add_argument_group(
'ControllableSeq2seqAgent Arguments')
agent.add_argument(
'--init-model',
type=str,
default=None,
help='load dict/model/opts from this path',
)
agent.add_argument(
'-hs',
'--hiddensize',
type=int,
default=128,
help='size of the hidden layers',
)
agent.add_argument(
'-esz',
'--embeddingsize',
type=int,
default=128,
help='size of the token embeddings',
)
agent.add_argument('-nl',
'--numlayers',
type=int,
default=2,
help='number of hidden layers')
agent.add_argument('-dr',
'--dropout',
type=float,
default=0.1,
help='dropout rate')
agent.add_argument(
'-bi',
'--bidirectional',
type='bool',
default=False,
help='whether to encode the context with a '
'bidirectional rnn',
)
agent.add_argument(
'-att',
'--attention',
default='none',
choices=['none', 'concat', 'general', 'dot', 'local'],
help='Choices: none, concat, general, local. '
'If set local, also set attention-length. '
'(see arxiv.org/abs/1508.04025)',
)
agent.add_argument(
'-attl',
'--attention-length',
default=48,
type=int,
help='Length of local attention.',
)
agent.add_argument(
'--attention-time',
default='post',
choices=['pre', 'post'],
help='Whether to apply attention before or after '
'decoding.',
)
agent.add_argument(
'-rnn',
'--rnn-class',
default='lstm',
choices=Seq2seq.RNN_OPTS.keys(),
help='Choose between different types of RNNs.',
)
agent.add_argument(
'-dec',
'--decoder',
default='same',
choices=['same', 'shared'],
help='Choose between different decoder modules. '
'Default "same" uses same class as encoder, '
'while "shared" also uses the same weights. '
'Note that shared disabled some encoder '
'options--in particular, bidirectionality.',
)
agent.add_argument(
'-lt',
'--lookuptable',
default='unique',
choices=['unique', 'enc_dec', 'dec_out', 'all'],
help='The encoder, decoder, and output modules can '
'share weights, or not. '
'Unique has independent embeddings for each. '
'Enc_dec shares the embedding for the encoder '
'and decoder. '
'Dec_out shares decoder embedding and output '
'weights. '
'All shares all three weights.',
)
agent.add_argument(
'-soft',
'--numsoftmax',
default=1,
type=int,
help='default 1, if greater then uses mixture of '
'softmax (see arxiv.org/abs/1711.03953).',
)
agent.add_argument(
'--beam-size',
type=int,
default=1,
help='Beam size, if 1 then greedy search',
)
agent.add_argument(
'--beam-dot-log',
type='bool',
default=False,
help='Dump beam trees as png dot images into /tmp folder',
)
agent.add_argument(
'--beam-min-n-best',
type=int,
default=3,
help='Minimum number of nbest candidates to achieve '
'during the beam search',
)
agent.add_argument(
'--beam-min-length',
type=int,
default=3,
help='Minimum length of prediction to be generated by '
'the beam search',
)
agent.add_argument(
'-idr',
'--input-dropout',
type=float,
default=0.0,
help='Each token from the input will be masked with'
' __unk__ token with this probability.',
)
agent.add_argument(
'--beam-block-ngram',
type=int,
default=0,
help='Block all repeating ngrams up to history length n-1',
)
agent.add_argument(
'-cv',
'--control-vars',
type=str,
default='',
help='Comma-separated list of control variables to use',
)
agent.add_argument(
'-cnb',
'--control-num-buckets',
type=str,
default='',
help='Number of buckets for each of the control variables',
)
agent.add_argument(
'-cesz',
'--control-embeddingsize',
type=str,
default='',
help='Sizes for the control variable embeddings',
)
agent.add_argument(
'--add-control',
type='bool',
default=False,
help='If True, takes an existing saved model, adds necessary'
'parameters for new CT controls, and saves in a new model '
'file',
)
agent.add_argument(
'--set-controls',
type=str,
default='',
help='Specify fixed settings for CT control variables. '
'For example, avg_niwf:6',
)
agent.add_argument(
'--beam-reorder',
default='none',
choices=['none', 'best_extrep2gram_qn'],
help='Choices: none, best_extrep2gram_qn.'
'Apply the specified function for reordering the '
'n-best beam search candidates. '
'If best_extrep2gram_qn, then pick candidate which '
'contains question mark and has lowest extrep_2gram',
)
agent.add_argument(
'-wd',
'--weighted-decoding',
type=str,
default='',
help='List of WD features and their corresponding weights '
'For example, intrep_word:-1,extrep_2gram:-1,nidf:3',
)
agent.add_argument(
'--verbose',
type='bool',
default=False,
help='If true, print out beam search info',
)
TorchAgent.add_cmdline_args(argparser)
ControllableSeq2seqAgent.dictionary_class().add_cmdline_args(argparser)
return agent
def test_map_unmap(self):
observations = []
observations.append({
"text": "What is a painting?",
"labels": ["Paint on a canvas."]
})
observations.append({})
observations.append({})
observations.append({
"text": "What is a painting?",
"labels": ["Paint on a canvas."]
})
observations.append({})
observations.append({})
opt = {}
opt['no_cuda'] = True
opt['history_tokens'] = 10000
opt['history_dialog'] = 10
opt['history_replies'] = 'label_else_model'
dict = MockDict()
shared = {'opt': opt, 'dict': dict}
agent = TorchAgent(opt, shared)
vec_observations = [agent.vectorize(obs) for obs in observations]
mapped_valid = agent.map_valid(vec_observations)
text_vecs, label_vecs, labels, valid_inds = mapped_valid
self.assertTrue(text_vecs is not None, "Missing \'text_vecs\' field.")
self.assertTrue(text_vecs.numpy().tolist() == [[1, 3, 5], [1, 3, 5]],
"Incorrectly vectorized text field of obs_batch.")
self.assertTrue(label_vecs is not None, "Missing \'label_vec\' field.")
self.assertTrue(
label_vecs.numpy().tolist() == [[1, 3, 5, 2], [1, 3, 5, 2]],
"Incorrectly vectorized text field of obs_batch.")
self.assertTrue(labels == ["Paint on a canvas.", "Paint on a canvas."],
"Doesn't return correct labels.")
self.assertTrue(valid_inds == [0, 3],
"Returns incorrect indices of valid observations.")
observations = []
observations.append({
"text": "What is a painting?",
"eval_labels": ["Paint on a canvas."]
})
observations.append({})
observations.append({})
observations.append({
"text": "What is a painting?",
"eval_labels": ["Paint on a canvas."]
})
observations.append({})
observations.append({})
vec_observations = [agent.vectorize(obs) for obs in observations]
mapped_valid = agent.map_valid(vec_observations)
text_vecs, label_vecs, labels, valid_inds = mapped_valid
self.assertTrue(label_vecs is not None, "Missing \'label_vec\' field.")
self.assertTrue(
label_vecs.numpy().tolist() == [[1, 3, 5, 2], [1, 3, 5, 2]],
"Incorrectly vectorized text field of obs_batch.")
predictions = ["Oil on a canvas.", "Oil on a canvas."]
expected_unmapped = [
"Oil on a canvas.", None, None, "Oil on a canvas.", None, None
]
self.assertTrue(
agent.unmap_valid(predictions, valid_inds, 6) == expected_unmapped,
"Unmapped predictions do not match expected results.")
def add_cmdline_args(cls, argparser):
"""Add command-line arguments specifically for this agent."""
# first we need to add the general torch agent operations
TorchAgent.add_cmdline_args(argparser)
agent = argparser.add_argument_group('Fairseq Arguments')
agent.add_argument('--fp16',
default=False,
type=bool,
help='Use fp16 training')
agent.add_argument('--seed',
default=1,
type=int,
metavar='N',
help='pseudo random number generator seed')
agent.add_argument(
'--skip-generation',
default=False,
type=bool,
metavar='BOOL',
help=
'Skips test time beam search. Much faster if you only need PPL',
)
# Dictionary construction stuff. Using the subclass in case we end up
# needing any fairseq specific things
cls.dictionary_class().add_cmdline_args(argparser)
# Check subargs for generation, optimizers, criterions, archs, etc
options.add_generation_args(argparser)
options.add_optimization_args(argparser)
# make sure we set defaults according to the model before parsing
argparser.set_defaults(**cls.DEFAULT_OPTIONS)
known_args = argparser.parse_known_args(nohelp=True)[0]
if hasattr(known_args, "optimizer"):
optimizer = known_args.optimizer
opt_group = argparser.add_argument_group(
'{} optimizer arguments'.format(optimizer))
optim.OPTIMIZER_REGISTRY[optimizer].add_args(opt_group)
if hasattr(known_args, "lr_scheduler"):
lr_scheduler = known_args.lr_scheduler
lr_group = argparser.add_argument_group(
'{} scheduler arguments'.format(lr_scheduler))
optim.lr_scheduler.LR_SCHEDULER_REGISTRY[lr_scheduler].add_args(
lr_group)
# We need to find out the fairseq model-specific options, so grab the
# architecture stuff and look up its options
arch_group = options.add_model_args(argparser)
# Fairseq marks the arch flag as required, but it may be specified
# by a saved model cache, so we do some weird stuff to undo that
for a in arch_group._actions:
if a.dest == "arch":
a.required = False
a.default = None
break
# make sure we set defaults according to parlai model before parsing
argparser.set_defaults(**cls.DEFAULT_OPTIONS)
known_args = argparser.parse_known_args(nohelp=True)[0]
if hasattr(known_args, "arch") and known_args.arch is not None:
arch = known_args.arch
arch_group = argparser.add_argument_group(
"{} architecture arguments".format(arch))
models.ARCH_MODEL_REGISTRY[arch].add_args(arch_group)
if hasattr(known_args, "criterion"):
crit_group = argparser.add_argument_group(
'{} criterion arguments'.format(known_args.criterion))
criterions.CRITERION_REGISTRY[known_args.criterion].add_args(
crit_group)
# As one final check, let's make sure we set defaults correctly
argparser.set_defaults(**cls.DEFAULT_OPTIONS)
def test_map_unmap(self):
try:
from parlai.core.torch_agent import TorchAgent
except ImportError as e:
if 'pytorch' in e.msg:
print('Skipping TestTorchAgent.test_map_unmap, no pytorch.')
return
observations = []
observations.append({
"text": "What is a painting?",
"labels": ["Paint on a canvas."]
})
observations.append({})
observations.append({})
observations.append({
"text": "What is a painting?",
"labels": ["Paint on a canvas."]
})
observations.append({})
observations.append({})
opt = {}
opt['no_cuda'] = True
opt['truncate'] = 10000
opt['history_dialog'] = 10
opt['history_replies'] = 'label_else_model'
mdict = MockDict()
shared = {'opt': opt, 'dict': mdict}
agent = TorchAgent(opt, shared)
vec_observations = [agent.vectorize(obs) for obs in observations]
mapped_valid = agent.map_valid(vec_observations)
text_vecs, text_lengths, label_vecs, labels, valid_inds = mapped_valid
self.assertTrue(text_vecs is not None, "Missing \'text_vecs\' field.")
self.assertTrue(text_vecs.numpy().tolist() == [[7, 8, 9], [7, 8, 9]],
"Incorrectly vectorized text field of obs_batch.")
self.assertTrue(text_lengths.numpy().tolist() == [3, 3],
"Incorrect text vector lengths returned.")
self.assertTrue(label_vecs is not None, "Missing \'label_vec\' field.")
self.assertTrue(
label_vecs.numpy().tolist() == [[
mdict.START_IDX, 7, 8, 9, mdict.END_IDX
], [mdict.START_IDX, 7, 8, 9, mdict.END_IDX]],
"Incorrectly vectorized text field of obs_batch.")
self.assertTrue(labels == ["Paint on a canvas.", "Paint on a canvas."],
"Doesn't return correct labels.")
self.assertTrue(valid_inds == [0, 3],
"Returns incorrect indices of valid observations.")
observations = []
observations.append({
"text": "What is a painting?",
"eval_labels": ["Paint on a canvas."]
})
observations.append({})
observations.append({})
observations.append({
"text": "What is a painting?",
"eval_labels": ["Paint on a canvas."]
})
observations.append({})
observations.append({})
vec_observations = [agent.vectorize(obs) for obs in observations]
mapped_valid = agent.map_valid(vec_observations)
text_vecs, text_lengths, label_vecs, labels, valid_inds = mapped_valid
self.assertTrue(label_vecs is not None,
"Missing \'eval_label_vec\' field.")
self.assertTrue(
label_vecs.numpy().tolist() == [[
mdict.START_IDX, 7, 8, 9, mdict.END_IDX
], [mdict.START_IDX, 7, 8, 9, mdict.END_IDX]],
"Incorrectly vectorized text field of obs_batch.")
predictions = ["Oil on a canvas.", "Oil on a canvas."]
expected_unmapped = [
"Oil on a canvas.", None, None, "Oil on a canvas.", None, None
]
self.assertTrue(
agent.unmap_valid(predictions, valid_inds, 6) == expected_unmapped,
"Unmapped predictions do not match expected results.")
