def setUp(self):
events.clear()
ParamManager.init_param_col()
# Load a pre-trained model
load_experiment = LoadSerialized(filename=f"test/data/tiny_jaen.model",
overwrite=[
{
"path": "train",
"val": None
},
{
"path": "status",
"val": None
},
])
EXP_DIR = '.'
EXP = "decode"
uninitialized_experiment = YamlPreloader.preload_obj(load_experiment,
exp_dir=EXP_DIR,
exp_name=EXP)
loaded_experiment = initialize_if_needed(uninitialized_experiment)
ParamManager.populate()
# Pull out the parts we need from the experiment
self.model = loaded_experiment.model
src_vocab = self.model.src_reader.vocab
trg_vocab = self.model.trg_reader.vocab
event_trigger.set_train(False)
self.src_data = list(
self.model.src_reader.read_sents("test/data/head.ja"))
self.trg_data = list(
self.model.trg_reader.read_sents("test/data/head.en"))
def setUp(self):
layer_dim = 512
events.clear()
ParamManager.init_param_col()
src_vocab = Vocab(vocab_file="examples/data/head.ja.vocab")
trg_vocab = Vocab(vocab_file="examples/data/head.en.vocab")
self.model = DefaultTranslator(
src_reader=PlainTextReader(vocab=src_vocab),
trg_reader=PlainTextReader(vocab=trg_vocab),
src_embedder=SimpleWordEmbedder(emb_dim=layer_dim, vocab_size=100),
encoder=BiLSTMSeqTransducer(input_dim=layer_dim,
hidden_dim=layer_dim),
attender=MlpAttender(input_dim=layer_dim,
state_dim=layer_dim,
hidden_dim=layer_dim),
decoder=AutoRegressiveDecoder(
input_dim=layer_dim,
embedder=SimpleWordEmbedder(emb_dim=layer_dim, vocab_size=100),
rnn=UniLSTMSeqTransducer(input_dim=layer_dim,
hidden_dim=layer_dim,
decoder_input_dim=layer_dim,
yaml_path="model.decoder.rnn"),
transform=NonLinear(input_dim=layer_dim * 2,
output_dim=layer_dim),
scorer=Softmax(input_dim=layer_dim, vocab_size=100),
bridge=CopyBridge(dec_dim=layer_dim, dec_layers=1)),
)
event_trigger.set_train(False)
self.src_data = list(
self.model.src_reader.read_sents("examples/data/head.ja"))
def setUp(self):
xnmt.events.clear()
ParamManager.init_param_col()
self.src_reader = PlainTextReader(vocab=Vocab(vocab_file="examples/data/head.ja.vocab"))
self.trg_reader = PlainTextReader(vocab=Vocab(vocab_file="examples/data/head.en.vocab"))
self.src_data = list(self.src_reader.read_sents("examples/data/head.ja"))
self.trg_data = list(self.trg_reader.read_sents("examples/data/head.en"))
def setUp(self):
# Seeding
numpy.random.seed(2)
random.seed(2)
layer_dim = 4
xnmt.events.clear()
ParamManager.init_param_col()
self.segment_encoder_bilstm = BiLSTMSeqTransducer(input_dim=layer_dim, hidden_dim=layer_dim)
self.segment_composer = SumComposer()
self.src_reader = CharFromWordTextReader(vocab=Vocab(vocab_file="test/data/head.ja.charvocab"))
self.trg_reader = PlainTextReader(vocab=Vocab(vocab_file="test/data/head.en.vocab"))
self.loss_calculator = FeedbackLoss(child_loss=MLELoss(), repeat=5)
baseline = Linear(input_dim=layer_dim, output_dim=1)
policy_network = Linear(input_dim=layer_dim, output_dim=2)
self.poisson_prior = PoissonPrior(mu=3.3)
self.eps_greedy = EpsilonGreedy(eps_prob=0.0, prior=self.poisson_prior)
self.conf_penalty = ConfidencePenalty()
self.policy_gradient = PolicyGradient(input_dim=layer_dim,
output_dim=2,
baseline=baseline,
policy_network=policy_network,
z_normalization=True,
conf_penalty=self.conf_penalty)
self.length_prior = PoissonLengthPrior(lmbd=3.3, weight=1)
self.segmenting_encoder = SegmentingSeqTransducer(
embed_encoder = self.segment_encoder_bilstm,
segment_composer = self.segment_composer,
final_transducer = BiLSTMSeqTransducer(input_dim=layer_dim, hidden_dim=layer_dim),
policy_learning = self.policy_gradient,
eps_greedy = self.eps_greedy,
length_prior = self.length_prior,
)
self.model = DefaultTranslator(
src_reader=self.src_reader,
trg_reader=self.trg_reader,
src_embedder=SimpleWordEmbedder(emb_dim=layer_dim, vocab_size=100),
encoder=self.segmenting_encoder,
attender=MlpAttender(input_dim=layer_dim, state_dim=layer_dim, hidden_dim=layer_dim),
decoder=AutoRegressiveDecoder(input_dim=layer_dim,
rnn=UniLSTMSeqTransducer(input_dim=layer_dim, hidden_dim=layer_dim,
decoder_input_dim=layer_dim, yaml_path="decoder"),
transform=AuxNonLinear(input_dim=layer_dim, output_dim=layer_dim,
aux_input_dim=layer_dim),
scorer=Softmax(vocab_size=100, input_dim=layer_dim),
embedder=SimpleWordEmbedder(emb_dim=layer_dim, vocab_size=100),
bridge=CopyBridge(dec_dim=layer_dim, dec_layers=1)),
)
event_trigger.set_train(True)
self.layer_dim = layer_dim
self.src_data = list(self.model.src_reader.read_sents("test/data/head.ja"))
self.trg_data = list(self.model.trg_reader.read_sents("test/data/head.en"))
my_batcher = batchers.TrgBatcher(batch_size=3)
self.src, self.trg = my_batcher.pack(self.src_data, self.trg_data)
dy.renew_cg(immediate_compute=True, check_validity=True)
def setUp(self):
# Seeding
numpy.random.seed(2)
random.seed(2)
layer_dim = 32
xnmt.events.clear()
ParamManager.init_param_col()
edge_vocab = Vocab(vocab_file="examples/data/parse/head.en.edge_vocab")
node_vocab = Vocab(vocab_file="examples/data/parse/head.en.node_vocab")
value_vocab = Vocab(vocab_file="examples/data/head.en.vocab")
self.src_reader = input_readers.PlainTextReader(vocab=value_vocab)
self.trg_reader = input_readers.CoNLLToRNNGActionsReader(
surface_vocab=value_vocab,
nt_vocab=node_vocab,
edg_vocab=edge_vocab)
self.layer_dim = layer_dim
self.src_data = list(
self.src_reader.read_sents("examples/data/head.en"))
self.trg_data = list(
self.trg_reader.read_sents("examples/data/parse/head.en.conll"))
self.loss_calculator = MLELoss()
self.head_composer = composer.DyerHeadComposer(
fwd_combinator=UniLSTMSeqTransducer(input_dim=layer_dim,
hidden_dim=layer_dim),
bwd_combinator=UniLSTMSeqTransducer(input_dim=layer_dim,
hidden_dim=layer_dim),
transform=AuxNonLinear(input_dim=layer_dim,
aux_input_dim=layer_dim,
output_dim=layer_dim))
self.model = DefaultTranslator(
src_reader=self.src_reader,
trg_reader=self.trg_reader,
src_embedder=LookupEmbedder(emb_dim=layer_dim,
vocab_size=len(value_vocab)),
encoder=IdentitySeqTransducer(),
attender=MlpAttender(input_dim=layer_dim,
state_dim=layer_dim,
hidden_dim=layer_dim),
decoder=RNNGDecoder(
input_dim=layer_dim,
rnn=UniLSTMSeqTransducer(input_dim=layer_dim,
hidden_dim=layer_dim,
decoder_input_dim=layer_dim),
transform=AuxNonLinear(input_dim=layer_dim,
output_dim=layer_dim,
aux_input_dim=layer_dim),
bridge=NoBridge(dec_dim=layer_dim, dec_layers=1),
graph_reader=self.trg_reader,
head_composer=self.head_composer))
event_trigger.set_train(True)
my_batcher = batchers.TrgBatcher(batch_size=1)
self.src, self.trg = my_batcher.pack(self.src_data, self.trg_data)
dy.renew_cg(immediate_compute=True, check_validity=True)
def setUp(self):
events.clear()
ParamManager.init_param_col()
src_vocab = Vocab(vocab_file="test/data/head.ja.vocab")
trg_vocab = Vocab(vocab_file="test/data/head.en.vocab")
self.src_reader = PlainTextReader(vocab=src_vocab)
self.trg_reader = PlainTextReader(vocab=trg_vocab)
self.src_data = list(self.src_reader.read_sents("test/data/head.ja"))
self.trg_data = list(self.trg_reader.read_sents("test/data/head.en"))
def setUp(self):
# Seeding
numpy.random.seed(2)
random.seed(2)
layer_dim = 4
xnmt.events.clear()
ParamManager.init_param_col()
self.segment_composer = SumComposer()
self.src_reader = CharFromWordTextReader(vocab=Vocab(
vocab_file="examples/data/head.ja.charvocab"))
self.trg_reader = PlainTextReader(vocab=Vocab(
vocab_file="examples/data/head.en.vocab"))
self.loss_calculator = FeedbackLoss(child_loss=MLELoss(), repeat=5)
self.segmenting_encoder = SegmentingSeqTransducer(
segment_composer=self.segment_composer,
final_transducer=BiLSTMSeqTransducer(input_dim=layer_dim,
hidden_dim=layer_dim),
)
self.model = DefaultTranslator(
src_reader=self.src_reader,
trg_reader=self.trg_reader,
src_embedder=SimpleWordEmbedder(emb_dim=layer_dim, vocab_size=100),
encoder=self.segmenting_encoder,
attender=MlpAttender(input_dim=layer_dim,
state_dim=layer_dim,
hidden_dim=layer_dim),
decoder=AutoRegressiveDecoder(
input_dim=layer_dim,
rnn=UniLSTMSeqTransducer(input_dim=layer_dim,
hidden_dim=layer_dim,
decoder_input_dim=layer_dim,
yaml_path="decoder"),
transform=AuxNonLinear(input_dim=layer_dim,
output_dim=layer_dim,
aux_input_dim=layer_dim),
scorer=Softmax(vocab_size=100, input_dim=layer_dim),
embedder=SimpleWordEmbedder(emb_dim=layer_dim, vocab_size=100),
bridge=CopyBridge(dec_dim=layer_dim, dec_layers=1)),
)
event_trigger.set_train(True)
self.layer_dim = layer_dim
self.src_data = list(
self.model.src_reader.read_sents("examples/data/head.ja"))
self.trg_data = list(
self.model.trg_reader.read_sents("examples/data/head.en"))
my_batcher = batchers.TrgBatcher(batch_size=3)
self.src, self.trg = my_batcher.pack(self.src_data, self.trg_data)
dy.renew_cg(immediate_compute=True, check_validity=True)
def setUp(self):
# Seeding
numpy.random.seed(2)
random.seed(2)
layer_dim = 32
xnmt.events.clear()
ParamManager.init_param_col()
self.src_reader = PlainTextReader(vocab=Vocab(
vocab_file="test/data/head.ja.vocab"))
self.trg_reader = PlainTextReader(vocab=Vocab(
vocab_file="test/data/head.en.vocab"))
self.layer_dim = layer_dim
self.src_data = list(self.src_reader.read_sents("test/data/head.ja"))
self.trg_data = list(self.trg_reader.read_sents("test/data/head.en"))
self.input_vocab_size = len(self.src_reader.vocab.i2w)
self.output_vocab_size = len(self.trg_reader.vocab.i2w)
self.loss_calculator = MLELoss()
self.model = SimultaneousTranslator(
src_reader=self.src_reader,
trg_reader=self.trg_reader,
src_embedder=SimpleWordEmbedder(emb_dim=layer_dim,
vocab_size=self.input_vocab_size),
encoder=UniLSTMSeqTransducer(input_dim=layer_dim,
hidden_dim=layer_dim),
attender=MlpAttender(input_dim=layer_dim,
state_dim=layer_dim,
hidden_dim=layer_dim),
decoder=AutoRegressiveDecoder(
input_dim=layer_dim,
rnn=UniLSTMSeqTransducer(input_dim=layer_dim,
hidden_dim=layer_dim,
decoder_input_dim=layer_dim,
yaml_path="decoder"),
transform=AuxNonLinear(input_dim=layer_dim,
output_dim=layer_dim,
aux_input_dim=layer_dim),
scorer=Softmax(vocab_size=self.output_vocab_size,
input_dim=layer_dim),
embedder=SimpleWordEmbedder(emb_dim=layer_dim,
vocab_size=self.output_vocab_size),
bridge=NoBridge(dec_dim=layer_dim, dec_layers=1)),
)
event_trigger.set_train(True)
my_batcher = batchers.TrgBatcher(batch_size=3)
self.src, self.trg = my_batcher.pack(self.src_data, self.trg_data)
dy.renew_cg(immediate_compute=True, check_validity=True)
def setUp(self):
# Seeding
numpy.random.seed(2)
random.seed(2)
layer_dim = 32
xnmt.events.clear()
ParamManager.init_param_col()
src_vocab = Vocab(vocab_file="examples/data/head.ja.vocab")
self.src_reader = CompoundReader(readers=[
PlainTextReader(vocab=src_vocab),
SimultActionTextReader()
], vocab=src_vocab)
self.trg_reader = PlainTextReader(vocab=Vocab(vocab_file="examples/data/head.en.vocab"))
self.layer_dim = layer_dim
self.src_data = list(self.src_reader.read_sents(["examples/data/head.ja", "examples/data/simult/head.jaen.actions"]))
self.trg_data = list(self.trg_reader.read_sents("examples/data/head.en"))
self.input_vocab_size = len(self.src_reader.vocab.i2w)
self.output_vocab_size = len(self.trg_reader.vocab.i2w)
self.loss_calculator = loss_calculators.MLELoss()
self.model = SimultaneousTranslator(
src_reader=self.src_reader,
trg_reader=self.trg_reader,
src_embedder=LookupEmbedder(emb_dim=layer_dim, vocab_size=self.input_vocab_size),
encoder=UniLSTMSeqTransducer(input_dim=layer_dim, hidden_dim=layer_dim),
attender=MlpAttender(input_dim=layer_dim, state_dim=layer_dim, hidden_dim=layer_dim),
decoder=AutoRegressiveDecoder(input_dim=layer_dim,
rnn=UniLSTMSeqTransducer(input_dim=layer_dim, hidden_dim=layer_dim,
decoder_input_dim=layer_dim, yaml_path="decoder"),
transform=AuxNonLinear(input_dim=layer_dim, output_dim=layer_dim,
aux_input_dim=layer_dim),
scorer=Softmax(vocab_size=self.output_vocab_size, input_dim=layer_dim),
embedder=LookupEmbedder(emb_dim=layer_dim, vocab_size=self.output_vocab_size),
bridge=NoBridge(dec_dim=layer_dim, dec_layers=1)),
policy_network = network.PolicyNetwork(transforms.MLP(2*self.layer_dim, self.layer_dim, 2)),
policy_train_oracle=True,
policy_test_oracle=True
)
event_trigger.set_train(True)
my_batcher = batchers.TrgBatcher(batch_size=3)
self.src, self.trg = my_batcher.pack(self.src_data, self.trg_data)
dy.renew_cg(immediate_compute=True, check_validity=True)
def setUp(self):
# Seeding
np.random.seed(2)
random.seed(2)
layer_dim = 4
xnmt.events.clear()
ParamManager.init_param_col()
self.src_vocab = Vocab(vocab_file="examples/data/head.ja.vocab")
self.src_char_vocab = CharVocab(vocab_file="examples/data/head.ja.vocab")
self.ngram_vocab = Vocab(vocab_file="examples/data/head.ngramcount.ja")
self.trg_vocab = Vocab(vocab_file="examples/data/head.en.vocab")
self.src_reader = CharFromWordTextReader(vocab= self.src_vocab, char_vocab= self.src_char_vocab)
self.trg_reader = PlainTextReader(vocab=self.trg_vocab)
self.layer_dim = layer_dim
self.src_data = list(self.src_reader.read_sents("examples/data/head.ja"))
self.trg_data = list(self.trg_reader.read_sents("examples/data/head.en"))
self.src, self.trg = batchers.TrgBatcher(batch_size=3).pack(self.src_data, self.trg_data)
dy.renew_cg(immediate_compute=True, check_validity=True)
def run(self):
seed = 13
random.seed(seed)
np.random.seed(seed)
EXP_DIR = os.path.dirname(__file__)
EXP = "annot"
model_file = f"{EXP_DIR}/results/{EXP}.mod"
log_file = f"{EXP_DIR}/results/{EXP}.log"
xnmt.tee.utils.dy.DynetParams().set_mem(
1024) #Doesnt work figure out how to set memory
xnmt.tee.set_out_file(log_file, exp_name=EXP)
ParamManager.init_param_col()
ParamManager.param_col.model_file = model_file
pre_runner = PreprocRunner(
tasks=[
PreprocTokenize(
in_files=
[ #f'{EXP_DIR}/conala-corpus/conala-trainnodev.snippet',
#f'{EXP_DIR}/conala-corpus/conala-trainnodev.intent',
#f'{EXP_DIR}/conala-corpus/conala-dev.intent',
#f'{EXP_DIR}/conala-corpus/conala-dev.snippet',
#f'{EXP_DIR}/conala-corpus/conala-test.intent',
#f'{EXP_DIR}/conala-corpus/conala-test.snippet',
f'{EXP_DIR}/conala-corpus/attack_code_train.txt',
f'{EXP_DIR}/conala-corpus/attack_text_train.txt',
f'{EXP_DIR}/conala-corpus/attack_code_test.txt',
f'{EXP_DIR}/conala-corpus/attack_text_test.txt'
#f'{EXP_DIR}/conala-corpus/all.code',
#f'{EXP_DIR}/conala-corpus/all.anno'
],
out_files=
[ #f'{EXP_DIR}/conala-corpus/conala-trainnodev.tmspm4000.snippet',
#f'{EXP_DIR}/conala-corpus/conala-trainnodev.tmspm4000.intent',
#f'{EXP_DIR}/conala-corpus/conala-dev.tmspm4000.intent',
#f'{EXP_DIR}/conala-corpus/conala-dev.tmspm4000.snippet',
#f'{EXP_DIR}/conala-corpus/conala-test.tmspm4000.intent',
#f'{EXP_DIR}/conala-corpus/conala-test.tmspm4000.snippet',
f'{EXP_DIR}/conala-corpus/attack-train.tmspm4000.snippet',
f'{EXP_DIR}/conala-corpus/attack-train.tmspm4000.intent',
f'{EXP_DIR}/conala-corpus/attack-test.tmspm4000.snippet',
f'{EXP_DIR}/conala-corpus/attack-test.tmspm4000.intent'
#f'{EXP_DIR}/conala-corpus/django.tmspm4000.snippet',
#f'{EXP_DIR}/conala-corpus/django.tmspm4000.intent'
],
specs=[{
'filenum':
'all',
'tokenizers': [
SentencepieceTokenizer(
hard_vocab_limit=False,
train_files=[
f'{EXP_DIR}/conala-corpus/attack_text_train.txt',
f'{EXP_DIR}/conala-corpus/attack_code_train.txt'
],
vocab_size=self.vocab_size,
model_type=self.model_type,
model_prefix=
'conala-corpus/attack-train.tmspm4000.spm')
]
}]),
PreprocVocab(
in_files=[
f'{EXP_DIR}/conala-corpus/attack-train.tmspm4000.intent',
f'{EXP_DIR}/conala-corpus/attack-train.tmspm4000.snippet'
],
out_files
=[
f'{EXP_DIR}/conala-corpus/attack-train.tmspm4000.intent.vocab',
f'{EXP_DIR}/conala-corpus/attack-train.tmspm4000.snippet.vocab'
],
specs=[{
'filenum':
'all',
'filters': [VocabFiltererFreq(min_freq=self.min_freq)]
}])
],
overwrite=False)
src_vocab = Vocab(
vocab_file=
f"{EXP_DIR}/conala-corpus/attack-train.tmspm4000.intent.vocab")
trg_vocab = Vocab(
vocab_file=
f"{EXP_DIR}/conala-corpus/attack-train.tmspm4000.snippet.vocab")
batcher = Batcher(batch_size=64)
inference = AutoRegressiveInference(search_strategy=BeamSearch(
len_norm=PolynomialNormalization(apply_during_search=True),
beam_size=5),
post_process='join-piece')
layer_dim = self.layer_dim
model = DefaultTranslator(
src_reader=PlainTextReader(vocab=src_vocab),
trg_reader=PlainTextReader(vocab=trg_vocab),
src_embedder=SimpleWordEmbedder(emb_dim=layer_dim,
vocab=src_vocab),
encoder=BiLSTMSeqTransducer(input_dim=layer_dim,
hidden_dim=layer_dim,
layers=self.layers),
attender=MlpAttender(hidden_dim=layer_dim,
state_dim=layer_dim,
input_dim=layer_dim),
trg_embedder=SimpleWordEmbedder(emb_dim=layer_dim,
vocab=trg_vocab),
decoder=AutoRegressiveDecoder(
input_dim=layer_dim,
rnn=UniLSTMSeqTransducer(
input_dim=layer_dim,
hidden_dim=layer_dim,
),
transform=AuxNonLinear(input_dim=layer_dim,
output_dim=layer_dim,
aux_input_dim=layer_dim),
scorer=Softmax(vocab_size=len(trg_vocab), input_dim=layer_dim),
trg_embed_dim=layer_dim,
input_feeding=False,
bridge=CopyBridge(dec_dim=layer_dim)),
inference=inference)
#decoder = AutoRegressiveDecoder(bridge=CopyBridge(),inference=inference))
train = SimpleTrainingRegimen(
name=f"{EXP}",
model=model,
batcher=WordSrcBatcher(avg_batch_size=64),
trainer=AdamTrainer(alpha=self.alpha),
patience=3,
lr_decay=0.5,
restart_trainer=True,
run_for_epochs=self.epochs,
src_file=f"{EXP_DIR}/conala-corpus/attack-train.tmspm4000.intent",
trg_file=f"{EXP_DIR}/conala-corpus/attack-train.tmspm4000.snippet",
dev_tasks=[
LossEvalTask(
src_file=
f"{EXP_DIR}/conala-corpus/attack-test.tmspm4000.intent",
ref_file=
f'{EXP_DIR}/conala-corpus/attack-test.tmspm4000.snippet',
model=model,
batcher=WordSrcBatcher(avg_batch_size=64)),
AccuracyEvalTask(
eval_metrics='bleu',
src_file=
f'{EXP_DIR}/conala-corpus/attack-test.tmspm4000.intent',
ref_file=f'{EXP_DIR}/conala-corpus/attack_text_test.txt',
hyp_file=f'results/{EXP}.dev.hyp',
model=model)
])
evaluate = [
AccuracyEvalTask(
eval_metrics="bleu",
#src_file=f"{EXP_DIR}/conala-corpus/conala-test.tmspm4000.intent",
src_file=
f"{EXP_DIR}/conala-corpus/attack-test.tmspm4000.intent",
#ref_file=f"{EXP_DIR}/conala-corpus/all.code",
#ref_file = f"{EXP_DIR}/conala-corpus/conala-test.snippet",
ref_file=f"{EXP_DIR}/conala-corpus/attack_text_test.txt",
hyp_file=f"results/{EXP}.test.hyp",
inference=inference,
model=model)
]
standard_experiment = Experiment(exp_global=ExpGlobal(
default_layer_dim=512,
dropout=0.3,
log_file=log_file,
model_file=model_file),
name="annot",
model=model,
train=train,
evaluate=evaluate)
# run experiment
standard_experiment(
save_fct=lambda: save_to_file(model_file, standard_experiment))
exit()
def setUp(self):
xnmt.events.clear()
ParamManager.init_param_col()
from xnmt.models.translators.default import DefaultTranslator
from xnmt.vocabs import Vocab
seed = 13
random.seed(seed)
np.random.seed(seed)
EXP_DIR = os.path.dirname(__file__)
EXP = "programmatic"
model_file = f"{EXP_DIR}/models/{EXP}.mod"
log_file = f"{EXP_DIR}/logs/{EXP}.log"
xnmt.tee.set_out_file(log_file, EXP)
ParamManager.init_param_col()
ParamManager.param_col.model_file = model_file
src_vocab = Vocab(vocab_file="examples/data/head.ja.vocab")
trg_vocab = Vocab(vocab_file="examples/data/head.en.vocab")
batcher = SrcBatcher(batch_size=64)
inference = AutoRegressiveInference(batcher=InOrderBatcher(batch_size=1))
layer_dim = 512
model = DefaultTranslator(
src_reader=PlainTextReader(vocab=src_vocab),
trg_reader=PlainTextReader(vocab=trg_vocab),
src_embedder=SimpleWordEmbedder(emb_dim=layer_dim,
def main(overwrite_args: Optional[Sequence[str]] = None) -> None:
with tee.Tee(), tee.Tee(error=True):
argparser = argparse.ArgumentParser()
utils.add_backend_argparse(argparser)
argparser.add_argument("--settings",
type=str,
default="standard",
help="settings (standard, debug, or unittest)"
"must be given in '=' syntax, e.g."
" --settings=standard")
argparser.add_argument(
"--resume",
action='store_true',
help="whether a saved experiment is being resumed, and"
"locations of output files should be re-used.")
argparser.add_argument("--backend",
type=str,
default="dynet",
help="backend (dynet or torch)")
argparser.add_argument("experiments_file")
argparser.add_argument("experiment_name",
nargs='*',
help="Run only the specified experiments")
argparser.set_defaults(generate_doc=False)
args = argparser.parse_args(overwrite_args)
if xnmt.backend_dynet and args.dynet_seed: args.seed = args.dynet_seed
if getattr(args, "seed", None):
random.seed(args.seed)
np.random.seed(args.seed)
if xnmt.backend_torch: torch.manual_seed(0)
if xnmt.backend_dynet and args.dynet_gpu and settings.CHECK_VALIDITY:
settings.CHECK_VALIDITY = False
log_preamble(
"disabling CHECK_VALIDITY because it is not supported in the DyNet/GPU setting",
logging.WARNING)
config_experiment_names = YamlPreloader.experiment_names_from_file(
args.experiments_file)
results = []
# Check ahead of time that all experiments exist, to avoid bad surprises
experiment_names = args.experiment_name or config_experiment_names
if args.experiment_name:
nonexistent = set(experiment_names).difference(
config_experiment_names)
if len(nonexistent) != 0:
raise Exception("Experiments {} do not exist".format(",".join(
list(nonexistent))))
log_preamble(
f"running XNMT revision {tee.get_git_revision()} on {socket.gethostname()} with {'DyNet' if xnmt.backend_dynet else 'PyTorch'} on {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
)
for experiment_name in experiment_names:
ParamManager.init_param_col()
uninitialized_exp_args = YamlPreloader.preload_experiment_from_file(
args.experiments_file, experiment_name, resume=args.resume)
logger.info(f"=> Running {experiment_name}")
glob_args = uninitialized_exp_args.data.exp_global
log_file = glob_args.log_file
if not settings.OVERWRITE_LOG:
log_files_exist = []
if os.path.isfile(log_file): log_files_exist.append(log_file)
if os.path.isdir(log_file + ".tb"):
log_files_exist.append(log_file + ".tb/")
if log_files_exist:
logger.warning(
f"log file(s) {' '.join(log_files_exist)} already exists, skipping experiment; "
f"please delete log file by hand if you want to overwrite it "
f"(or activate OVERWRITE_LOG, by either specifying an environment variable OVERWRITE_LOG=1, "
f"or specifying --settings=debug, or changing xnmt.settings.Standard.OVERWRITE_LOG manually)"
)
continue
elif settings.OVERWRITE_LOG and os.path.isdir(log_file + ".tb"):
shutil.rmtree(
log_file + ".tb/"
) # remove tensorboard logs from previous run that is being overwritten
tee.set_out_file(log_file, exp_name=experiment_name)
try:
model_file = glob_args.model_file
uninitialized_exp_args.data.exp_global.commandline_args = vars(
args)
# Create the model
experiment = initialize_if_needed(uninitialized_exp_args)
ParamManager.param_col.model_file = experiment.exp_global.model_file
ParamManager.param_col.save_num_checkpoints = experiment.exp_global.save_num_checkpoints
ParamManager.populate()
# Run the experiment
eval_scores = experiment(
save_fct=lambda: save_to_file(model_file, experiment))
results.append((experiment_name, eval_scores))
print_results(results)
except Exception as e:
file_logger.error(traceback.format_exc())
raise e
finally:
tee.unset_out_file()
def setUp(self):
events.clear()
self.input_reader = PlainTextReader(vocab=Vocab(vocab_file="examples/data/head.ja.vocab"))
list(self.input_reader.read_sents('examples/data/head.ja'))
ParamManager.init_param_col()