def setUp(self):
layer_dim = 512
xnmt.events.clear()
ParamManager.init_param_col()
self.model = DefaultTranslator(
src_reader=PlainTextReader(),
trg_reader=PlainTextReader(),
src_embedder=SimpleWordEmbedder(emb_dim=layer_dim, vocab_size=100),
encoder=BiLSTMSeqTransducer(input_dim=layer_dim,
hidden_dim=layer_dim),
attender=MlpAttender(input_dim=layer_dim,
state_dim=layer_dim,
hidden_dim=layer_dim),
trg_embedder=SimpleWordEmbedder(emb_dim=layer_dim, vocab_size=100),
decoder=AutoRegressiveDecoder(
input_dim=layer_dim,
trg_embed_dim=layer_dim,
rnn=UniLSTMSeqTransducer(input_dim=layer_dim,
hidden_dim=layer_dim,
decoder_input_dim=layer_dim,
yaml_path="model.decoder.rnn"),
transform=NonLinear(input_dim=layer_dim * 2,
output_dim=layer_dim),
scorer=Softmax(input_dim=layer_dim, vocab_size=100),
bridge=CopyBridge(dec_dim=layer_dim, dec_layers=1)),
)
self.model.set_train(False)
self.src_data = list(
self.model.src_reader.read_sents("examples/data/head.ja"))
self.trg_data = list(
self.model.trg_reader.read_sents("examples/data/head.en"))
def setUp(self):
layer_dim = 512
xnmt.events.clear()
ParamManager.init_param_col()
self.model = DefaultTranslator(
src_reader=PlainTextReader(),
trg_reader=PlainTextReader(),
src_embedder=SimpleWordEmbedder(emb_dim=layer_dim, vocab_size=100),
encoder=BiLSTMSeqTransducer(input_dim=layer_dim,
hidden_dim=layer_dim),
attender=MlpAttender(input_dim=layer_dim,
state_dim=layer_dim,
hidden_dim=layer_dim),
trg_embedder=SimpleWordEmbedder(emb_dim=layer_dim, vocab_size=100),
decoder=MlpSoftmaxDecoder(input_dim=layer_dim,
lstm_dim=layer_dim,
mlp_hidden_dim=layer_dim,
trg_embed_dim=layer_dim,
vocab_size=100,
bridge=CopyBridge(dec_dim=layer_dim,
dec_layers=1)),
)
self.model.set_train(False)
self.model.initialize_generator()
self.src_data = list(
self.model.src_reader.read_sents("examples/data/head.ja"))
self.trg_data = list(
self.model.trg_reader.read_sents("examples/data/head.en"))
def setUp(self):
xnmt.events.clear()
ParamManager.init_param_col()
self.src_reader = PlainTextReader()
self.trg_reader = PlainTextReader()
self.src_data = list(self.src_reader.read_sents("examples/data/head.ja"))
self.trg_data = list(self.trg_reader.read_sents("examples/data/head.en"))
def main(overwrite_args=None):
with tee.Tee(), tee.Tee(error=True):
argparser = argparse.ArgumentParser()
argparser.add_argument("--dynet-mem", type=str)
argparser.add_argument("--dynet-seed",
type=int,
help="set random seed for DyNet and XNMT.")
argparser.add_argument("--dynet-autobatch", type=int)
argparser.add_argument("--dynet-devices", type=str)
argparser.add_argument("--dynet-viz",
action='store_true',
help="use visualization")
argparser.add_argument("--dynet-gpu",
action='store_true',
help="use GPU acceleration")
argparser.add_argument("--dynet-gpu-ids", type=int)
argparser.add_argument("--dynet-gpus", type=int)
argparser.add_argument("--dynet-weight-decay", type=float)
argparser.add_argument("--dynet-profiling", type=int)
argparser.add_argument("--settings",
type=str,
default="standard",
help="settings (standard, debug, or unittest)"
"must be given in '=' syntax, e.g."
" --settings=standard")
argparser.add_argument("experiments_file")
argparser.add_argument("experiment_name",
nargs='*',
help="Run only the specified experiments")
argparser.set_defaults(generate_doc=False)
args = argparser.parse_args(overwrite_args)
if args.dynet_seed:
random.seed(args.dynet_seed)
np.random.seed(args.dynet_seed)
if args.dynet_gpu:
if settings.CHECK_VALIDITY:
settings.CHECK_VALIDITY = False
log_preamble(
"disabling CHECK_VALIDITY because it is not supported on GPU currently",
logging.WARNING)
config_experiment_names = YamlPreloader.experiment_names_from_file(
args.experiments_file)
results = []
# Check ahead of time that all experiments exist, to avoid bad surprises
experiment_names = args.experiment_name or config_experiment_names
if args.experiment_name:
nonexistent = set(experiment_names).difference(
config_experiment_names)
if len(nonexistent) != 0:
raise Exception("Experiments {} do not exist".format(",".join(
list(nonexistent))))
log_preamble(
f"running XNMT revision {tee.get_git_revision()} on {socket.gethostname()} on {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
)
for experiment_name in experiment_names:
ParamManager.init_param_col()
uninitialized_exp_args = YamlPreloader.preload_experiment_from_file(
args.experiments_file, experiment_name)
logger.info(f"=> Running {experiment_name}")
glob_args = uninitialized_exp_args.data.exp_global
log_file = glob_args.log_file
if os.path.isfile(log_file) and not settings.OVERWRITE_LOG:
logger.warning(
f"log file {log_file} already exists, skipping experiment; please delete log file by hand if you want to overwrite it "
f"(or activate OVERWRITE_LOG, by either specifying an environment variable as OVERWRITE_LOG=1, "
f"or specifying --settings=debug, or changing xnmt.settings.Standard.OVERWRITE_LOG manually)"
)
continue
tee.set_out_file(log_file)
model_file = glob_args.model_file
uninitialized_exp_args.data.exp_global.commandline_args = args
# Create the model
experiment = initialize_if_needed(uninitialized_exp_args)
ParamManager.param_col.model_file = experiment.exp_global.model_file
ParamManager.param_col.save_num_checkpoints = experiment.exp_global.save_num_checkpoints
ParamManager.populate()
# Run the experiment
eval_scores = experiment(save_fct=lambda: save_to_file(
model_file, experiment, ParamManager.param_col))
results.append((experiment_name, eval_scores))
print_results(results)
tee.unset_out_file()
import xnmt.tee
import xnmt.serialize.imports
from xnmt.param_collection import ParamManager
from xnmt.serialize.serializer import YamlSerializer
from xnmt.serialize.options import OptionParser, LoadSerialized
EXP_DIR = os.path.dirname(__file__)
EXP = "programmatic-load"
model_file = f"{EXP_DIR}/models/{EXP}.mod"
log_file = f"{EXP_DIR}/logs/{EXP}.log"
xnmt.tee.set_out_file(log_file)
ParamManager.init_param_col()
load_experiment = LoadSerialized(filename=f"{EXP_DIR}/models/programmatic.mod",
overwrite=[{
"path": "train",
"val": None
}])
config_parser = OptionParser()
uninitialized_experiment = config_parser.parse_loaded_experiment(
load_experiment, exp_dir=EXP_DIR, exp_name=EXP)
loaded_experiment = YamlSerializer().initialize_if_needed(
uninitialized_experiment)
# if we were to continue training, we would need to set a save model file like this:
# ParamManager.param_col.model_file = model_file
def setUp(self):
xnmt.events.clear()
ParamManager.init_param_col()