from editor_code.copy_editor.vocab import HardCopyDynamicVocab
from gtd.utils import bleu
print os.environ['COPY_EDIT_DATA']
# no-profile
profile = False
config_file = 'default.txt'
config = Config.from_file('editor_code/configs/editor/' + config_file)
src_dir_noret = os.environ['COPY_EDIT_DATA'] + '/edit_runs/7' #for codalab
load_expt_noret = EditTrainingRun(config, src_dir_noret)
src_dir = os.environ['COPY_EDIT_DATA'] + 'edit_runs/7' #for codalab
load_expt = RetrieveEditTrainingRun(config, src_dir)
###
# retedit model
import numpy as np
ret_model = load_expt.editor.ret_model
# edit_model = load_expt.editor.edit_model # since we only care about the retriever here
examples = load_expt._examples
from gtd.utils import chunks
from tqdm import tqdm
new_vecs = []
for batch in tqdm(chunks(examples.train, 32), total=len(examples.train) / 32):
encin = ret_model.encode(batch, train_mode=False).data.cpu().numpy()
with io.open(validation_dir + '.out', 'r', encoding='utf-8') as fopen:
for line in fopen:
output_list.append(line.strip())
out_proc = [tok_str(proc_str(out)) for out in output_list]
iin = load_input(validation_dir)
valid_ex = make_eexs(iin, out_proc)
#no-profile
profile = False
config = Config.from_file('editor_code/configs/editor/default.txt')
src_dir = os.environ['COPY_EDIT_DATA'] + '/edit_runs/0'
print 'loading model'
print src_dir
load_expt = RetrieveEditTrainingRun(config, src_dir) #highest valid bleu.
import numpy as np
vae_editor = load_expt.editor.vae_model
ret_model = load_expt.editor.ret_model
edit_model = load_expt.editor.edit_model
examples = load_expt._examples
new_vecs = ret_model.batch_embed(examples.train, train_mode=False)
full_lsh = ret_model.make_lsh(new_vecs)
valid_eval = ret_model.ret_and_make_ex(valid_ex,
full_lsh,
examples.train,
0,
train_mode=False)
from gtd.utils import Config
from editor_code.copy_editor.retrieve_edit_run import RetrieveEditTrainingRuns, RetrieveEditTrainingRun
print os.environ['COPY_EDIT_DATA']
import sys, pathlib2
#no-profile
profile = False
src_dir = os.environ['COPY_EDIT_DATA'] + '/edit_runs/7' #for codalab
# load_expt = RetrieveEditTrainingRun(config, src_dir)
# runs = RetrieveEditTrainingRuns()
config_file = 'default.txt'
config = Config.from_file('editor_code/configs/editor/' + config_file)
run = RetrieveEditTrainingRun(config, src_dir)
# run = runs.new(config)
if profile:
from gtd.chrono import Profiling, Profiler
profiler = Profiler.default()
import editor_code.copy_editor.retriever
import editor_code.copy_editor.editor
profiler.add_module(editor_code.copy_editor.editor)
profiler.add_module(editor_code.copy_editor.retriever)
Profiling.start()
run.train()
Profiler.report(profiler) # prints out report