def retrain(load_from, save_to=None):
print_file = sys.stderr
if gconfig.printout:
print_file = sys.stdout
train_data_src = read_corpus(paths.train_source, source='src')
train_data_tgt = read_corpus(paths.train_target, source='tgt')
dev_data_src = read_corpus(paths.dev_source, source='src')
dev_data_tgt = read_corpus(paths.dev_target, source='tgt')
train_data = zip_data(train_data_src, train_data_tgt)
dev_data = zip_data(dev_data_src, dev_data_tgt)
train_batch_size = tconfig.batch_size
valid_niter = gconfig.valid_niter
log_every = gconfig.log_every
if save_to is None:
model_save_path = paths.model_postfactorized_retrained
else:
model_save_path = save_to
max_epoch = cconfig.max_epoch_retraining
if gconfig.sanity:
log_every = 1
train_data = train_data[:150]
dev_data = dev_data[:150]
max_epoch = 2
print("Loading from ", load_from)
model = PostFactorizedModel.load(load_from)
print("Loaded.")
if gconfig.cuda:
model.to_gpu()
else:
print("No cuda support")
num_trial = 0
train_iter = patience = cum_loss = report_loss = cumulative_tgt_words = report_tgt_words = 0
cumulative_examples = report_examples = epoch = valid_num = 0
hist_valid_scores = []
train_time = begin_time = time.time()
lr = tconfig.lr
max_patience = tconfig.patience
max_num_trial = tconfig.max_num_trial
lr_decay = tconfig.lr_decay
model = routine.train_model(model, train_data, dev_data, model_save_path,
train_batch_size, valid_niter, log_every,
max_epoch, lr, max_patience, max_num_trial,
lr_decay)
model.to_cpu()
def postfactorize(load_from, save_to=None):
print_file = sys.stderr
if gconfig.printout:
print_file = sys.stdout
model_load_path = load_from
if save_to is None:
model_save_path = paths.model_postfactorized
else:
model_save_path = save_to
print("Loading uncompressed model from " + model_load_path)
model = PostFactorizedModel.load(model_load_path)
print("Loaded.")
if cconfig.prune_before:
print("Pruning model")
model.prune()
print("Done pruning.")
model.postfactorize()
model.save(model_save_path, no_opt=True)
print("Saved.")
print("Evaluating on the dev set :")
dev_data_src = read_corpus(paths.dev_source, source='src')
dev_data_tgt = read_corpus(paths.dev_target, source='tgt')
dev_data = zip_data(dev_data_src, dev_data_tgt)
if gconfig.cuda:
model.to_gpu()
else:
print("No cuda support")
dev_ppl = model.evaluate_ppl(dev_data, batch_size=tconfig.batch_size)
print('validation: dev. ppl %f' % dev_ppl, file=print_file)
model.to_cpu()
def check_memory(model):
MAX_BATCH_SIZE = 1024
model.to_gpu()
logfile = paths.memory_log
train_data_src = read_corpus(paths.train_source, source='src')
train_data_tgt = read_corpus(paths.train_target, source='tgt')
train_data = zip_data(train_data_src, train_data_tgt)
now = datetime.datetime.now()
with open(logfile, 'a') as f:
f.write("\n==================\n")
f.write(str(now))
f.write("model class "+model.__class__.__name__)
for b in range(1, MAX_BATCH_SIZE+1):
examples = train_data[:b]
examples = sorted(examples, key=lambda e: len(e[0]), reverse=True)
src_sents = [e[0] for e in examples]
tgt_sents = [e[1] for e in examples]
mem_before_no_retain = int(cuda.memory_allocated()//1e6)
begin = time.time()
try:
loss = model(src_sents, tgt_sents, update_params=False, return_attached_loss=True)
mem_forward = int(cuda.memory_allocated()//1e6)
forward = round(time.time() - begin, 2)
loss.backward(retain_graph=True)
mem_backward_retain = int(cuda.memory_allocated()//1e6)
backward_retain = round(time.time()-(begin+forward), 2)
model.zero_grad()
loss.backward(retain_graph=False)
mem_backward_no_retain = int(cuda.memory_allocated()//1e6)
backward_no_retain = round(time.time()-(begin+forward+backward_retain), 2)
model.zero_grad()
with open(logfile, 'a') as f:
print("\nWith batch size", b, ":")
print("Initial memory", mem_before_no_retain, "forward ("+str(forward)+"s)", mem_forward, "backward retain (" +
str(backward_no_retain)+"s)", mem_backward_no_retain, "backward free ("+str(backward_no_retain)+"s)", mem_backward_retain)
print("\nWith batch size", b, ":", file=f)
print("Initial memory", mem_before_no_retain, "forward ("+str(forward)+"s)", mem_forward, "backward retain (" +
str(backward_no_retain)+"s)", mem_backward_no_retain, "backward free ("+str(backward_no_retain)+"s)", mem_backward_retain, file=f)
if b == MAX_BATCH_SIZE:
with open(logfile, 'a') as f:
print("MAX_BATCH_SIZE", MAX_BATCH_SIZE, "reached !")
print("MAX_BATCH_SIZE", MAX_BATCH_SIZE, "reached !", file=f)
except:
with open(logfile, 'a') as f:
print("\nError caught at batch", b)
print("\nError caught at batch", b, file=f)
break
def train(helper=False):
print_file = sys.stderr
if config.printout:
print_file = sys.stdout
train_data_src = read_corpus(paths.train_source, source='src')
train_data_tgt = read_corpus(paths.train_target, source='tgt')
if config.use_helper:
train_data_src_helper = read_corpus(paths.train_source_helper,
source='src',
lg=config.helper_language(
config.language))
train_data_tgt_helper = read_corpus(paths.train_target_helper,
source='tgt',
lg=config.helper_language(
config.language))
train_data_src = train_data_src + train_data_src_helper
train_data_tgt = train_data_tgt + train_data_tgt_helper
dev_data_src = read_corpus(paths.dev_source, source='src')
dev_data_tgt = read_corpus(paths.dev_target, source='tgt')
train_data = zip_data(train_data_src, train_data_tgt)
dev_data = zip_data(dev_data_src, dev_data_tgt)
train_batch_size = config.batch_size
valid_niter = config.valid_niter
log_every = config.log_every
model_save_path = paths.model(
helper=False) + (".subwords" if config.subwords else "")
max_epoch = config.max_epoch
if config.sanity:
log_every = 1
train_data = train_data[:150]
dev_data = dev_data[:150]
max_epoch = 2
pretraining = config.pretraining
pretraining_encoder = config.pretraining_encoder
loaded_model = False
if config.load:
model = NMTModel.load(model_save_path)
try:
model = NMTModel.load(model_save_path)
pretraining = False
pretraining_encoder = False
loaded_model = True
except:
print("Impossible to load the model ; creating a new one.")
if not loaded_model:
model = NMTModel()
if config.encoder_embeddings:
if config.mode == "normal":
print("loading encoder embeddings")
encoder_embeddings = np.load(paths.get_enc_vec())
model.initialize_enc_embeddings(encoder_embeddings)
if config.mode == "multi":
print("loading encoder embeddings")
lrl_embedding_path, hrl_embedding_path = paths.get_enc_vec()
lrl_embedding, hrl_embedding = np.load(
lrl_embedding_path), np.load(hrl_embedding_path)
model.initialize_enc_embeddings((lrl_embedding, hrl_embedding))
if config.decoder_embeddings:
print("loading decoder embeddings")
decoder_embeddings = np.load(paths.get_dec_vec())
model.initialize_dec_embeddings(decoder_embeddings)
if config.cuda:
model.to_gpu()
else:
print("No cuda support")
num_trial = 0
train_iter = patience = cum_loss = report_loss = cumulative_tgt_words = report_tgt_words = 0
cumulative_examples = report_examples = epoch = valid_num = 0
hist_valid_scores = []
train_time = begin_time = time.time()
lr = config.lr
max_patience = config.patience
max_num_trial = config.max_num_trial
lr_decay = config.lr_decay
if pretraining_encoder:
#print("Pretraining the encoder")
#pretrain.train_encoder(model, train_data, dev_data)
print("Loading monolingual data")
mono_data_src = read_corpus(paths.data_monolingual)
mono_data_tgt = [[] for i in range(len(mono_data_src))]
#train_helper_src = read_corpus(paths.train_source_helper)
#train_helper_tgt = [[] for i in range(len(train_helper_src))]
source_data = zip_data(mono_data_src, mono_data_tgt, "mono",
train_data_src, train_data_tgt, "low")
print("Pretraining the encoder")
routine.train_encoder(model, source_data, dev_data, model_save_path,
config.mono_batch_size, valid_niter, log_every,
config.max_epoch_pretraining_encoder, lr,
max_patience, max_num_trial, lr_decay)
if pretraining:
#print("Pretraining the encoder")
#pretrain.train_encoder(model, train_data, dev_data)
print("loading all target data")
#target_data_tgt = []
# for lg in config.all_languages:
# target_data_tgt = target_data_tgt + \
# read_corpus(paths.get_data_path(set="train", mode="tg", lg=lg))
#train_helper_tgt = read_corpus(paths.train_target_helper)
#train_helper_src = [[] for i in range(len(train_helper_tgt))]
#target_data = zip_data(train_helper_src, train_helper_tgt, "one")
print("Pretraining the decoder")
routine.train_decoder(model, train_data, dev_data, model_save_path,
train_batch_size, valid_niter, log_every,
config.max_epoch_pretraining, lr, max_patience,
max_num_trial, lr_decay)
model = routine.train_model(model, train_data, dev_data, model_save_path,
train_batch_size, valid_niter, log_every,
max_epoch, lr, max_patience, max_num_trial,
lr_decay)
model.to_cpu()
exit(0)
from setup import hardware_specs """**Hardware Properties**""" hardware_specs() """**Initialize Tensorboard Object**""" # Commented out IPython magic to ensure Python compatibility. # Load the TensorBoard notebook extension # %load_ext tensorboard writer = SummaryWriter() """**Unzip the File**""" path = '/content/drive/My Drive/ZipFiles/data_TenK101.zip' save = '/content/drive/My Drive/Videos' zip_data(zipfile, path, save) """**Set the path for all 4 type of Images**""" import os zip_path = "/content/drive/My Drive/ZipExtract/data_TenK101" subd = os.listdir(zip_path) #Returns sub-directories bg_path = '/content/drive/My Drive/ZipExtract/data_TenK101/BG' bgfg_path = '/content/drive/My Drive/ZipExtract/data_TenK101/BG_FG' bgfg_mask_path = '/content/drive/My Drive/ZipExtract/data_TenK101/BG_FG Masks' depthmap_path = '/content/drive/My Drive/ZipExtract/data_TenK101/DepthMaps' """**Some basic checks**"""
def train(load_from=None, save_to=None):
print_file = sys.stderr
if gconfig.printout:
print_file = sys.stdout
train_data_src = read_corpus(paths.train_source, source='src')
train_data_tgt = read_corpus(paths.train_target, source='tgt')
dev_data_src = read_corpus(paths.dev_source, source='src')
dev_data_tgt = read_corpus(paths.dev_target, source='tgt')
train_data = zip_data(train_data_src, train_data_tgt)
dev_data = zip_data(dev_data_src, dev_data_tgt)
train_batch_size = tconfig.batch_size
valid_niter = gconfig.valid_niter
log_every = gconfig.log_every
if save_to is not None:
model_save_path = save_to
else:
model_save_path = paths.model_mixed
max_epoch = tconfig.max_epoch
if gconfig.sanity:
log_every = 1
train_data = train_data[:150]
dev_data = dev_data[:150]
max_epoch = 2
pretraining = gconfig.pretraining
pretraining_encoder = gconfig.pretraining_encoder
if load_from is not None:
print("Loading from", load_from)
model = MixedPrecisionModel.load(load_from)
pretraining = False
pretraining_encoder = False
else:
print("No loading file provided : training from scratch")
model = MixedPrecisionModel()
if gconfig.cuda:
model.to_gpu()
else:
print("No cuda support")
model.quantize()
num_trial = 0
train_iter = patience = cum_loss = report_loss = cumulative_tgt_words = report_tgt_words = 0
cumulative_examples = report_examples = epoch = valid_num = 0
hist_valid_scores = []
train_time = begin_time = time.time()
lr = tconfig.lr
max_patience = tconfig.patience
max_num_trial = tconfig.max_num_trial
lr_decay = tconfig.lr_decay
if pretraining_encoder:
#print("Pretraining the encoder")
#pretrain.train_encoder(model, train_data, dev_data)
print("Pretraining the encoder")
routine.train_encoder(model, train_data, dev_data, model_save_path,
train_batch_size, valid_niter, log_every,
tconfig.max_epoch_pretraining_encoder, lr,
max_patience, max_num_trial, lr_decay)
model.reset_optimizer()
if pretraining:
print("Pretraining the decoder")
routine.train_decoder(model, train_data, dev_data, model_save_path,
train_batch_size, valid_niter, log_every,
tconfig.max_epoch_pretraining, lr, max_patience,
max_num_trial, lr_decay)
model.reset_optimizer()
model = routine.train_model(model, train_data, dev_data, model_save_path,
train_batch_size, valid_niter, log_every,
max_epoch, lr, max_patience, max_num_trial,
lr_decay)
model.to_cpu()
def train():
print_file = sys.stderr
if config.printout:
print_file = sys.stdout
train_data_src_low = read_corpus(paths.train_source, source='src')
train_data_tgt_low = read_corpus(paths.train_target, source='tgt')
dev_data_src_low = read_corpus(paths.dev_source, source='src')
dev_data_tgt_low = read_corpus(paths.dev_target, source='tgt')
train_data_src_helper = read_corpus(paths.train_source_helper,
source='src')
train_data_tgt_helper = read_corpus(paths.train_target_helper,
source='tgt')
dev_data_src_helper = read_corpus(paths.dev_source_helper, source='src')
dev_data_tgt_helper = read_corpus(paths.dev_target_helper, source='tgt')
train_data = zip_data(train_data_src_low, train_data_tgt_low, "low",
train_data_src_helper, train_data_tgt_helper,
"helper")
train_data_low = zip_data(train_data_src_low, train_data_tgt_low, "low")
train_data_helper = zip_data(train_data_src_helper, train_data_tgt_helper,
"helper")
dev_data_low = zip_data(dev_data_src_low, dev_data_tgt_low, "low")
dev_data_helper = zip_data(dev_data_src_helper, dev_data_tgt_helper,
"helper")
train_batch_size = config.batch_size
valid_niter = config.valid_niter
log_every = config.log_every
model_save_path = paths.model(helper=False) + ".multi"
max_epoch = config.max_epoch
sampling = config.sampling
if config.sanity:
log_every = 1
valid_niter = 5
train_data = dict([(k, v[:150]) for (k, v) in train_data.items()])
dev_data_low = dict([(k, v[:150]) for (k, v) in dev_data_low.items()])
dev_data_helper = dict([(k, v[:150])
for (k, v) in dev_data_helper.items()])
train_data_low = dict([(k, v[:150])
for (k, v) in train_data_low.items()])
train_data_helper = dict([(k, v[:150])
for (k, v) in train_data_helper.items()])
max_epoch = 2
pretraining_decoder = config.pretraining_decoder
pretraining_encoders = config.pretraining_encoders
if config.load:
#model = MultiWayModel.load(model_save_path)
try:
model = MultiWayModel.load(model_save_path)
pretraining_decoder = False
pretraining_encoders = False
except:
print("Impossible to load the model ; creating a new one.")
model = MultiWayModel()
else:
model = MultiWayModel()
if config.cuda:
model.to_gpu()
else:
print("No cuda support")
num_trial = 0
train_iter = patience = cum_loss = report_loss = cumulative_tgt_words = report_tgt_words = 0
cumulative_examples = report_examples = epoch = valid_num = 0
hist_valid_scores = []
train_time = begin_time = time.time()
lr = config.lr
max_patience = config.patience
max_num_trial = config.max_num_trial
lr_decay = config.lr_decay
if pretraining_decoder:
# print("Pretraining the encoder")
# pretrain.train_encoder(model, train_data, dev_data)
print("Pretraining the decoder")
model.activate_discriminator = False
routine.train_decoder(model, train_data_helper, dev_data_helper,
model_save_path, train_batch_size, valid_niter,
log_every, config.max_epoch_pretraining_decoder,
lr, max_patience, max_num_trial, lr_decay)
routine.train_decoder(model, train_data_low, dev_data_low,
model_save_path, train_batch_size, valid_niter,
log_every, config.max_epoch_pretraining_decoder,
lr, max_patience, max_num_trial, lr_decay)
if pretraining_encoders:
# print("Pretraining the encoder")
# pretrain.train_encoder(model, train_data, dev_data)
model.activate_discriminator = False
print("Pretraining the helper encoder")
routine.train_model(model, train_data_helper, dev_data_helper,
model_save_path, train_batch_size, valid_niter,
log_every, config.max_epoch_pretraining_helper, lr,
max_patience, max_num_trial, lr_decay)
print("Pretraining the low-resource encoder")
routine.train_model(model, train_data_low, dev_data_low,
model_save_path, train_batch_size, valid_niter,
log_every, config.max_epoch_pretraining_low, lr,
max_patience, max_num_trial, lr_decay)
print("Multitask training")
model.activate_discriminator = True
model = routine.train_model(model,
train_data,
dev_data_low,
model_save_path,
train_batch_size,
valid_niter,
log_every,
max_epoch,
lr,
max_patience,
max_num_trial,
lr_decay,
sampling_multi=sampling)
model.to_cpu()
exit(0)