def initialize_train_val_examples(self, max_seq_len, batch_size,
data_path):
"""
Initialize iterators for train and validation data and fetch them for iterating over epochs
:param max_seq_len: Maximum sequence length to filter data in data loader
:param batch_size: Batch size for data loader
:param data_path: Data path where pre processed subject, content and answer data is located
:return:
"""
# Create data loader
data_loader_1 = data_loader.data_loader(max_seq_len, batch_size,
data_path)
# Receive training and validation iterator
training_iterator, validation_iterator = data_loader_1.input_fn()
# Receive train and validation examples to give them into the graph
train_examples = training_iterator.get_next()
validation_examples = validation_iterator.get_next()
return train_examples, validation_examples
def main():
print('begin preprocessing')
config = Config()
train_indices = range(config.train_size)
# val_indices = [x+config.train_size for x in range(config.val_size)]
# test_indices = [x+config.train_size+config.val_size for x in range(config.test_size)]
train_loader = data_loader(config, train_indices)
# val_loader = data_loader(config,val_indices)
# test_loader = data_loader(config,test_indices)
# create the experiments dirs
create_dirs(
[config.summary_dir, config.checkpoint_dir, config.visualization_dir])
model = Model(config)
print('preprocessing done')
#Trainer start
trainer = Trainer(model, train_loader, config, None)
trainer.train()
maindict = { "start_epoch" : 0,
"end_epoch" : 100000,
"save_dir" : 'saved_ff_models',
"ckpt_interval": 10, # for check pointing
"val_interval" : 10,
"verb" : 10 } # peroid for printing out losses
print_dict('trainer',traindict)
print_dict('loss',lossdict)
print_dict('data',data)
print_dict('main',maindict)
data_set = my_data(data["train_file"],data["valid_file"],data["test_file"],data["n_chain"],
data["train_pts"],data["vald_pts"],data["test_pts"])
loader = data_loader(data_set,data["batch_size"])
train = trainer(traindict,lossdict)
train.load_models()
for e in range(maindict["start_epoch"], maindict["end_epoch"]):
for qpl_input,qpl_label in loader.train_loader:
mydevice.load(qpl_input)
q_init,p_init,q_label,p_label,l_init = pack_data(qpl_input,qpl_label)
train.one_step(q_init,p_init,q_label,p_label,l_init)