def train(data_dir, arch, hidden_units, output_size, dropout, lr, epochs, gpu,
checkpoint):
print(
'Dir: {},\t Arch:{},\t HiddenUints: {},\t lr: {},\t Epochs: {},\t gpu: {}\n'
.format(data_dir, arch, hidden_units, lr, epochs, gpu))
print('Loading Images from Directory...')
trainloader, validloader, testloader, class_to_idx = get_loaders(data_dir)
print('Images Loaded.\n')
print('Building the Model...')
model, criterion, optimizer = build_model(arch, hidden_units, output_size,
dropout, lr)
print('Model Built.\n')
print('Beggining the Training...')
model, optimizer = training(model, trainloader, validloader, epochs, 20,
criterion, optimizer, gpu)
print('Training Done.\n')
if checkpoint:
print('Saving the Checkpoint...')
save_checkpoint(checkpoint, model, optimizer, arch,
model.classifier[0].in_features, output_size,
hidden_units, dropout, class_to_idx, epochs, lr)
print('Done.')
def train(self):
"""
Train MemN2N model using training data for tasks.
"""
np.random.seed(42) # for reproducing
assert self.data_dir is not None, "data_dir is not specified."
print("Reading data from %s ..." % self.data_dir)
# Parse training data
train_data_path = glob.glob('%s/qa*_*_train.txt' % self.data_dir)
dictionary = {"nil": 0}
train_story, train_questions, train_qstory = parse_babi_task(train_data_path, dictionary, False)
# Parse test data just to expand the dictionary so that it covers all words in the test data too
test_data_path = glob.glob('%s/qa*_*_test.txt' % self.data_dir)
parse_babi_task(test_data_path, dictionary, False)
# Get reversed dictionary mapping index to word
self.reversed_dict = dict((ix, w) for w, ix in dictionary.items())
# Construct model
self.general_config = BabiConfigJoint(train_story, train_questions, dictionary)
self.memory, self.model, self.loss = build_model(self.general_config)
# Train model
if self.general_config.linear_start:
train_linear_start(train_story, train_questions, train_qstory,
self.memory, self.model, self.loss, self.general_config)
else:
train(train_story, train_questions, train_qstory,
self.memory, self.model, self.loss, self.general_config)
# Save model
self.save_model()
def run_task(data_dir, task_id):
"""
Train and test for each task
"""
print("Train and test for task %d ..." % task_id)
# Parse data
train_files = glob.glob('%s/qa%d_*_train.txt' % (data_dir, task_id))
test_files = glob.glob('%s/qa%d_*_test.txt' % (data_dir, task_id))
dictionary = {"nil": 0}
train_story, train_questions, train_qstory = parse_babi_task(
train_files, dictionary, False)
test_story, test_questions, test_qstory = parse_babi_task(
test_files, dictionary, False)
general_config = BabiConfig(train_story, train_questions, dictionary)
memory, model, loss = build_model(general_config)
if general_config.linear_start:
train_linear_start(train_story, train_questions, train_qstory, memory,
model, loss, general_config)
else:
train(train_story, train_questions, train_qstory, memory, model, loss,
general_config)
test(test_story, test_questions, test_qstory, memory, model, loss,
general_config)
def train(self):
"""
Train MemN2N model using training data for tasks.
"""
np.random.seed(42) # for reproducing
assert self.data_dir is not None, "data_dir is not specified."
print("Reading data from %s ..." % self.data_dir)
# Parse training data
train_data_path = glob.glob('%s/qa*_*_train.txt' % self.data_dir)
dictionary = {"nil": 0}
train_story, train_questions, train_qstory = parse_babi_task(train_data_path, dictionary, False)
# Parse test data just to expand the dictionary so that it covers all words in the test data too
test_data_path = glob.glob('%s/qa*_*_test.txt' % self.data_dir)
parse_babi_task(test_data_path, dictionary, False)
# Get reversed dictionary mapping index to word
self.reversed_dict = dict((ix, w) for w, ix in dictionary.items())
# Construct model
self.general_config = BabiConfigJoint(train_story, train_questions, dictionary)
self.memory, self.model, self.loss = build_model(self.general_config)
# Train model
if self.general_config.linear_start:
train_linear_start(train_story, train_questions, train_qstory,
self.memory, self.model, self.loss, self.general_config)
else:
train(train_story, train_questions, train_qstory,
self.memory, self.model, self.loss, self.general_config)
# Save model
self.save_model()
def main(training_from_scratch, args):
if (training_from_scratch):
text = open(args.filename, 'rb').read().decode(encoding='utf-8')
text, char2idx, idx2char = preprocessing(
text, args.checkpoint_dir,
args.minocc) # note that we are replacing the text here
vocab_size = len(idx2char)
config = Config(vocab_size, args.epochs)
model = build_model(config)
else:
model = tf.keras.models.load_model(args.checkpoint)
char2idx = unpickle(args.checkpoint_dir, 'char2idx')
idx2char = unpickle(args.checkpoint_dir, 'idx2char')
text = unpickle(args.checkpoint_dir, 'dataset')
vocab_size = len(idx2char)
config = Config(vocab_size, args.epochs, args.initepochs)
text_as_int = np.array([char2idx[c] for c in text
]) # works because text is a list of words
train_model(args.checkpoint_dir, text_as_int, model, config)
def train(max_eps=6000):
env = gym.make('CartPole-v0')
eval_env = gym.make('CartPole-v0')
state_space_size = env.observation_space.shape[0]
action_space_size = env.action_space.n
print('action space size is {0}, state space size is {1}'.format(
action_space_size, state_space_size))
model = build_model(state_space_size, action_space_size)
model.summary()
gamma = 0.9
optimizer = tf.optimizers.Adam(learning_rate=0.001)
best_score = 0.0
for eps in range(max_eps):
done = False
state = env.reset()
rewards = []
action_probs = []
actions = []
with tf.GradientTape() as tape:
while not done:
state_tensor = tf.convert_to_tensor([state], dtype=tf.float32)
action_prob = model(state_tensor)
action = sample_action(action_prob, action_space_size)
next_state, reward, done, _ = env.step(action)
rewards.append(reward)
action_probs.append(action_prob[0])
actions.append(action)
state = next_state
discounted_rewards = compute_discounted_rewards(rewards, gamma)
normalized_discounted_rewards = tf.convert_to_tensor(
normalize_discounted_rewards(discounted_rewards),
dtype=tf.float32)
probs = tf.stack(action_probs)
clipped_probs = tf.clip_by_value(probs, 1e-8, 1.0 - 1e-8)
onehot_actions = tf.one_hot(actions,
action_space_size,
dtype=tf.float32)
log_likelihood = tf.multiply(onehot_actions,
tf.math.log(clipped_probs))
loss = tf.math.reduce_sum(
tf.multiply(
-log_likelihood,
tf.expand_dims(normalized_discounted_rewards, axis=1)))
policy_gradients = tape.gradient(loss, model.trainable_weights)
optimizer.apply_gradients(
zip(policy_gradients, model.trainable_weights))
score = eval(model, eval_env, 10, action_space_size)
if score >= best_score:
best_score = score
tf.saved_model.save(model, './best_model')
print('Finished episode {0}/{1} with score {2}'.format(
eps, max_eps, score))
env.close()
eval_env.close()
def run_joint_tasks(data_dir):
"""
Train and test for all tasks but the trained model is built using training data from all tasks.
"""
print("Jointly train and test for all tasks ...")
tasks = range(20)
# Parse training data
train_data_path = []
for t in tasks:
train_data_path += glob.glob('%s/qa%d_*_train.txt' % (data_dir, t + 1))
dictionary = {"nil": 0}
train_story, train_questions, train_qstory = parse_babi_task(
train_data_path, dictionary, False)
# Parse test data for each task so that the dictionary covers all words before training
for t in tasks:
test_data_path = glob.glob('%s/qa%d_*_test.txt' % (data_dir, t + 1))
parse_babi_task(test_data_path, dictionary,
False) # ignore output for now
general_config = BabiConfigJoint(train_story, train_questions, dictionary)
memory, model, loss = build_model(general_config)
if general_config.linear_start:
train_linear_start(train_story, train_questions, train_qstory, memory,
model, loss, general_config)
else:
train(train_story, train_questions, train_qstory, memory, model, loss,
general_config)
# Test on each task
for t in tasks:
print("Testing for task %d ..." % (t + 1))
test_data_path = glob.glob('%s/qa%d_*_test.txt' % (data_dir, t + 1))
dc = len(dictionary)
test_story, test_questions, test_qstory = parse_babi_task(
test_data_path, dictionary, False)
assert dc == len(
dictionary
) # make sure that the dictionary already covers all words
test(test_story, test_questions, test_qstory, memory, model, loss,
general_config)
def main(training_from_scratch, args):
### opening the file ###
text = open(args.filename, 'rb').read().decode(encoding='utf-8')
vocab_size = len(set(text))
config = Config(vocab_size, args.epochs, args.initepochs)
### looking at shit ###
print('Length of text: {} characters'.format(len(text)))
print(text[:250])
### creating vocab, converting text to long integer sequence ###
char2idx, idx2char = create_vocab_from_file(text, args.checkpoint_dir)
text_as_int = np.array([char2idx[c] for c in text])
if (training_from_scratch):
model = build_model(config)
else:
model = tf.keras.models.load_model(args.checkpoint)
train_model(args.checkpoint_dir, text_as_int, model, config)
def run_joint_tasks(data_dir):
"""
Train and test for all tasks but the trained model is built using training data from all tasks.
"""
print("Jointly train and test for all tasks ...")
tasks = range(20)
# Parse training data
train_data_path = []
for t in tasks:
train_data_path += glob.glob('%s/qa%d_*_train.txt' % (data_dir, t + 1))
dictionary = {"nil": 0}
train_story, train_questions, train_qstory = parse_babi_task(train_data_path, dictionary, False)
# Parse test data for each task so that the dictionary covers all words before training
for t in tasks:
test_data_path = glob.glob('%s/qa%d_*_test.txt' % (data_dir, t + 1))
parse_babi_task(test_data_path, dictionary, False) # ignore output for now
general_config = BabiConfigJoint(train_story, train_questions, dictionary)
memory, model, loss = build_model(general_config)
if general_config.linear_start:
train_linear_start(train_story, train_questions, train_qstory, memory, model, loss, general_config)
else:
train(train_story, train_questions, train_qstory, memory, model, loss, general_config)
# Test on each task
for t in tasks:
print("Testing for task %d ..." % (t + 1))
test_data_path = glob.glob('%s/qa%d_*_test.txt' % (data_dir, t + 1))
dc = len(dictionary)
test_story, test_questions, test_qstory = parse_babi_task(test_data_path, dictionary, False)
assert dc == len(dictionary) # make sure that the dictionary already covers all words
test(test_story, test_questions, test_qstory, memory, model, loss, general_config)
def run_tableQA(data_path, model_file):
"""
Train and test for table QA
"""
# Parse data
train_files = glob.glob(data_path.format('train'))
test_files = glob.glob(data_path.format('test'))
# SV: init dict with pre-trained vectors, e.g. from fastText
# dictionary = fasttext.load_model(EMBEDDINGS_MODEL_PATH)
dictionary = {"nil": 0}
train_story, train_questions, train_qstory = parse_babi_task(
train_files, dictionary, False)
test_story, test_questions, test_qstory = parse_babi_task(
test_files, dictionary, False)
# print test_questions
print 'Dictionary:', len(dictionary)
general_config = BabiConfig(train_story, train_questions, dictionary)
memory, model, loss = build_model(general_config)
if general_config.linear_start:
train_linear_start(train_story, train_questions, train_qstory, memory,
model, loss, general_config)
else:
train(train_story, train_questions, train_qstory, memory, model, loss,
general_config)
test(test_story, test_questions, test_qstory, memory, model, loss,
general_config)
# save_model
with gzip.open(model_file, "wb") as f:
print("Saving model to file %s ..." % model_file)
reversed_dict = dict((ix, w) for w, ix in dictionary.items())
pickle.dump((reversed_dict, memory, model, loss, general_config), f)
def run_task(data_dir, task_id):
"""
Train and test for each task
"""
print("Train and test for task %d ..." % task_id)
# Parse data
train_files = glob.glob('%s/qa%d_*_train.txt' % (data_dir, task_id))
test_files = glob.glob('%s/qa%d_*_test.txt' % (data_dir, task_id))
dictionary = {"nil": 0}
train_story, train_questions, train_qstory = parse_babi_task(train_files, dictionary, False)
test_story, test_questions, test_qstory = parse_babi_task(test_files, dictionary, False)
general_config = BabiConfig(train_story, train_questions, dictionary)
memory, model, loss = build_model(general_config)
if general_config.linear_start:
train_linear_start(train_story, train_questions, train_qstory, memory, model, loss, general_config)
else:
train(train_story, train_questions, train_qstory, memory, model, loss, general_config)
test(test_story, test_questions, test_qstory, memory, model, loss, general_config)
n_mb = None
rpn_rois_per_img = 256 # number of rois to sample to train rpn
frcn_rois_per_img = 128 # number of rois to sample to train frcn
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
be.enable_winograd = 4 # default to winograd 4 for fast autotune
year = '2007'
train_set = PASCAL('trainval', year, path=args.data_dir, n_mb=n_mb,
rpn_rois_per_img=rpn_rois_per_img, frcn_rois_per_img=frcn_rois_per_img,
add_flipped=True, shuffle=True, rebuild_cache=True, subset_pct=args.subset_pct)
# build the Faster-RCNN model
model = util.build_model(train_set, frcn_rois_per_img, inference=False)
# set up cost different branches, respectively
weights = 1.0 / (rpn_rois_per_img)
roi_w = 1.0 / (frcn_rois_per_img)
frcn_tree_cost = Multicost(costs=[GeneralizedCostMask(costfunc=CrossEntropyMulti(), weights=roi_w),
GeneralizedCostMask(costfunc=SmoothL1Loss(), weights=roi_w)
], weights=[1, 1])
cost = Multicost(costs=[GeneralizedCostMask(costfunc=CrossEntropyMulti(), weights=weights),
GeneralizedCostMask(costfunc=SmoothL1Loss(sigma=3.0), weights=weights),
frcn_tree_cost,
],
weights=[1, 1, 1])
def modeling(self):
self.embedding()
model = build_model(len(self.word2int), 300, self.embeddingMatrix,
self.wordLimit)
model.load_weights("./Microsoft_model.h5")
return model
def _load_model(self, file_path):
self.network = build_model(self.network_config, num_classes=10)
with open(file_path, "rb") as f_:
self.network.load_state_dict(torch.load(f_))
def train_and_test(self, seed=None):
"""
Train MemN2N model using training data for tasks.
"""
if seed is None:
np.random.seed(42) # for reproducing
else:
np.random.seed(seed)
train_data_arg = None
test_data_arg = None
if type(self.data_dir) is tuple:
assert self.data_dir[
0] is not None, "training data_dir is not specified."
assert self.data_dir[
1] is not None, "test data_dir is not specified."
print("Reading training data from %s ..." % self.data_dir[0])
print("Reading test data from %s ..." % self.data_dir[1])
train_data_arg = '%s/qa*_train.txt' % self.data_dir[0]
test_data_arg = '%s/qa*_valid.txt' % self.data_dir[1]
else:
assert self.data_dir is not None, "data_dir is not specified."
print("Reading data from %s ..." % self.data_dir)
train_data_arg = '%s/qa*_*_train.txt' % self.data_dir
test_data_arg = '%s/qa*_*_test.txt' % self.data_dir
assert train_data_arg is not None and test_data_arg is not None
# Parse training data
train_data_path = glob.glob(train_data_arg)
dictionary = {"nil": 0}
train_story, train_questions, train_qstory = parse_babi_task(
train_data_path, dictionary, False)
# Parse test data just to expand the dictionary so that it covers all words in the test data too
test_data_path = glob.glob(test_data_arg)
test_story, test_questions, test_qstory = parse_babi_task(
test_data_path, dictionary, False)
# Get reversed dictionary mapping index to word
self.reversed_dict = dict((ix, w) for w, ix in dictionary.items())
# Construct model
self.general_config = BabiConfigJoint(train_story, train_questions,
dictionary)
#check for config switches format [initial learning rate, linear start option, hops]
if self.config_switches is not None:
self.general_config.train_config[
"init_lrate"] = self.config_switches[0]
#linear start option is passed to babi config constructor function so no need to set here
# want equal of number of epochs for linear start and non linear start runs
if self.general_config.linear_start is True:
self.general_config.nepochs = 40
self.general_config.ls_nepochs = 20
else:
self.general_config.nepochs = 60
self.general_config.nhops = self.config_switches[2]
self.memory, self.model, self.loss = build_model(self.general_config)
# Train model
train_val_results = []
if self.general_config.linear_start:
train_val_results += train_linear_start(train_story,
train_questions,
train_qstory, self.memory,
self.model, self.loss,
self.general_config)
else:
train_val_results += train(train_story, train_questions,
train_qstory, self.memory, self.model,
self.loss, self.general_config)
test_error = test(test_story, test_questions, test_qstory, self.memory,
self.model, self.loss, self.general_config)
model_test_accuracy = (1.0 - test_error) * 100.0
train_val_file = self.model_file + 'train_val_accuracy.csv'
with open(train_val_file, 'w') as f:
f.write('epoch, TrainAccuracy, ValAccuracy\n')
epoch = 1
for item in train_val_results:
line = '{}, {:.3f}, {:.3f}\n'.format(epoch, item[0], item[1])
f.write(line)
epoch += 1
self.model_file += '_TestAcc{:.1f}percent_.pickle'.format(
model_test_accuracy)
# Save model
self.save_model()
year = '2007'
train_set = PASCAL('trainval',
year,
path=args.data_dir,
n_mb=n_mb,
rpn_rois_per_img=rpn_rois_per_img,
frcn_rois_per_img=frcn_rois_per_img,
add_flipped=True,
shuffle=True,
rebuild_cache=True,
subset_pct=args.subset_pct)
# build the Faster-RCNN model
model = util.build_model(train_set, frcn_rois_per_img, inference=False)
# set up cost different branches, respectively
weights = 1.0 / (rpn_rois_per_img)
roi_w = 1.0 / (frcn_rois_per_img)
frcn_tree_cost = Multicost(costs=[
GeneralizedCostMask(costfunc=CrossEntropyMulti(), weights=roi_w),
GeneralizedCostMask(costfunc=SmoothL1Loss(), weights=roi_w)
],
weights=[1, 1])
cost = Multicost(costs=[
GeneralizedCostMask(costfunc=CrossEntropyMulti(), weights=weights),
GeneralizedCostMask(costfunc=SmoothL1Loss(sigma=3.0), weights=weights),
frcn_tree_cost,
def run_task(data_dir, task_id):
"""
Train and test for each task
"""
print("Train and test for task %d ..." % task_id)
# Parse data
train_files = glob.glob('%s/qa%d_*_train.txt' % (data_dir, task_id))
test_files = glob.glob('%s/qa%d_*_test.txt' % (data_dir, task_id))
#train_files = glob.glob('%s/qa%d_*train.txt' % (data_dir, task_id))
#test_files = glob.glob('%s/qa%d_*test.txt' % (data_dir, task_id))
# #### empty dictionary
dictionary = {"nil": 0}
train_story, train_questions, train_qstory = parse_babi_task(train_files, dictionary, False)
test_story, test_questions, test_qstory = parse_babi_task(test_files, dictionary, False)
general_config = BabiConfig(train_story, train_questions, dictionary)
memory, model, loss = build_model(general_config)
if general_config.linear_start:
train_linear_start(train_story, train_questions, train_qstory, memory, model, loss, general_config)
else:
train(train_story, train_questions, train_qstory, memory, model, loss, general_config)
with open('R_trained.txt', 'a') as outfile:
json.dump(general_config.dictionary, outfile, indent=2)
print("######## trained dictionary")
print(general_config.dictionary)
ans_index = test(test_story, test_questions, test_qstory, memory, model, loss, general_config)
####R this line load model
memn2n = MemN2N(args.data_dir, args.model_file)
#Try to load model
memn2n.load_model()
dictionary2 = {"nil": 0}
train_story2, train_questions2, train_qstory2 = parse_babi_task(train_files, memn2n.general_config.dictionary, False)
test_story2, test_questions2, test_qstory2 = parse_babi_task(test_files, memn2n.general_config.dictionary, False)
#print(len(test_questions2))
#general_config2 = BabiConfig(train_story2, train_questions2,memn2n.general_config.dictionary)
with open('R_loaded.txt', 'a') as outfile2:
json.dump(memn2n.general_config.dictionary, outfile2, indent=2)
print("???????? loaded dictionary")
print(memn2n.general_config.dictionary)
ans_index = test(test_story2, test_questions2, test_qstory2, memn2n.memory, memn2n.model, memn2n.loss, memn2n.general_config)
def train(self):
"""
Train MemN2N model using training data for tasks.
"""
#np.random.seed(42) # for reproducing
np.random.seed(self.rnd_seed) # for reproducing
print("np.random.seed: %d" % self.rnd_seed)
assert self.data_dir is not None, "data_dir is not specified."
print("Reading data from %s ..." % self.data_dir)
# Parse training data
train_data_path = glob.glob('%s/qa*_*_train.txt' % self.data_dir)
dictionary = {"nil": 0}
train_story, train_questions, train_qstory = parse_babi_task(train_data_path, dictionary, False)
# Parse test data just to expand the dictionary so that it covers all words in the test data too
test_data_path = glob.glob('%s/qa*_*_test.txt' % self.data_dir)
parse_babi_task(test_data_path, dictionary, False)
# Get reversed dictionary mapping index to word
self.reversed_dict = dict((ix, w) for w, ix in dictionary.items())
# Construct model
#self.general_config = Babi10kConfigJoint(train_story, train_questions, dictionary)
self.general_config = BabiConfigJoint(train_story, train_questions, dictionary)
self.memory, self.model, self.loss = build_model(self.general_config)
# Train model
if self.general_config.linear_start:
print('We will use LS training')
self.best_model, self.best_memory = \
train_linear_start(train_story,
train_questions,
train_qstory,
self.memory,
self.model,
self.loss,
self.general_config,
self.log_path)
else:
train_logger = open(os.path.join(self.log_path, 'train.log'), 'w')
train_logger.write('epoch batch_iter lr loss err\n')
train_logger.flush()
val_logger = open(os.path.join(self.log_path, 'val.log'), 'w')
val_logger.write('epoch batch_iter lr loss err\n')
val_logger.flush()
global_batch_iter = 0
train_logger, val_logger, self.best_model, self.best_memory, _ = \
train(train_story,
train_questions,
train_qstory,
self.memory,
self.model,
self.loss,
self.general_config,
train_logger,
val_logger,
global_batch_iter)
train_logger.close()
val_logger.close()
# Save model
self.save_model()
def train(self):
"""
Train MemN2N model using training data for tasks.
"""
#np.random.seed(42) # for reproducing
np.random.seed(self.rnd_seed) # for reproducing
print("np.random.seed: %d" % self.rnd_seed)
assert self.data_dir is not None, "data_dir is not specified."
print("Reading data from %s ..." % self.data_dir)
# Parse training data
train_data_path = glob.glob('%s/qa*_*_train.txt' % self.data_dir)
dictionary = {"nil": 0}
train_story, train_questions, train_qstory = parse_babi_task(
train_data_path, dictionary, False)
# Parse test data just to expand the dictionary so that it covers all words in the test data too
test_data_path = glob.glob('%s/qa*_*_test.txt' % self.data_dir)
parse_babi_task(test_data_path, dictionary, False)
# Get reversed dictionary mapping index to word
self.reversed_dict = dict((ix, w) for w, ix in dictionary.items())
# Construct model
#self.general_config = Babi10kConfigJoint(train_story, train_questions, dictionary)
self.general_config = BabiConfigJoint(train_story, train_questions,
dictionary)
self.memory, self.model, self.loss = build_model(self.general_config)
# Train model
if self.general_config.linear_start:
print('We will use LS training')
self.best_model, self.best_memory = \
train_linear_start(train_story,
train_questions,
train_qstory,
self.memory,
self.model,
self.loss,
self.general_config,
self.log_path)
else:
train_logger = open(os.path.join(self.log_path, 'train.log'), 'w')
train_logger.write('epoch batch_iter lr loss err\n')
train_logger.flush()
val_logger = open(os.path.join(self.log_path, 'val.log'), 'w')
val_logger.write('epoch batch_iter lr loss err\n')
val_logger.flush()
global_batch_iter = 0
train_logger, val_logger, self.best_model, self.best_memory, _ = \
train(train_story,
train_questions,
train_qstory,
self.memory,
self.model,
self.loss,
self.general_config,
train_logger,
val_logger,
global_batch_iter)
train_logger.close()
val_logger.close()
# Save model
self.save_model()
def run_task(data_dir, task_id, model_file, log_path):
"""
Train and test for each task
"""
print("Train and test for task %d ..." % task_id)
train_files = glob.glob('%s/qa%d_*_train.txt' % (data_dir, task_id))
test_files = glob.glob('%s/qa%d_*_test.txt' % (data_dir, task_id))
dictionary = {"nil": 0}
train_story, train_questions, train_qstory = \
parse_babi_task(train_files, dictionary, False)
test_story, test_questions, test_qstory = \
parse_babi_task(test_files, dictionary, False)
# Get reversed dictionary mapping index to word
# NOTE: this needed to real-time testing
reversed_dict = dict((ix, w) for w, ix in dictionary.items())
general_config = BabiConfig(train_story, train_questions, dictionary)
memory, model, loss_func = build_model(general_config)
if general_config.linear_start:
print('We will use LS training')
best_model, best_memory = \
train_linear_start(train_story,
train_questions,
train_qstory,
memory,
model,
loss_func,
general_config,
self.log_path)
else:
train_logger = open(os.path.join(self.log_path, 'train.log'), 'w')
train_logger.write('epoch batch_iter lr loss err\n')
train_logger.flush()
val_logger = open(os.path.join(self.log_path, 'val.log'), 'w')
val_logger.write('epoch batch_iter lr loss err\n')
val_logger.flush()
global_batch_iter = 0
train_logger, val_logger, _, _, _ = \
train(train_story,
train_questions,
train_qstory,
memory,
model,
loss_func,
general_config,
train_logger,
val_logger,
global_batch_iter)
train_logger.close()
val_logger.close()
model_file = os.path.join(log_path, model_file)
with gzip.open(model_file, 'wb') as f:
print('Saving model to file %s ...' % model_file)
pickle.dump((reversed_dict, memory, model, loss_func, general_config),
f)
print('Start to testing')
test(test_story, test_questions, test_qstory, memory, model, loss_func,
general_config)
year = '2007'
valid_set = PASCAL(image_set,
year,
path=args.data_dir,
n_mb=n_mb,
rpn_rois_per_img=rpn_rois_per_img,
frcn_rois_per_img=frcn_rois_per_img,
add_flipped=False,
shuffle=False,
rebuild_cache=True)
num_classes = valid_set.num_classes
# build the Faster-RCNN network
(model, proposalLayer) = util.build_model(valid_set,
frcn_rois_per_img,
inference=True)
# load parameters and initialize model
model.load_params(args.model_file)
model.initialize(dataset=valid_set)
# normalize the model by the bbtarget mean and std if needed
# if a full training run was completed using train.py, then normalization
# was already performed prior to saving the model.
if args.normalize:
model = util.scale_bbreg_weights(model, [0.0, 0.0, 0.0, 0.0],
[0.1, 0.1, 0.2, 0.2], num_classes)
# run inference
# model.benchmark(valid_set, inference=True)
rpn_rois_per_img = 256
frcn_rois_per_img = 128
# setup backend
be = gen_backend(**extract_valid_args(args, gen_backend))
image_set = 'test'
year = '2007'
valid_set = PASCAL(image_set, year, path=args.data_dir, n_mb=n_mb,
rpn_rois_per_img=rpn_rois_per_img, frcn_rois_per_img=frcn_rois_per_img,
add_flipped=False, shuffle=False, rebuild_cache=True)
num_classes = valid_set.num_classes
# build the Faster-RCNN network
(model, proposalLayer) = util.build_model(valid_set, frcn_rois_per_img, inference=True)
# load parameters and initialize model
model.load_params(args.model_file)
model.initialize(dataset=valid_set)
# normalize the model by the bbtarget mean and std if needed
# if a full training run was completed using train.py, then normalization
# was already performed prior to saving the model.
if args.normalize:
model = util.scale_bbreg_weights(model, [0.0, 0.0, 0.0, 0.0],
[0.1, 0.1, 0.2, 0.2], num_classes)
# run inference
# model.benchmark(valid_set, inference=True)
x_train, x_test, y_train, y_test = train_test_split(newHeadlinesData, normalizedOpeningprice, test_size=0.2,
random_state=3)
# converting into array
x_train = np.array(x_train)
x_test = np.array(x_test)
y_train = np.array(y_train)
y_test = np.array(y_test)
print()
print("Train data shape : ", x_train.shape)
print("Test data shape : ", x_test.shape)
trainedModel = build_model(wordsCount, embeddingDimension, embeddingMatrix, wordLimit)
trainedModel.summary()
save_best_weights = 'model.h5'
callbacks = [ModelCheckpoint(save_best_weights, monitor='val_loss', save_best_only=True),
EarlyStopping(monitor='val_loss', patience=5, verbose=1, mode='auto')]
history = trainedModel.fit([x_train, x_train], y_train, batch_size=128, epochs=100, validation_split=0.15, verbose=True,
shuffle=True, callbacks=callbacks)
# evaluation
trainedModel.load_weights("./model.h5")
predictions = list(chain.from_iterable(trainedModel.predict([x_test, x_test], verbose=True)))
error = mse(y_test, predictions)
print()
print("Mean Square Error : ", error)
def run_joint_tasks(data_dir, model_file, log_path):
"""
Train and test for all tasks but the trained model is built using training data from all tasks.
"""
print("Jointly train and test for all tasks ...")
tasks = range(20)
# Parse training data
train_data_path = []
for t in tasks:
train_data_path += glob.glob('%s/qa%d_*_train.txt' % (data_dir, t + 1))
dictionary = {"nil": 0}
train_story, train_questions, train_qstory = \
parse_babi_task(train_data_path,
dictionary,
False)
# Parse test data for each task so that the dictionary covers all words before training
for t in tasks:
test_data_path = glob.glob('%s/qa%d_*_test.txt' % (data_dir, t + 1))
parse_babi_task(test_data_path, dictionary,
False) # ignore output for now
# Get reversed dictionary mapping index to word
# NOTE: this needed to real-time testing
reversed_dict = dict((ix, w) for w, ix in dictionary.items())
general_config = BabiConfigJoint(train_story, train_questions, dictionary)
memory, model, loss_func = build_model(general_config)
if general_config.linear_start:
print('We will use LS training')
train_linear_start(train_story, train_questions, train_qstory, memory,
model, loss_func, general_config, log_path)
else:
train_logger = open(os.path.join(log_file, 'train.log'), 'w')
train_logger.write('epoch batch_iter lr loss err\n')
train_logger.flush()
val_logger = open(os.path.join(log_file, 'val.log'), 'w')
val_logger.write('epoch batch_iter lr loss err\n')
val_logger.flush()
train_logger, val_logger, best_model, best_memory = \
train(train_story,
train_questions,
train_qstory,
memory,
model,
loss_func,
general_config,
train_logger,
val_logger)
train_logger.close()
val_logger.close()
model_file = os.path.join(log_path, model_file)
with gzip.open(model_file, 'wb') as f:
print('Saving model to file %s ...' % model_file)
pickle.dump((reversed_dict, memory, model, loss_func, general_config),
f)
# Test on each task
print('Start to testing')
for t in tasks:
print("Testing for task %d ..." % (t + 1))
test_data_path = glob.glob('%s/qa%d_*_test.txt' % (data_dir, t + 1))
dc = len(dictionary)
test_story, test_questions, test_qstory = \
parse_babi_task(test_data_path,
dictionary,
False)
assert dc == len(
dictionary
) # make sure that the dictionary already covers all words
test(test_story, test_questions, test_qstory, memory, model, loss_func,
general_config)
import commons
import test
import train
import grl_util
######################################### SETUP #########################################
args = parser.parse_arguments()
start_time = datetime.now()
args.output_folder = f"runs/{args.exp_name}/{start_time.strftime('%Y-%m-%d_%H-%M-%S')}"
commons.setup_logging(args.output_folder)
commons.make_deterministic(args.seed)
logging.info(f"Arguments: {args}")
logging.info(f"The outputs are being saved in {args.output_folder}")
######################################### MODEL #########################################
model = util.build_model(args)
######################################### OPTIMIZER & LOSSES #########################################
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
criterion_netvlad = nn.TripletMarginLoss(margin=args.margin**0.5,
p=2,
reduction="sum")
######################################### RESUME #########################################
if args.resume:
model, optimizer, best_score, start_epoch = util.resume_train(
args, model, optimizer)
else:
start_epoch = 0
######################################### DATASETS #########################################
filename = 'shakespeare'
epochs = 20
batch_size = 64
seq_length = 100
rnn_units = 128 # 1024
embedding_dim = None # 256
dataset, idx_to_char, char_to_idx, vocab = preprocess(filename, batch_size,
seq_length)
vocab_size = len(vocab)
checkpoint_dir = './training_checkpoints/' + filename
checkpoint_prefix = checkpoint_dir + '/ckpt'
### Train model (comment out if only generating)
model = build_model(vocab_size, embedding_dim, rnn_units, batch_size)
# model.load_weights(tf.train.latest_checkpoint(checkpoint_dir)) # use this to continue training where it left off
history = train_model(model,
dataset,
epochs=epochs,
checkpoint_prefix=checkpoint_prefix)
### Generate sample (comment out if only training)
#model = build_model(vocab_size, embedding_dim, rnn_units, batch_size=1)
#model.load_weights(tf.train.latest_checkpoint(checkpoint_dir))
#model.build(tf.TensorShape([1, None]))
#print(generate_text(model,char_to_idx,idx_to_char,
# start_string=u"Romeo:\n"))