def main(cfg):
if cfg['model'] == 'mlp':
net = MLP(300, 768, cfg['class_num'])
elif cfg['model'] == 'cnn':
net = CNN(300, 768, cfg['class_num'])
elif cfg['model'] == 'lstm':
net = LSTM(300, cfg['class_num'], cfg['device'])
elif cfg['model'] == 'gru':
net = GRU(300, cfg['class_num'], cfg['device'])
else:
raise Exception(f'model {args.model} not available')
if cfg['device'] == 'cuda':
if len(cfg['gpu_ids']) == 1:
torch.cuda.set_device(cfg['gpu_ids'][0])
net = net.cuda()
else:
net = net.cuda()
net = nn.DataParallel(net, device_ids=cfg['gpu_ids'])
torch.backends.cudnn.benchmark = True
if cfg['mode'] == 'train':
train(cfg, net)
elif cfg['mode'] == 'predict':
predict(cfg, net, 'checkpoints/{}.pth'.format(cfg['model']))
def get_model(input_size, embed_size, output_size, model_type, dropout=DROPOUT):
if model_type.lower() == 'lstm':
return LSTM(input_size, embed_size, output_size, dropout)
if model_type.lower() == 'cnn':
return CNN(input_size, embed_size, output_size, dropout)
if model_type.lower() == 'gru':
return GRU(input_size, embed_size, output_size, dropout)
else:
return None
def __call__(self,
number_of_iterations=2,
learning_rate=0.005,
embedding_size=300,
hidden_size=100,
batch_size=100):
print("Starting 'Image Retrieval' in 'GRU' mode with '" +
self.difficulty + "' data")
self.model_full_path = self.model_path + "/" + self.model_name + "_" + self.timestamp + "_" + str(
learning_rate) + "_" + str(embedding_size) + ".pty"
self.output_file_name = self.output_path + "/" + self.model_name + "_" + self.timestamp + "_" + str(
learning_rate) + "_" + str(embedding_size) + ".csv"
self.number_of_iterations = number_of_iterations
self.learning_rate = learning_rate
self.embedding_size = embedding_size
self.hidden_size = hidden_size
self.batch_size = batch_size
self.model = GRU(self.nwords, self.embedding_size,
self.image_feature_size, self.output_vector_size,
self.hidden_size, self.batch_size)
self.criterion = nn.CrossEntropyLoss()
self.evaluate = Evaluate(self.model, self.img_features, self.minibatch,
self.preprocess, self.image_feature_size,
self.output_vector_size)
print(self.model)
self.optimizer = optim.Adam(self.model.parameters(),
lr=self.learning_rate)
self.train_loss_values = []
self.magic()
self.save_model()
self.save_data()
def test_func(test_y_file, word_file, pos1_file, pos2_file, ans_file,
res_file):
model_path = './model/ATT-MODEL'
word_embedding = np.load('./data/vec.npy')
wordpos1 = np.load('./data/wordpos1.npy')
wordpos2 = np.load('./data/wordpos2.npy')
#test_y = np.load('./data/test_y.npy')
#test_sentence_word = np.load('./data/test_sentence_word.npy')
#test_sentence_pos1 = np.load('./data/test_sentence_pos1.npy')
#test_sentence_pos2 = np.load('./data/test_sentence_pos2.npy')
#all_ans = np.load('./data/all_ans.npy')
test_y = np.load('./data/' + test_y_file)
test_sentence_word = np.load('./data/' + word_file)
test_sentence_pos1 = np.load('./data/' + pos1_file)
test_sentence_pos2 = np.load('./data/' + pos2_file)
all_ans = np.load('./data/' + ans_file)
all_alpha = np.load("./data/all_alpha.npy")
settings = Setting()
batch_size = settings.batch_size
with tf.Graph().as_default():
sess = tf.Session()
with sess.as_default():
print "model testing begin"
with tf.variable_scope("model"):
m = GRU(is_training=False,
word_embeddings=word_embedding,
word_pos1=wordpos1,
word_pos2=wordpos2,
settings=settings)
saver = tf.train.Saver()
i_length = int(
len(test_sentence_word) / float(settings.batch_size))
test_length = i_length * settings.batch_size
ClassEst = np.zeros((test_length, settings.num_classes - 1))
for epoch in range(settings.num_epochs - 25,
settings.num_epochs - 15):
alpha = all_alpha[epoch]
saver.restore(sess, model_path + str(epoch))
all_prob = []
for i in range(i_length):
print '%d/%d,test one batch' % (i, i_length)
temp_sentence_word = []
temp_sentence_pos1 = []
temp_sentence_pos2 = []
temp_sentence_y = []
temp_input = test_sentence_word[i *
settings.batch_size:(i +
1) *
settings.batch_size]
temp_sentence_word = test_sentence_word[
i * settings.batch_size:(i + 1) * settings.batch_size]
temp_sentence_pos1 = test_sentence_pos1[
i * settings.batch_size:(i + 1) * settings.batch_size]
temp_sentence_pos2 = test_sentence_pos2[
i * settings.batch_size:(i + 1) * settings.batch_size]
temp_sentence_y = test_y[i * settings.batch_size:(i + 1) *
settings.batch_size]
batch_sentence_word = np.array(temp_sentence_word)
batch_sentence_pos1 = np.array(temp_sentence_pos1)
batch_sentence_pos2 = np.array(temp_sentence_pos2)
batch_y = np.array(temp_sentence_y)
feed_dict = {}
feed_dict[m.input_word] = batch_sentence_word
feed_dict[m.input_pos1] = batch_sentence_pos1
feed_dict[m.input_pos2] = batch_sentence_pos2
feed_dict[m.input_y] = batch_y
loss_cur, acc, prob = sess.run(
[m.loss, m.accuracy, m.probility], feed_dict=feed_dict)
#_, step = sess.run([train_op, global_step], feed_dict=feed_dict)
for single_prob in prob:
#all_prob.append(single_prob)
#all_prob.append(single_prob[1:])
all_prob.append(single_prob[1:])
ClassEst += np.multiply(all_prob, alpha)
all_prob2 = []
for index in range(test_length):
all_prob2.append(np.argmax(softmax(ClassEst[index])))
all_prob2 = np.reshape(np.array(all_prob2), -1)
all_prob_length = len(all_prob2)
#np.save('./output/all_prob.npy', all_prob2)
np.save('./output/' + res_file, all_prob2)
#print 'avg_pre_score'
#print metrics.average_precision_score(all_ans[:all_prob_length], all_prob2)
y_true = all_ans[:all_prob_length]
y_pred = all_prob2
#print 'PR curve area:' + str(average_precision)
#print(classification_report(all_ans[:all_prob_length], all_prob2))
order = np.argsort(-all_prob2)
top100 = order[:100]
correct_num_100 = 0.0
for i in top100:
if all_ans[i] == 1:
correct_num_100 += 1.0
top200 = order[:200]
correct_num_200 = 0.0
for i in top200:
if all_ans[i] == 1:
correct_num_200 += 1.0
top300 = order[:300]
correct_num_300 = 0.0
for i in top300:
if all_ans[i] == 1:
correct_num_300 += 1.0
top400 = order[:400]
correct_num_400 = 0.0
for i in top400:
if all_ans[i] == 1:
correct_num_400 += 1.0
top500 = order[:500]
correct_num_500 = 0.0
for i in top500:
if all_ans[i] == 1:
correct_num_500 += 1.0
top1000 = order[:1000]
correct_num_1000 = 0.0
for i in top1000:
if all_ans[i] == 1:
correct_num_1000 += 1.0
print 'P@100\n' + str(correct_num_100 / 100.0)
print 'P@200\n' + str(correct_num_200 / 200.0)
print 'P@300\n' + str(correct_num_300 / 300.0)
print 'P@400\n' + str(correct_num_400 / 400.0)
print 'P@500\n' + str(correct_num_500 / 500.0)
print 'P@1000\n' + str(correct_num_1000 / 1000.0)
print '~~~Ending'
def train_model(exp_name, train_tfrecord, val_tfrecord, dictionary_file,
n_hidden, learn_rate, batch_size, decouple_split=200,
patience=10, max_epochs=200, sample_length=16, resume=False):
"""
Train a GRU on some text data
:param exp_name: experiment name (saved to ~/experiments/story-gen/exp_name)
:param train_tfrecord: path to tfrecord of training set
:param val_tfrecord: path to tfrecord of validation set
:param dictionary_file: path to dictionary json file
:param n_hidden: number of hidden units in GRU
:param learn_rate: learning rate
:param batch_size: batch size
:param decouple_split: subsequence length between decoupled neural interface
or None to not use decoupled neural intefaces
:param patience: early stopping limit
:param max_epochs: maximum number of epochs to run
:param sample_length: length of sample to generate after each epoch
:param resume: resume from previous run
:return:
"""
exp_dir = os.path.join(os.path.expanduser('~/experiments/story-gen/'),
exp_name)
if not os.path.isdir(exp_dir):
os.makedirs(exp_dir)
with open(dictionary_file,'r') as f:
reverse_dict = json.load(f) # word -> int
reverse_dict = {v+1:k for k,v in reverse_dict.items()} # int -> word
# note: sequences are padded with zero, add to dict_size (for embedding)
reverse_dict[0] = '_END_' # this should be removed from sampled output
dict_size = max(reverse_dict.keys())+1
if not resume:
max_sequence = 20000 if decouple_split is not None else 100
pipeline = Vector_Pipeline(train_tfrecord, val_tfrecord, batch_size,
max_sequence=max_sequence)
init_train, init_val = pipeline.init_train, pipeline.init_val
model_input = tf.placeholder_with_default(pipeline.output[:,:-1],
[None, None], 'input')
# Embedding
embedding = orthogonal([dict_size, n_hidden], 'embedding')
embedded_input = tf.nn.embedding_lookup(embedding, model_input)
int_label = pipeline.output[:,1:]
# Decoupled neural interface (optional)
decoupled = decouple_split is not None
if decoupled:
# Split subsequences, reshape to [slow_time, batch, fast_time, feat]
seq_len = tf.shape(embedded_input)[1]
# pad so sequence length is divisible by subsequence length
pad_len = decouple_split-tf.mod(seq_len,tf.constant(decouple_split))
embedded_input = tf.pad(embedded_input, [[0,0], [0,pad_len], [0,0]],
mode='CONSTANT', constant_values=0)
int_label = tf.pad(int_label, [[0,0], [0,pad_len]])
# batch x features x time
dni_input = tf.transpose(embedded_input, [0,2,1])
# batch x features x slow_time x fast_time
dni_input = tf.reshape(
dni_input,
[-1, n_hidden,
(seq_len+pad_len)//decouple_split, decouple_split])
# fast_time x features x batch x slow_time
dni_input = tf.transpose(dni_input, [3,1,0,2])
# fast_time x features x (batch x slow_time)
dni_input = tf.reshape(dni_input, [decouple_split, n_hidden, -1])
# (batch x slow_time) x fast_time x features
dni_input = tf.transpose(dni_input, [2,0,1])
# (batch x slow_time) x (fast_time x features)
dni_input = tf.reshape(dni_input, [tf.shape(dni_input)[0],-1])
# Decoupled neural interface: simplify to single dense layer
dni = Dense(dni_input, n_hidden, tf.nn.relu, name='dni',
init='uniform', n_in=n_hidden*decouple_split)
# Reshape DNI out & embedded_input to new_batch x fast_time for GRU
gru_hidden = tf.reshape(dni.output, [-1, n_hidden])
embedded_input = tf.reshape(embedded_input,
[-1, decouple_split, n_hidden])
int_label = tf.reshape(int_label, [-1, decouple_split])
else:
gru_hidden = None
# model part2: GRU
# transpose: tf.scan needs time x batch x features
embedded_input = tf.transpose(embedded_input, [1,0,2])
training_toggle = tf.placeholder(tf.int32, name='training_toggle')
gru = GRU(embedded_input, n_hidden, training_toggle, h0=gru_hidden,
name='gru')
gru_h0 = gru.h0
gru_output = gru.output
# model part3: dropout and dense layer
dropout_rate = tf.placeholder(tf.float32, name='dropout_rate')
dropped = tf.nn.dropout(gru_output, 1-dropout_rate)
dense = Dense(dropped, dict_size)
model_output = tf.identity(dense.output, 'output')
# cross-entropy loss
# note: sequences padded with -1, mask these entries
mask = tf.not_equal(int_label, -1)
# swap -1's to avoid error in loss fcn, even though we're ignoring these
int_label = tf.where(mask, int_label, tf.zeros_like(int_label))
# mean over entries with mask==1
mask = tf.cast(mask, dtype=tf.float32)
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=int_label, logits=model_output)
loss = tf.reduce_sum(mask*loss)/tf.reduce_sum(mask)
if decoupled:
# decoupled neural interface loss
dni_label = tf.stop_gradient(gru.output)
dni_loss = tf.reduce_mean(tf.square(dni_label-dni.output),
name='dni_loss')
else:
dni_loss = tf.constant(0., dtype=tf.float32)
train_step = tf.train.AdamOptimizer(learn_rate).minimize(
loss+dni_loss,name='train_step')
else:
(model_input, training_toggle, dropout_rate, train_step, init_train,
init_val, loss, dni_loss, gru_output, gru_h0, model_output
) = reload_graph(exp_dir)
n_examples = tf.shape(model_input)[0]
sampled_out = tf.multinomial(model_output[0,:1,:],num_samples=1)
def epoch_callback(sess):
# TODO: not sure how to initialize this since it's usually from the DNI
h0 = np.random.rand(1, n_hidden)
sampled_text = [np.random.randint(0,dict_size,size=(1,1))]
for i in range(sample_length+1):
out,h0 = sess.run([sampled_out, gru_output],
feed_dict={gru_h0:h0,
model_input:sampled_text[i],
dropout_rate:0,
training_toggle:0})
h0 = h0[0]
sampled_text.append(out)
sampled_text = sampled_text[1:]
# temp bugfix: screwed up the reverse dictionary, missing keys
if any([int(o) not in reverse_dict.keys() for o in sampled_text]):
sampled_text = [
o if int(o) in reverse_dict.keys()
else int(np.random.choice(list(reverse_dict.keys())))
for o in sampled_text]
print(' '.join([reverse_dict[int(o)] for o in sampled_text]))
print('')
fit(training_toggle, dropout_rate, train_step, init_train, init_val, loss,
dni_loss, n_examples, patience, max_epochs, exp_dir, epoch_callback,
resume)
save_path = 'data/bmli_none.pth'
data_loader = PairedComparison(4,
ranking=ranking,
direction=direction,
dichotomized=False)
gini_coefficients = torch.zeros(len(file_names), args.num_experiments,
args.num_episodes, args.sequence_length)
map_performance = torch.zeros(len(file_names), args.num_experiments,
args.num_episodes, args.sequence_length)
avg_performance = torch.zeros(len(file_names), args.num_experiments,
args.num_episodes, args.sequence_length)
for m, file_name in enumerate(file_names):
model = GRU(data_loader.num_inputs, data_loader.num_targets, 128)
params, _ = torch.load(file_name, map_location='cpu')
model.load_state_dict(params)
for k in tqdm(range(args.num_experiments)):
for i in range(args.num_episodes):
inputs, targets, _, _ = data_loader.get_batch(
1, args.sequence_length)
predictive_distribution, weights, variances = model(
inputs, targets)
map_performance[m, k, i] = (
(predictive_distribution.probs >
0.5).float() == targets).squeeze().detach()
avg_performance[m, k,
def main(trial_num):
# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model_type = "lstm"
# Hyper-parameters
sequence_length = 28
input_size = 28
num_layers = 1
hidden_size = 128
num_classes = 10
batch_size = 100
num_epochs = 20
learning_rate = 0.01
num_trials = 100
a_range = [1.0, 3.0]
# a_s = [1.5, 2.0, 2.2, 2.5, 2.7, 3.0]
# just for testing
# num_trials = 1
# num_epochs = 20
# a_s = [1.0]
# for a in a_s:
trials = {}
for num_trial in range(num_trials):
a = random.random() * (a_range[1] - a_range[0]) + a_range[0]
print('trial Num: ', trial_num, "a: ", a, "num_trial: ", num_trial)
trial = {}
trial['a'] = a
# define model
if model_type == 'lstm':
model = LSTM(input_size, hidden_size, num_layers, num_classes, a, device).to(device)
elif model_type == 'gru':
model = GRU(input_size, hidden_size, num_layers, num_classes, a, device).to(device)
# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
train_dataloader = MNIST_dataloader(batch_size, train=True)
test_dataloader = MNIST_dataloader(batch_size, train=False)
# Train the model
total_step = len(train_dataloader.dataloader)
total = 0
total_loss = 0
for epoch in range(num_epochs):
model.train()
for i, (images, labels) in enumerate(train_dataloader.dataloader):
images = images.reshape(-1, sequence_length, input_size).to(device)
labels = labels.to(device)
# Forward pass
outputs, hts = model(images)
loss = criterion(outputs, labels)
total_loss += loss * labels.size(0)
total += labels.size(0)
# print(LEs, rvals)
# Backward and optimize
optimizer.zero_grad()
loss.backward()
optimizer.step()
# if (i + 1) % 300 == 0:
# print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'
# .format(epoch + 1, num_epochs, i + 1, total_step, total_loss / total))
# for i, (name, param) in enumerate(model.named_parameters()):
# if i == 3:
# print(name, param)
# Test the model
model.eval()
with torch.no_grad():
correct = 0
total = 0
total_loss = 0
for i, (images, labels) in enumerate(test_dataloader.dataloader):
images = images.reshape(-1, sequence_length, input_size).to(device)
labels = labels.to(device)
outputs, _ = model(images)
# h = torch.zeros(model.num_layers, images.size(0), model.hidden_size).to(model.device)
# c = torch.zeros(model.num_layers, images.size(0), model.hidden_size).to(model.device)
# params = (images, (h, c))
# if i == 0:
# LEs, rvals = calc_LEs_an(*params, model=model)
loss = criterion(outputs, labels)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
total_loss += loss * labels.size(0)
if epoch == (num_epochs - 1):
print('Epoch [{}/{}] Loss: {}, Test Accuracy: {} %'.format(epoch + 1, num_epochs, total_loss / total, 100 * correct / total))
saved_model = copy.deepcopy(model)
trial[epoch] = {"model": saved_model, "accuracy": 100 * correct / total, "loss": total_loss / total}
del saved_model
trials[num_trial] = trial
pickle.dump(trials, open('trials/{}/models/{}_{}_trials_{}.pickle'.format(model_type, model_type, hidden_size, trial_num), 'wb'))
def main(_):
print 'reading word-embedding'
wordembedding = np.load('./data/vec.npy')
wordpos1 = np.load('./data/wordpos1.npy')
wordpos2 = np.load('./data/wordpos2.npy')
print 'reading training data'
train_y = np.load('./data/train_y.npy')
train_sentence_word = np.load('./data/train_sentence_word.npy')
train_sentence_pos1 = np.load('./data/train_sentence_pos1.npy')
train_sentence_pos2 = np.load('./data/train_sentence_pos2.npy')
precious_record = open('./precious_record.txt', 'w')
save_path = './model/ATT-MODEL'
settings = Setting()
settings.num_classes = len(train_y[0])
settings.voc_size = len(wordembedding)
batch_size = settings.batch_size
with tf.Graph().as_default():
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
with sess.as_default():
initializer = tf.contrib.layers.xavier_initializer()
print "model training begin"
with tf.variable_scope("model",
reuse=None,
initializer=initializer):
m = GRU(is_training=True,
word_embeddings=wordembedding,
word_pos1=wordpos1,
word_pos2=wordpos2,
settings=settings)
merged = tf.summary.merge_all()
writer = tf.summary.FileWriter("logs/",
sess.graph) # tensorflow >=0.12
global_step = tf.Variable(0, name="global_step", trainable=False)
optimizer = tf.train.AdamOptimizer(settings.learning_rate)
all_alpha = []
i_length = int(
len(train_sentence_word) / float(settings.batch_size))
train_length = i_length * settings.batch_size
global ada_length
ada_length = i_length
global ada_weight
ada_weight = np.array(np.ones(ada_length) / ada_length)
global i
i = 0
ggClassEst = np.zeros((train_length, 1))
params = tf.trainable_variables()
g = tf.get_default_graph()
with g.gradient_override_map({"Ada": "AdaGrad"}):
m.loss = tf.identity(m.loss, name="Ada")
train_op = optimizer.minimize(m.loss, global_step=global_step)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=None)
print 'Starting Learning'
feed_dict_list = []
for epoch in range(settings.num_epochs):
rand_order = range(i_length)
#np.random.shuffle(rand_order)
ii = 0
all_prob = []
all_err = []
all_comp = []
batch_err = []
batch_comp = []
for index in range(10):
print ada_weight[index]
for i in rand_order:
print('epoch-%d, %d/%d (%d):run one batch' %
(epoch, ii, i_length, i))
ii += 1
temp_sentence_word = []
temp_sentence_pos1 = []
temp_sentence_pos2 = []
temp_sentence_y = []
temp_sentence_word = train_sentence_word[
i * settings.batch_size:(i + 1) * settings.batch_size]
temp_sentence_pos1 = train_sentence_pos1[
i * settings.batch_size:(i + 1) * settings.batch_size]
temp_sentence_pos2 = train_sentence_pos2[
i * settings.batch_size:(i + 1) * settings.batch_size]
temp_sentence_y = train_y[i * settings.batch_size:(i + 1) *
settings.batch_size]
batch_sentence_word = np.array(temp_sentence_word)
batch_sentence_pos1 = np.array(temp_sentence_pos1)
batch_sentence_pos2 = np.array(temp_sentence_pos2)
batch_y = np.array(temp_sentence_y)
feed_dict = {}
feed_dict[m.input_word] = batch_sentence_word
feed_dict[m.input_pos1] = batch_sentence_pos1
feed_dict[m.input_pos2] = batch_sentence_pos2
feed_dict[m.input_y] = batch_y
#temp, step, loss_cur, acc = sess.run([train_op, global_step, m.loss, m.accuracy], feed_dict=feed_dict)
temp, step, loss_cur, acc, prob, comp, err = sess.run(
[
train_op, global_step, m.loss, m.accuracy,
m.probility, m.comparison, m.difference
],
feed_dict=feed_dict)
time_str = datetime.datetime.now().isoformat()
for single_prob in prob:
all_prob.append(single_prob)
for single_err in err:
all_err.append(single_err)
for single_comp in comp:
all_comp.append(single_comp)
batch_err.append(np.sum(err) / settings.batch_size)
batch_comp.append(np.sum(comp) / settings.batch_size)
if step % 10 == 0:
print(" {}: step {}, loss {:g}, acc {:g}".format(
time_str, step, loss_cur, acc))
rs = sess.run(merged, feed_dict=feed_dict)
writer.add_summary(rs, step)
#all_prob = np.reshape(np.array(all_prob), -1)
#all_prob_length = len(all_prob)
#average_precision = average_precision_score(all_ans[:all_prob_length], all_prob)
#print 'PR curve area:' + str(average_precision)
#errorMisClassified = sum([np.exp(all_err[index]) for index in range(train_length) if not all_comp[index]])
#errorWellClassified = sum([np.exp(all_err[index]) for index in range(train_length) if all_comp[index]])
errorMisClassified = sum([
np.exp(batch_err[index]) for index in range(ada_length)
if batch_comp[index] < 0.5
])
errorWellClassified = sum([
np.exp(batch_err[index]) for index in range(ada_length)
if batch_comp[index] >= 0.5
])
alpha = 0.5 * np.log(errorWellClassified / errorMisClassified)
print 'alpha:', alpha
all_alpha.append(alpha)
for index in range(ada_length):
if batch_comp[index] >= 0.5:
ada_weight[index] *= np.exp(-alpha + batch_err[index])
else:
ada_weight[index] *= np.exp(alpha + batch_err[index])
max_ada_weight = max(ada_weight)
global delta
delta = 1.0 / max_ada_weight
print 'delta:', delta
ada_weight = np.dot(ada_weight, 1. / sum(ada_weight))
precious_record.writelines("epoch: " + str(epoch) +
" alpha: " + str(alpha) + "\n")
saver.save(sess, save_path + str(epoch))
np.save('./data/all_alpha.npy', all_alpha)
saver.save(sess, save_path + "_final")
print 'Trainning finished, Model being saved'
#current_step = tf.train.global_step(sess, global_step)
#path = saver.save(sess, save_path+'ATT_Model', global_step=current_step)
#saver.save(sess, save_path)
print '-------end------'
def main(datasets, U, n_epochs=20, batch_size=20, max_l=100, hidden_size=100, \
word_embedding_size=100, session_hidden_size=50, session_input_size=50, \
model_name='SMN_last.bin', learning_rate=0.001, r_seed=3435, \
val_frequency=100):
hiddensize = hidden_size
U = U.astype(dtype=theano.config.floatX)
rng = np.random.RandomState(r_seed)
lsize, rsize = max_l, max_l
sessionmask = T.matrix()
lx = [] #tokens from previous turns
lxmask = [] #masks from previous turns
for i in range(max_turn):
lx.append(T.matrix())
lxmask.append(T.matrix())
index = T.lscalar()
rx = T.matrix('rx') #tokens from response
rxmask = T.matrix() #masks from response
y = T.ivector('y')
Words = theano.shared(value=U, name="Words")
llayer0_input = []
for i in range(max_turn):
llayer0_input.append(Words[T.cast(lx[i].flatten(), dtype="int32")] \
.reshape((lx[i].shape[0], lx[i].shape[1], Words.shape[1])))
# input: word embeddings of the mini batch
rlayer0_input = Words[T.cast(rx.flatten(), dtype="int32")].\
reshape((rx.shape[0], rx.shape[1], Words.shape[1]))
train_set, dev_set, test_set = datasets[0], datasets[1], datasets[2]
train_set_lx = []
train_set_lx_mask = []
q_embedding = []
q_embedding_Cat = []
q_embedding_Cat_mask = []
q_embedding_self_att = []
q_embedding_self_att_rnn = []
q_embedding_hiddenequal = []
offset = 2 * lsize
for i in range(max_turn):
train_set_lx.append(theano.shared(
np.asarray(a=train_set[:, offset*i:offset*i+lsize], \
dtype=theano.config.floatX), \
borrow=True))
train_set_lx_mask.append(theano.shared(
np.asarray(a=train_set[:, offset*i + lsize:offset*i + 2*lsize], \
dtype=theano.config.floatX), \
borrow=True))
train_set_rx = theano.shared(
np.asarray(a=train_set[:, offset*max_turn:offset*max_turn + lsize], \
dtype=theano.config.floatX), \
borrow=True)
train_set_rx_mask = theano.shared(
np.asarray(a=train_set[:, offset*max_turn+lsize:offset*max_turn + 2*lsize], \
dtype=theano.config.floatX), \
borrow=True)
train_set_session_mask = theano.shared(
np.asarray(a=train_set[:, -max_turn-1:-1], \
dtype=theano.config.floatX), \
borrow=True)
train_set_y = theano.shared(np.asarray(train_set[:, -1], dtype="int32"), \
borrow=True)
val_set_lx = []
val_set_lx_mask = []
for i in range(max_turn):
val_set_lx.append(theano.shared(
np.asarray(a=dev_set[:, offset*i:offset*i + lsize], \
dtype=theano.config.floatX), \
borrow=True))
val_set_lx_mask.append(theano.shared(
np.asarray(a=dev_set[:, offset*i + lsize:offset*i + 2*lsize], \
dtype=theano.config.floatX), \
borrow=True))
val_set_rx = theano.shared(
np.asarray(a=dev_set[:, offset*max_turn:offset*max_turn + lsize], \
dtype=theano.config.floatX), \
borrow=True)
val_set_rx_mask = theano.shared(
np.asarray(a=dev_set[:, offset*max_turn + lsize:offset*max_turn + 2*lsize], \
dtype=theano.config.floatX), \
borrow=True)
val_set_session_mask = theano.shared(np.asarray(a=dev_set[:, -max_turn-1:-1], \
dtype=theano.config.floatX), \
borrow=True)
val_set_y = theano.shared(np.asarray(dev_set[:, -1], dtype="int32"),
borrow=True)
test_set_lx = []
test_set_lx_mask = []
for i in range(max_turn):
test_set_lx.append(theano.shared(
np.asarray(a=test_set[:, offset*i:offset*i + lsize], \
dtype=theano.config.floatX), \
borrow=True))
test_set_lx_mask.append(theano.shared(
np.asarray(a=test_set[:, offset*i + lsize:offset*i + 2*lsize], \
dtype=theano.config.floatX), \
borrow=True))
test_set_rx = theano.shared(
np.asarray(a=test_set[:, offset*max_turn:offset*max_turn + lsize], \
dtype=theano.config.floatX), \
borrow=True)
test_set_rx_mask = theano.shared(
np.asarray(a=test_set[:, offset*max_turn + lsize:offset*max_turn + 2*lsize], \
dtype=theano.config.floatX), \
borrow=True)
test_set_session_mask = theano.shared(np.asarray(a=test_set[:, -max_turn-1:-1], \
dtype=theano.config.floatX), \
borrow=True)
test_set_y = theano.shared(np.asarray(test_set[:, -1], dtype="int32"), \
borrow=True)
dic = {}
for i in range(max_turn):
dic[lx[i]] = train_set_lx[i][index * batch_size:(index + 1) *
batch_size]
dic[lxmask[i]] = train_set_lx_mask[i][index * batch_size:(index + 1) *
batch_size]
dic[rx] = train_set_rx[index * batch_size:(index + 1) * batch_size]
dic[sessionmask] = train_set_session_mask[index * batch_size:(index + 1) *
batch_size]
dic[rxmask] = train_set_rx_mask[index * batch_size:(index + 1) *
batch_size]
dic[y] = train_set_y[index * batch_size:(index + 1) * batch_size]
val_dic = {}
for i in range(max_turn):
val_dic[lx[i]] = val_set_lx[i][index * batch_size:(index + 1) *
batch_size]
val_dic[lxmask[i]] = val_set_lx_mask[i][index *
batch_size:(index + 1) *
batch_size]
val_dic[rx] = val_set_rx[index * batch_size:(index + 1) * batch_size]
val_dic[sessionmask] = val_set_session_mask[index *
batch_size:(index + 1) *
batch_size]
val_dic[rxmask] = val_set_rx_mask[index * batch_size:(index + 1) *
batch_size]
val_dic[y] = val_set_y[index * batch_size:(index + 1) * batch_size]
test_dic = {}
for i in range(max_turn):
test_dic[lx[i]] = test_set_lx[i][index * batch_size:(index + 1) *
batch_size]
test_dic[lxmask[i]] = test_set_lx_mask[i][index *
batch_size:(index + 1) *
batch_size]
test_dic[rx] = test_set_rx[index * batch_size:(index + 1) * batch_size]
test_dic[sessionmask] = test_set_session_mask[index *
batch_size:(index + 1) *
batch_size]
test_dic[rxmask] = test_set_rx_mask[index * batch_size:(index + 1) *
batch_size]
test_dic[y] = test_set_y[index * batch_size:(index + 1) * batch_size]
# This is the first RNN.
sentence2vec = GRU(n_in=word_embedding_size, n_hidden=hiddensize, \
n_out=hiddensize, batch_size=batch_size)
for i in range(max_turn):
q_embedding.append(sentence2vec(llayer0_input[i], lxmask[i], True))
r_embedding = sentence2vec(rlayer0_input, rxmask, True)
# This is the concat/elementwise_produce of the after the first RNN which
# concat the tenth sentence to the first nine sentences.
for i in range(max_turn):
q_embedding_Cat.append(T.concatenate([q_embedding[i], \
q_embedding[-1]], \
axis=2))
q_embedding_Cat_mask.append(lxmask[i])
r_embedding_Cat = T.concatenate([r_embedding, q_embedding[-1]], axis=2)
r_embedding_Cat_mask = rxmask
# This is the self_attention step
sa = self_attention(n_in=hiddensize * 2)
for i in range(max_turn):
q_embedding_self_att.append(T.concatenate([q_embedding_Cat[i], \
sa(q_embedding_Cat[i], \
q_embedding_Cat_mask[i])], \
axis=2))
r_embedding_self_att = T.concatenate([r_embedding_Cat, \
sa(r_embedding_Cat, \
r_embedding_Cat_mask)], \
axis=2)
# This is the SRNN
vec2svec = SGRU(n_in=hiddensize*2, n_hidden=hiddensize, \
n_out=hiddensize, batch_size=batch_size)
for i in range(max_turn):
q_embedding_self_att_rnn.append(vec2svec(q_embedding_self_att[i], \
q_embedding_Cat_mask[i], \
True))
r_embedding_self_att_rnn = vec2svec(r_embedding_self_att, \
r_embedding_Cat_mask, \
True)
# This is the CNN with pooling and full-connection
pooling_layer = ConvSim(rng=rng, n_in=max_l, n_out=session_input_size, \
hidden_size=hiddensize, session_size=session_hidden_size, \
batch_size=batch_size)
poolingoutput = []
for i in range(max_turn):
poolingoutput.append(pooling_layer(llayer0_input[i], \
rlayer0_input, \
q_embedding_self_att_rnn[i], \
r_embedding_self_att_rnn))
# This is the second RNN
session2vec = GRU(n_in=session_input_size, n_hidden=session_hidden_size, \
n_out=session_hidden_size, batch_size=batch_size)
res = session2vec(T.stack(poolingoutput, 1), sessionmask, True)
# This is the final Attention and put the output to a classifier
W = theano.shared(ortho_weight(session_hidden_size), borrow=True)
W2 = theano.shared(glorot_uniform((hiddensize, session_hidden_size)),
borrow=True)
b = theano.shared(value=np.zeros((session_hidden_size, ), dtype='float32'),
borrow=True)
U_s = theano.shared(glorot_uniform((session_hidden_size, 1)), borrow=True)
final = T.dot(T.tanh(T.dot(res, W) + \
T.dot(T.stack(q_embedding_self_att_rnn, 1)[:, :, -1, :], W2) \
+ b), U_s)
weight = T.exp(T.max(final, 2)) * sessionmask
weight2 = weight / T.sum(weight, 1)[:, None]
final2 = T.sum(res * weight2[:, :, None], 1) + 1e-6
# This is the classifier
classifier = LogisticRegression(final2, session_hidden_size, 2, rng)
# Calculate the cost and updata the param with gradient
cost = classifier.negative_log_likelihood(y)
error = classifier.errors(y)
predict = classifier.predict_prob
opt = Adam()
# Make params
params = classifier.params
params += sentence2vec.params
params += session2vec.params
params += pooling_layer.params
params += [Words, W, b, W2, U_s]
params += vec2svec.params
params += sa.params
# Make updater
grad_updates = opt.Adam(cost=cost, params=params, lr=learning_rate)
# The training step
train_model = theano.function([index], cost, updates=grad_updates, \
givens=dic, on_unused_input='ignore')
val_model = theano.function([index], [cost, error], givens=val_dic, \
on_unused_input='ignore')
best_dev = 1.
n_train_batches = datasets[0].shape[0] / batch_size
for i in xrange(n_epochs):
cost_all = 0
total = 0.
for minibatch_index in np.random.permutation(range(n_train_batches)):
batch_cost = train_model(minibatch_index)
total = total + 1
cost_all = cost_all + batch_cost
if total % val_frequency == 0:
sf.write('epcho %d, num %d, train_loss %f' %
(i, total, cost_all / total))
sf.write('\n')
sf.flush()
cost_dev = 0
errors_dev = 0
j = 0
for minibatch_index in xrange(datasets[1].shape[0] /
batch_size):
tcost, terr = val_model(minibatch_index)
cost_dev += tcost
errors_dev += terr
j = j + 1
cost_dev = cost_dev / j
errors_dev = errors_dev / j
if cost_dev < best_dev:
best_dev = cost_dev
save_params(params, model_name + 'dev')
sf.write("epcho %d, num %d, dev_loss %f" %
(i, total, cost_dev))
sf.write('\n')
sf.write("epcho %d, num %d, dev_accuracy %f" %
(i, total, 1 - errors_dev))
sf.write('\n')
sf.flush()
cost_all = cost_all / n_train_batches
sf.write("epcho %d loss %f" % (i, cost_all))
sf.write('\n')
sf.flush()
'trained_models/none_1_0.pth',
'trained_models/none_2_0.pth',
'trained_models/none_3_0.pth',
'trained_models/none_4_0.pth',
'trained_models/none_5_0.pth',
'trained_models/none_6_0.pth',
'trained_models/none_pretrained_0.pth'
]
logprobs = torch.zeros(inputs_a.shape[0], len(model_paths))
# for each participant
with torch.no_grad():
for participant in tqdm(range(inputs_a.shape[0])):
# for each model
for m, model_path in enumerate(model_paths):
model = GRU(4, 1, 128)
params, _ = torch.load(model_path, map_location='cpu')
model.load_state_dict(params)
participant_inputs = inputs_a[participant] - inputs_b[participant]
participant_targets = targets[participant]
avg_probs = 0
for sample in range(args.samples):
predictive_distribution, _, _ = model(participant_inputs, participant_targets)
avg_probs += predictive_distribution.probs
avg_predictive_distribution = Bernoulli(avg_probs / args.samples)
logprobs[participant, m] = avg_predictive_distribution.log_prob(predictions[participant]).sum()
class Train():
def __init__(self, difficulty):
self.data_path = "../data"
self.model_path = "../models"
self.output_path = "../outputs"
self.difficulty = difficulty
self.timestamp = str(int(time.time()))
self.model_name = "gru_" + self.difficulty
self.data = Data(difficulty=self.difficulty, data_path=self.data_path)
(self.img_features, self.w2i, self.i2w, self.nwords, self.UNK,
self.PAD) = self.data()
self.train = list(self.data.get_train_data())
self.dev = list(self.data.get_validation_data())
self.test = list(self.data.get_test_data())
self.image_feature_size = 2048
self.output_vector_size = 10
def __call__(self,
number_of_iterations=2,
learning_rate=0.005,
embedding_size=300,
hidden_size=100,
batch_size=100):
print("Starting 'Image Retrieval' in 'GRU' mode with '" +
self.difficulty + "' data")
self.model_full_path = self.model_path + "/" + self.model_name + "_" + self.timestamp + "_" + str(
learning_rate) + "_" + str(embedding_size) + ".pty"
self.output_file_name = self.output_path + "/" + self.model_name + "_" + self.timestamp + "_" + str(
learning_rate) + "_" + str(embedding_size) + ".csv"
self.number_of_iterations = number_of_iterations
self.learning_rate = learning_rate
self.embedding_size = embedding_size
self.hidden_size = hidden_size
self.batch_size = batch_size
self.model = GRU(self.nwords, self.embedding_size,
self.image_feature_size, self.output_vector_size,
self.hidden_size, self.batch_size)
self.criterion = nn.CrossEntropyLoss()
self.evaluate = Evaluate(self.model, self.img_features, self.minibatch,
self.preprocess, self.image_feature_size,
self.output_vector_size)
print(self.model)
self.optimizer = optim.Adam(self.model.parameters(),
lr=self.learning_rate)
self.train_loss_values = []
self.magic()
self.save_model()
self.save_data()
def minibatch(self, data, batch_size=50):
for i in range(0, len(data), batch_size):
yield data[i:i + batch_size]
def preprocess(self, batch):
"""Helper function for functional batches"""
correct_indexes = [observation[2] for observation in batch]
img_ids = [observation[1] for observation in batch]
text_features = [observation[0] for observation in batch]
last_words = [len(dialog) for dialog in text_features]
#Add Padding to max len of sentence in batch
max_length = max(map(len, text_features))
text_features = [
txt + [self.PAD] * (max_length - len(txt)) for txt in text_features
]
#return in "stacked" format, added last_words for excluding padding effects on GRU
return text_features, img_ids, correct_indexes, last_words
def magic(self):
for ITER in range(self.number_of_iterations):
random.shuffle(self.train)
train_loss = 0.0
start = time.time()
iteration = 0
for batch in self.minibatch(self.train, self.batch_size):
self.model.zero_grad()
self.optimizer.zero_grad()
self.model.hidden = self.model.init_hidden()
#Load data for model
text_features, h5_ids, correct_index, last_words = self.preprocess(
batch)
lookup_text_tensor = Variable(torch.LongTensor([text_features
])).squeeze()
full_img_batch = np.empty([
len(batch), self.output_vector_size,
self.image_feature_size
])
for obs, img_ids in enumerate(h5_ids):
for index, h5_id in enumerate(img_ids):
full_img_batch[obs, index] = self.img_features[h5_id]
full_img_batch = Variable(
torch.from_numpy(full_img_batch).type(torch.FloatTensor))
#Target
target = Variable(torch.LongTensor([correct_index])).squeeze()
#Vector for excluding padding effects
last_words = Variable(torch.LongTensor(last_words))
#Run model and calculate loss
prediction = self.model(lookup_text_tensor, full_img_batch,
last_words)
loss = self.criterion(prediction, target)
train_loss += loss.data[0]
iteration += self.batch_size
print(iteration)
loss.backward()
self.optimizer.step()
print(
"ITERATION %r: train loss/sent=%.4f, time=%.2fs" %
(ITER + 1, train_loss / len(self.train), time.time() - start))
self.train_loss_values.append(train_loss / len(self.train))
def save_model(self):
#Save model
torch.save(self.model, self.model_full_path)
print("Saved model has test score",
self.evaluate(self.test, self.batch_size))
def plot(self):
plt.plot(self.train_loss_values, label="Train loss")
plt.legend(loc='best')
plt.xlabel("Epochs")
plt.ylabel("Loss")
plt.title(self.model_name +
" - has loss with lr = %.4f, embedding size = %r" %
(self.learning_rate, self.embedding_size))
plt.show()
def save_data(self):
file = open(self.output_file_name, "w")
file.write(", ".join(map(str, self.train_loss_values)))
file.write("\n")
file.write(str(self.evaluate(self.test, self.batch_size)))
file.write("\n")
file.close()
def main():
# Device configuration
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model_type = "lstm"
# Hyper-parameters
sequence_length = 28
input_size = 28
num_layers = 1
num_classes = 10
batch_size = 100
num_epochs = 10
learning_rate = 0.01
num_trials = 100
a_s = [2]
trials = {}
# just for testing
num_trials = 1
num_epochs = 5
a_s = np.random.uniform(0.1, 2, [2])
for a in a_s:
for num_trial in range(num_trials):
print("a: ", a, "num_trial: ", num_trial)
hidden_size = 8
trial = {}
if model_type == 'lstm':
model = LSTM(input_size, hidden_size, num_layers, num_classes, a, device).to(device)
elif model_type == 'gru':
model = GRU(input_size, hidden_size, num_layers, num_classes, a, device).to(device)
# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
train_dataloader = MNIST_dataloader(batch_size, train=True)
test_dataloader = MNIST_dataloader(batch_size, train=False)
# Train the model
total_step = len(train_dataloader.dataloader)
for epoch in range(num_epochs):
model.train()
for i, (images, labels) in enumerate(train_dataloader.dataloader):
images = images.reshape(-1, sequence_length, input_size).to(device)
labels = labels.to(device)
# Forward pass
outputs, hts = model(images)
loss = criterion(outputs, labels)
# print(LEs, rvals)
# Backward and optimize
optimizer.zero_grad()
loss.backward()
optimizer.step()
if (i + 1) % 300 == 0:
print('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'
.format(epoch + 1, num_epochs, i + 1, total_step, loss.item()))
# Test the model
model.eval()
with torch.no_grad():
correct = 0
total = 0
for i, (images, labels) in enumerate(test_dataloader.dataloader):
images = images.reshape(-1, sequence_length, input_size).to(device)
labels = labels.to(device)
outputs, _ = model(images)
# calculate LEs
# h = torch.zeros(model.num_layers, images.size(0), model.hidden_size).to(model.device)
# c = torch.zeros(model.num_layers, images.size(0), model.hidden_size).to(model.device)
# params = (images, (h, c))
# if i == 0:
# LEs, rvals = calc_LEs_an(*params, model=model)
loss = criterion(outputs, labels)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
if epoch == (num_epochs - 1):
print('Epoch [{}/{}] Loss: {}, Test Accuracy: {} %'.format(epoch + 1, num_epochs, loss, 100 * correct / total))
trial[epoch] = {'model': model, 'accuracy': 100 * correct / total,
'loss': loss}
trials[num_trial] = trial
saved_path = f'../../../dataset/trials/{model_type}/models/'
pickle.dump(trials, open(f'{saved_path}/lstm_{hidden_size}_trials_0.pickle', 'wb'))
def build_model(self):
self.c2d = C2D().cuda()
self.gru = GRU(self.c2d).cuda()
class Trainer(object):
def __init__(self, config, h_loader, r_loader, test_loader):
self.config = config
self.h_loader = h_loader
self.r_loader = r_loader
self.test_loader = test_loader
self.lr = config.lr
self.beta1 = config.beta1
self.beta2 = config.beta2
self.weight_decay = config.weight_decay
self.n_epochs = config.n_epochs
self.n_steps = config.n_steps
self.log_interval = int(config.log_interval) # in case
self.checkpoint_step = int(config.checkpoint_step)
self.use_cuda = config.cuda
self.outf = config.outf
self.build_model()
self.vis = vis_tool.Visualizer()
def build_model(self):
self.c2d = C2D().cuda()
self.gru = GRU(self.c2d).cuda()
def train(self):
cfig = get_config()
opt = optim.Adam(filter(lambda p: p.requires_grad,
self.gru.parameters()),
lr=self.lr,
betas=(self.beta1, self.beta2),
weight_decay=self.weight_decay)
start_time = time.time()
criterion = nn.BCELoss()
max_acc = 0.
for epoch in range(self.n_epochs):
self.gru.train()
epoch_loss = []
for step, (h, r) in enumerate(zip(self.h_loader, self.r_loader)):
h_video = h[0]
r_video = r[0]
h_video = Variable(h_video).cuda()
r_video = Variable(r_video).cuda()
self.gru.zero_grad()
predicted = self.gru(h_video)
target = torch.ones(len(predicted), dtype=torch.float32).cuda()
h_loss = criterion(predicted, target) # compute loss
h_loss.backward()
opt.step()
self.gru.zero_grad()
predicted = self.gru(r_video) # predicted snippet's score
target = torch.zeros(len(predicted),
dtype=torch.float32).cuda()
r_loss = criterion(predicted, target) # compute loss
r_loss.backward()
opt.step()
step_end_time = time.time()
total_loss = r_loss + h_loss
epoch_loss.append((total_loss.data).cpu().numpy())
print(
'[%d/%d][%d/%d] - time: %.2f, h_loss: %.3f, r_loss: %.3f, total_loss: %.3f'
% (epoch + 1, self.n_epochs, step + 1, self.n_steps,
step_end_time - start_time, h_loss, r_loss, total_loss))
self.vis.plot(
'H_LOSS with lr:%.4f, b1:%.1f, b2:%.3f, wd:%.5f' %
(cfig.lr, cfig.beta1, cfig.beta2, cfig.weight_decay),
(h_loss.data).cpu().numpy())
self.vis.plot(
'R_LOSS with lr:%.4f, b1:%.1f, b2:%.3f, wd:%.5f' %
(cfig.lr, cfig.beta1, cfig.beta2, cfig.weight_decay),
(r_loss.data).cpu().numpy())
self.vis.plot("Avg loss plot", np.mean(epoch_loss))
# Test accuracy
# self.gru.eval()
# test_avg_acc = 0.
# test_cnt = 0
# for idx, (video, label, filename) in enumerate(self.test_loader):
# video = Variable(video).cuda()
# predicted = self.gru(video) # [ frame 수, 1]
#
# predicted = predicted.view(1, -1)
# predicted = predicted.cpu().detach().numpy()
#
# predicted = predicted[0]
# label = label.cpu().numpy()
#
# # print(type(predicted), type(label))
#
# gt_label_predicted_score = predicted * label
# gt_label_predicted_score = list(gt_label_predicted_score)
#
# # gt_label_predicted_score = gt_label_predicted_score.cpu().numpy()
# # print("Highlight frame predicted score:", gt_label_predicted_score)
#
# # print(gt_label_predicted_score)
# # print(gt_label_predicted_score.shape)
#
# # print(gt_label_predicted_score)
#
# for sc in gt_label_predicted_score[0]:
# if sc != 0.:
# print("%.3f" % sc, end=' ')
#
# for i in range(len(predicted)):
# if predicted[i] >= 0.45:
# predicted[i] = 1.
# else:
# predicted[i] = 0.
#
# # print("After threshold predicted:", predicted)
# # print("Actual label:", label)
#
# acc = (predicted == label).sum().item() / float(len(predicted))
# print("filename: %s accuracy: %.4f" % (filename, acc))
# test_avg_acc += acc
# test_cnt += 1
#
# print()
#
# test_avg_acc = test_avg_acc / test_cnt
# print("Epoch %d Test accuracy: %.5f" % (epoch+1, test_avg_acc))
# self.vis.plot("Test Accuracy plot", test_avg_acc)
# print("Epoch %d predicted output list" % (epoch+1), output_list)
# save max test accuracy checkpoint
# if test_avg_acc >= max_acc:
# max_acc = test_avg_acc
# torch.save(self.gru.state_dict(), 'max_test_acc_chkpoint' + str(epoch + 1) + '.pth')
# print("checkpoint saved")
if epoch % self.checkpoint_step == 0:
accuracy, savelist = self.test(self.test_loader)
if accuracy > max_acc:
max_acc = accuracy
torch.save(
self.gru.state_dict(),
'./samples/lr_%.4f_chkpoint' % cfig.lr +
str(epoch + 1) + '.pth')
for f in savelist:
np.save("./samples/" + f[0][0] + ".npy", f[1])
print(np.load("./samples/testRV04(198,360).mp4.npy"))
print("checkpoint saved")
def test(self, t_loader):
# self.gru.eval()
# accuracy = 0.
#
# savelist = []
#
# total_len = len(t_loader)
#
# for step, (tv, label, filename) in enumerate(t_loader):
# filename = filename[0].split(".")[0]
#
# label = label.squeeze()
#
# start = 0
# end = 24
#
# correct = 0
# count = 0
#
# npy = np.zeros(tv.shape[1])
#
# while end < tv.shape[1]:
#
# t_video = Variable(tv[:, start:end, :, :, :]).cuda()
# predicted = self.gru(t_video)
#
# gt_label = label[start:end]
#
# if len(gt_label[gt_label == 1.]) > 12:
# gt_label = torch.ones(predicted.shape, dtype=torch.float32).cuda()
#
# else:
# gt_label = torch.zeros(predicted.shape, dtype=torch.float32).cuda()
#
# if predicted < 0.5:
# npy[start:end] = 1.
#
# predicted[predicted < 0.5] = 1.
# predicted[predicted >= 0.5] = 0.
#
# correct += (predicted == gt_label).item()
#
# start += 24
# end += 24
# count += 1
#
# accuracy += (correct / count) / total_len
#
# savelist.append([filename, npy])
# Test accuracy
self.gru.eval()
test_avg_acc = 0.
test_cnt = 0
savelist = []
for idx, (video, label, filename) in enumerate(self.test_loader):
video = Variable(video).cuda()
predicted = self.gru(video) # [ frame 수, 1]
predicted = predicted.view(1, -1)
predicted = predicted.cpu().detach().numpy()
predicted = predicted[0]
label = label.cpu().numpy()
# print(type(predicted), type(label))
gt_label_predicted_score = predicted * label
gt_label_predicted_score = list(gt_label_predicted_score)
# gt_label_predicted_score = gt_label_predicted_score.cpu().numpy()
# print("Highlight frame predicted score:", gt_label_predicted_score)
# print(gt_label_predicted_score)
# print(gt_label_predicted_score.shape)
# print(gt_label_predicted_score)
for sc in gt_label_predicted_score[0]:
if sc != 0.:
print("%.3f" % sc, end=' ')
for i in range(len(predicted)):
if predicted[i] >= 0.45:
predicted[i] = 1.
else:
predicted[i] = 0.
# print("After threshold predicted:", predicted)
# print("Actual label:", label)
acc = (predicted == label).sum().item() / float(len(predicted))
print("filename: %s accuracy: %.4f" % (filename, acc))
test_avg_acc += acc
test_cnt += 1
savelist.append([filename, predicted])
print()
test_avg_acc = test_avg_acc / test_cnt
print("Accuracy:", round(test_avg_acc, 4))
self.vis.plot("Accuracy with lr:%.3f" % self.lr, test_avg_acc)
return test_avg_acc, savelist
batch_size = 256
hidden_size = 128
num_layers = 1
dropout = 0
testnum = 500
# interval is sample interval between last input and first output.
interval = 0
epoch = 100
device = 'cuda'
# Generate sin dataset for training and testing.
dataset = np.sin([i / 50 * 2 * np.pi for i in range(2000)])
x_train, y_train, x_test, y_test, normalizer = generate_data(
dataset, 'minmax', input_length, output_length, testnum, interval)
# Build, train and predict.
model = GRU(1, hidden_size, num_layers, 1, dropout)
optimizer = opt.Adam(model.parameters())
loss = nn.MSELoss()
batch_train_loss, batch_val_loss = train(model, x_train, y_train, epoch,
batch_size, optimizer, loss, device)
y_predict, y_real, _ = predict(model, x_test, y_test, loss, device, normalizer,
batch_size)
# Draw result
plt.plot(y_predict, label='prediction')
plt.plot(y_real, label='real')
plt.legend()
plt.show()
def main():
# Training settings
parser = argparse.ArgumentParser(description='From PyTorch MNIST Example')
parser.add_argument('--batch-size',
type=int,
default=32,
metavar='N',
help='input batch size for training')
parser.add_argument('--epochs',
type=int,
default=10000,
metavar='E',
help='number of epochs to train')
parser.add_argument('--warmup-epochs',
type=int,
default=5000,
metavar='WE',
help='number of epochs to warmup')
parser.add_argument('--num-steps',
type=int,
default=100,
metavar='N',
help='number of batches in one epochs')
parser.add_argument('--num-points',
type=int,
default=10,
metavar='NS',
help='number of query points')
parser.add_argument('--num-hidden',
type=int,
default=128,
metavar='NE',
help='number of hidden units')
parser.add_argument('--lr',
type=float,
default=0.0003,
metavar='LR',
help='learning rate')
parser.add_argument('--alpha',
type=float,
default=0,
metavar='A',
help='kl factor')
parser.add_argument('--sampling',
action='store_true',
default=False,
help='uses sampling')
parser.add_argument('--direction',
action='store_true',
default=False,
help='uses directed data-sets')
parser.add_argument('--ranking',
action='store_true',
default=False,
help='sort data-set according to importance')
parser.add_argument('--num-runs',
type=int,
default=1,
metavar='NR',
help='number of runs')
parser.add_argument('--save-path',
default='trained_models/random_',
help='directory to save results')
parser.add_argument('--load-path',
default='trained_models/default_model_0.pth',
help='path to load model')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
args = parser.parse_args()
use_cuda = not args.no_cuda and torch.cuda.is_available()
device = torch.device("cuda" if use_cuda else "cpu")
for i in range(args.num_runs):
writer = SummaryWriter()
performance = torch.zeros(args.epochs)
accuracy_test = 0
data_loader = PairedComparison(4,
direction=args.direction,
ranking=args.ranking)
model = GRU(data_loader.num_inputs, data_loader.num_targets,
args.num_hidden).to(device)
if args.alpha > 0:
print('Loading pretrained network...')
params, _ = torch.load(args.load_path)
model.load_state_dict(params)
model.reset_log_sigma()
max_alpha = args.alpha
optimizer = optim.Adam(model.parameters(), lr=args.lr, amsgrad=True)
with trange(args.epochs) as t:
for j in t:
loss_train = 0
for k in range(args.num_steps):
inputs, targets, _, _ = data_loader.get_batch(
args.batch_size, args.num_points, device=device)
predictive_distribution, _, _ = model(
inputs, targets, args.sampling)
loss = -predictive_distribution.log_prob(targets).mean()
writer.add_scalar('NLL', loss.item(),
j * args.num_steps + k)
if args.alpha > 0:
alpha = min(j / args.warmup_epochs, 1.0) * max_alpha
kld = model.regularization(alpha)
loss = loss + kld
writer.add_scalar('KLD', kld.item(),
j * args.num_steps + k)
loss_train += loss
optimizer.zero_grad()
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 40.0)
optimizer.step()
t.set_description('Loss (train): {:5.4f}'.format(
loss_train.item() / args.num_steps))
performance[j] = loss_train.item() / args.num_steps
torch.save([model.state_dict(), performance],
args.save_path + str(i) + '.pth')