def forward_pass(model, data_gen, batch_size, quantiles, gpu = True):
model.reset(batch_size, gpu = gpu)
#Get input and target data, one-hot encode discrete variables, continuous variables have already been normalized
in_seq_continuous, in_seq_discrete, future_in_seq_discrete, target_seq = next(data_gen)
in_seq_discrete = one_hot(in_seq_discrete, [24, 31, 12])
future_in_seq_discrete = one_hot(future_in_seq_discrete, [24, 31, 12])
#forward pass
net_out, vs_weights = model(in_seq_continuous, in_seq_discrete, None, future_in_seq_discrete)
loss = torch.mean(QuantileLoss(net_out, target_seq ,quantiles))
return loss, net_out, vs_weights, (in_seq_continuous, in_seq_discrete, future_in_seq_discrete, target_seq)
def predict(data, labels, n, c):
x = tf.placeholder(tf.float32, [None, n])
W = tf.Variable(tf.zeros([n, c]))
b = tf.Variable(tf.zeros([c]))
y = tf.nn.softmax(tf.matmul(x, W) + b)
y_ = tf.placeholder(tf.float32, [None, c])
saver = tf.train.Saver()
sess = tf.Session()
saver.restore(sess, 'model.ckpt')
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
a, b = data, d.one_hot(labels)
actual = (sess.run(tf.argmax(y_, 1), feed_dict={y_: b}))
predicted = (sess.run(tf.argmax(y, 1), feed_dict={x: a, y_: b}))
# print(predicted)
# print(actual)
display(actual, predicted)
print('Accuracy = ', sess.run(accuracy, feed_dict={x: a, y_: b}))
def predict(data, labels, n, c):
x = tf.placeholder(tf.float32, [None, n])
W = tf.Variable(tf.zeros([n, c]))
b = tf.Variable(tf.zeros([c]))
y = tf.nn.softmax(tf.matmul(x, W) + b)
y_ = tf.placeholder(tf.float32, [None, c])
saver = tf.train.Saver()
sess = tf.Session()
saver.restore(sess, "model.ckpt")
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
a, b = data, d.one_hot(labels)
actual = sess.run(tf.argmax(y_, 1), feed_dict={y_: b})
predicted = sess.run(tf.argmax(y, 1), feed_dict={x: a, y_: b})
# print(predicted)
# print(actual)
display(actual, predicted)
print("Accuracy = ", sess.run(accuracy, feed_dict={x: a, y_: b}))
print(f'Sigmoid function: {network.y}')
operation = lo.LogicalOperations()
print(f'NOT: {operation.not_func(1)}')
print(f'AND: {operation.and_func(1, 1)}')
print(f'OR: {operation.or_func(0, 0)}')
print(f'XOR: {operation.xor_func(0, 1)}')
'''
''' layer.py activation.py '''
# INPUT
X, y = data.data_spiral()
y_onehot = data.one_hot(y)
'''
print(f"X: {X}")
print(f"y: {y}")
print(f"y_onehot: {y_onehot}")
'''
# Model Architecture
layer1 = layer.Layer_Dense(2, 4)
activation1 = activation.Activation_LeakyReLU()
layer2 = layer.Layer_Dense(4, 3)
activation2 = activation.Activation_Softmax()
loss1 = loss.Loss_CategoricalCrossEntropy()
metric1 = metric.Metric_Accuracy()
def train(sess, model, optimizer, log_dir, batch_size, num_sweeps_per_summary,
num_sweeps_per_save, train_input_seqs, train_reset_seqs,
train_label_seqs, test_input_seqs, test_reset_seqs, test_label_seqs):
""" Train a model and export summaries.
`log_dir` will be *replaced* if it already exists, so it certainly
shouldn't be anything generic like `/home/user`.
Args:
sess: A TensorFlow `Session`.
model: An `LSTMModel`.
optimizer: An `Optimizer`.
log_dir: A string. The full path to the log directory.
batch_size: An integer. The number of sequences in a batch.
num_sweeps_per_summary: An integer. The number of sweeps between
summaries.
num_sweeps_per_save: An integer. The number of sweeps between saves.
train_input_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, input_size]`.
train_reset_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, 1]`.
train_label_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, 1]`.
test_input_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, input_size]`.
test_reset_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, 1]`.
test_label_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, 1]`.
"""
ema = tf.train.ExponentialMovingAverage(decay=0.5)
update_train_loss_ema = ema.apply([model.loss])
train_loss_ema = ema.average(model.loss)
tf.summary.scalar('train_loss_ema', train_loss_ema)
train_accuracy = tf.placeholder(tf.float32, name='train_accuracy')
train_edit_dist = tf.placeholder(tf.float32, name='train_edit_dist')
test_accuracy = tf.placeholder(tf.float32, name='test_accuracy')
test_edit_dist = tf.placeholder(tf.float32, name='test_edit_dist')
#values = [train_accuracy, train_edit_dist, test_accuracy, test_edit_dist]
#tags = [value.op.name for value in values]
tf.summary.scalar('learning_rate', optimizer.learning_rate)
for value in [
train_accuracy, train_edit_dist, test_accuracy, test_edit_dist
]:
tf.summary.scalar(value.op.name, value)
#tf.summary.scalar(tags, tf.stack(values))
summary_op = tf.summary.merge_all()
if os.path.exists(log_dir):
shutil.rmtree(log_dir)
summary_writer = tf.summary.FileWriter(logdir=log_dir, graph=sess.graph)
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
num_sweeps_visited = 0
start_time = time.time()
train_gen = data.sweep_generator(
[train_input_seqs, train_reset_seqs, train_label_seqs],
batch_size=batch_size,
shuffle=True,
num_sweeps=None)
while num_sweeps_visited <= optimizer.num_train_sweeps:
if num_sweeps_visited % num_sweeps_per_summary == 0:
train_prediction_seqs = models.predict(sess, model,
train_input_seqs,
train_reset_seqs)
train_accuracy_, train_edit_dist_ = metrics.compute_metrics(
train_prediction_seqs, train_label_seqs)
test_prediction_seqs = models.predict(sess, model, test_input_seqs,
test_reset_seqs)
test_accuracy_, test_edit_dist_ = metrics.compute_metrics(
test_prediction_seqs, test_label_seqs)
summary = sess.run(summary_op,
feed_dict={
train_accuracy: train_accuracy_,
train_edit_dist: train_edit_dist_,
test_accuracy: test_accuracy_,
test_edit_dist: test_edit_dist_
})
summary_writer.add_summary(summary, global_step=num_sweeps_visited)
status_path = os.path.join(log_dir, 'status.txt')
with open(status_path, 'w') as f:
line = '%05.1f ' % ((time.time() - start_time) / 60)
line += '%04d ' % num_sweeps_visited
line += '%.6f %08.3f ' % (train_accuracy_,
train_edit_dist_)
line += '%.6f %08.3f ' % (test_accuracy_, test_edit_dist_)
print(line, file=f)
label_path = os.path.join(log_dir, 'test_label_seqs.pkl')
with open(label_path, 'wb') as f:
cPickle.dump(test_label_seqs, f)
pred_path = os.path.join(log_dir, 'test_prediction_seqs.pkl')
with open(pred_path, 'wb') as f:
cPickle.dump(test_prediction_seqs, f)
vis_filename = 'test_visualizations_%06d.png' % num_sweeps_visited
vis_path = os.path.join(log_dir, vis_filename)
fig, axes = data.visualize_predictions(test_prediction_seqs,
test_label_seqs,
model.target_size)
axes[0].set_title(line)
plt.tight_layout()
plt.savefig(vis_path)
plt.close(fig)
if num_sweeps_visited % num_sweeps_per_save == 0:
saver.save(sess, os.path.join(log_dir, 'model.ckpt'))
train_inputs, train_resets, train_labels = train_gen.__next__()
# We squeeze here because otherwise the targets would have shape
# [batch_size, duration, 1, num_classes].
train_targets = data.one_hot(train_labels, model.target_size)
train_targets = train_targets.squeeze(axis=2)
_, _, num_sweeps_visited = sess.run(
[
optimizer.optimize_op, update_train_loss_ema,
optimizer.num_sweeps_visited
],
feed_dict={
model.inputs: train_inputs,
model.resets: train_resets,
model.targets: train_targets,
model.training: True
})
def train(sess, model, optimizer, log_dir, batch_size, num_sweeps_per_summary,
num_sweeps_per_save, train_input_seqs, train_reset_seqs,
train_label_seqs, test_input_seqs, test_reset_seqs, test_label_seqs,
args):
""" Train a model and export summaries.
`log_dir` will be *replaced* if it already exists, so it certainly
shouldn't be anything generic like `/home/user`.
Args:
sess: A TensorFlow `Session`.
model: An `LSTMModel`.
optimizer: An `Optimizer`.
log_dir: A string. The full path to the log directory.
batch_size: An integer. The number of sequences in a batch.
num_sweeps_per_summary: An integer. The number of sweeps between
summaries.
num_sweeps_per_save: An integer. The number of sweeps between saves.
train_input_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, input_size]`.
train_reset_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, 1]`.
train_label_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, 1]`.
test_input_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, input_size]`.
test_reset_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, 1]`.
test_label_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, 1]`.
args: An object containing processed arguments as attributes.
"""
#print(test_label_seqs)
ema = tf.train.ExponentialMovingAverage(decay=0.5) #0.5
update_train_loss_ema = ema.apply([model.loss])
train_loss_ema = ema.average(model.loss)
tf.summary.scalar('train_loss_ema', train_loss_ema)
train_accuracy = tf.placeholder(tf.float32, name='train_accuracy')
train_edit_dist = tf.placeholder(tf.float32, name='train_edit_dist')
test_accuracy = tf.placeholder(tf.float32, name='test_accuracy')
test_edit_dist = tf.placeholder(tf.float32, name='test_edit_dist')
#values = [train_accuracy, train_edit_dist, test_accuracy, test_edit_dist]
#tags = [value.op.name for value in values]
tf.summary.scalar('learning_rate', optimizer.learning_rate)
for value in [
train_accuracy, train_edit_dist, test_accuracy, test_edit_dist
]:
tf.summary.scalar(value.op.name, value)
#tf.summary.scalar(tags, tf.stack(values))
summary_op = tf.summary.merge_all()
if os.path.exists(log_dir):
shutil.rmtree(log_dir)
summary_writer = tf.summary.FileWriter(logdir=log_dir, graph=sess.graph)
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
num_sweeps_visited = 0
start_time = time.time()
train_gen = data.sweep_generator(
[train_input_seqs, train_reset_seqs, train_label_seqs],
batch_size=batch_size,
shuffle=True,
num_sweeps=None)
while num_sweeps_visited <= optimizer.num_train_sweeps:
if num_sweeps_visited % num_sweeps_per_summary == 0:
test_prediction_seqs = []
train_prediction_seqs, logits = models.predict(
sess, model, train_input_seqs, train_reset_seqs)
train_accuracy_, train_edit_dist_, train_confusion_matrix = metrics.compute_metrics(
train_prediction_seqs, train_label_seqs, log_dir, 'train')
print('test_input_seqs:', len(test_input_seqs))
err = 0.0
# init empty predictions
no_of_samples = sum([len(seq) for seq in test_input_seqs])
entropy_matrix = np.zeros(
(args.sample_times, no_of_samples, 10)) #batch_size
entropy_matrix_1 = np.zeros(
(args.sample_times, no_of_samples, 1)) # batch_size
for sample_id in range(50):
test_prediction_seqs, softmax_seqs = models.predict(
sess, model, test_input_seqs, test_reset_seqs)
#print('softmax_dur:', softmax_seqs)
entropy_matrix[sample_id] = np.vstack(softmax_seqs)
entropy_matrix_1[sample_id] = np.vstack(test_prediction_seqs)
''' #Variation Ratio
value,count = mode(entropy_matrix_1,axis=0)
print('count.shape:', value[0,200,0], count[0,200,0])
varition_ratio=1-count/50.0
print(varition_ratio.shape)
entropy=np.squeeze(varition_ratio)
entropy[entropy > 0.5] = 222
entropy[entropy < 0.5] = 111'''
#MC Dropout - Entropy
#print('entropy_matrix.shape:',entropy_matrix.shape,entropy_matrix[0,0,:])
entropy_matrix_mean = np.mean(entropy_matrix, axis=0)
#print('entropy_matrix_mean.shape:', entropy_matrix_mean.shape,entropy_matrix_mean[0,:])
entropy_log = np.log2(entropy_matrix_mean,
where=(entropy_matrix_mean != 0.0))
#print('entropy_log:',entropy_log[0,:])
entropy_log_mul = entropy_matrix_mean * entropy_log
#print('entropy_log_mul.shape:', entropy_log_mul.shape, entropy_log_mul[0, :])
entropy = np.sum(entropy_log_mul, axis=1) * -1
#print('entropy.shape:', entropy.shape,entropy[0])
normalized_entropy = (entropy - np.mean(entropy, axis=0)) / np.std(
entropy, axis=0)
test_accuracy_, test_edit_dist_, test_confusion_matrix = metrics.compute_metrics(
test_prediction_seqs, test_label_seqs, log_dir, 'test')
summary = sess.run(summary_op,
feed_dict={
train_accuracy: train_accuracy_,
train_edit_dist: train_edit_dist_,
test_accuracy: test_accuracy_,
test_edit_dist: test_edit_dist_
})
print('kris_po:: num_sweeps_visited:', num_sweeps_visited)
summary_writer.add_summary(summary, global_step=num_sweeps_visited)
summary_writer.add_summary(train_confusion_matrix,
global_step=num_sweeps_visited)
summary_writer.add_summary(test_confusion_matrix,
global_step=num_sweeps_visited)
status_path = os.path.join(log_dir, 'status.txt')
with open(status_path, 'w') as f:
line = '%05.1f ' % ((time.time() - start_time) / 60)
line += '%04d ' % num_sweeps_visited
line += '%.6f %08.3f ' % (train_accuracy_,
train_edit_dist_)
line += '%.6f %08.3f ' % (test_accuracy_, test_edit_dist_)
print(line, file=f)
label_path = os.path.join(log_dir, 'test_label_seqs.pkl')
with open(label_path, 'wb') as f:
cPickle.dump(test_label_seqs, f)
pred_path = os.path.join(log_dir, 'test_prediction_seqs.pkl')
with open(pred_path, 'wb') as f:
cPickle.dump(test_prediction_seqs, f)
if num_sweeps_visited == 1197:
vis_filename = 'test_visualizations_%06d.png' % num_sweeps_visited
vis_path = os.path.join(log_dir, vis_filename)
fig, axes = data.visualize_predictions(test_prediction_seqs,
test_label_seqs,
model.target_size,
normalized_entropy)
axes[0].set_title(line)
plt.tight_layout()
plt.savefig(vis_path)
plt.close(fig)
if num_sweeps_visited % num_sweeps_per_save == 0:
saver.save(sess, os.path.join(log_dir, 'model.ckpt'))
train_inputs, train_resets, train_labels = train_gen.next()
# We squeeze here because otherwise the targets would have shape
# [batch_size, duration, 1, num_classes].
train_targets = data.one_hot(train_labels, model.target_size)
train_targets = train_targets.squeeze(axis=2)
_, _, num_sweeps_visited = sess.run(
[
optimizer.optimize_op, update_train_loss_ema,
optimizer.num_sweeps_visited
],
feed_dict={
model.inputs: train_inputs,
model.resets: train_resets,
model.targets: train_targets,
model.training: True
})
def fakeDataset(character):
x = torch.zeros(batch_size, 1, len(dataset.corpus))
x[0][0] = one_hot(dataset.getLabelFromChar(character), len(dataset.corpus))
y = None
return [x,y]
def on_hot_hamming(a, b):
return spatial.distance.hamming(
one_hot(a, n_cols=VOCAB).flatten(), one_hot(b, n_cols=VOCAB).flatten()
)
def train(sess, model, optimizer, log_dir, batch_size, num_sweeps_per_summary,
num_sweeps_per_save, train_input_seqs, train_reset_seqs,
train_label_seqs, test_input_seqs, test_reset_seqs, test_label_seqs):
""" Train a model and export summaries.
`log_dir` will be *replaced* if it already exists, so it certainly
shouldn't be anything generic like `/home/user`.
Args:
sess: A TensorFlow `Session`.
model: An `LSTMModel`.
optimizer: An `Optimizer`.
log_dir: A string. The full path to the log directory.
batch_size: An integer. The number of sequences in a batch.
num_sweeps_per_summary: An integer. The number of sweeps between
summaries.
num_sweeps_per_save: An integer. The number of sweeps between saves.
train_input_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, input_size]`.
train_reset_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, 1]`.
train_label_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, 1]`.
test_input_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, input_size]`.
test_reset_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, 1]`.
test_label_seqs: A list of 2-D NumPy arrays, each with shape
`[duration, 1]`.
"""
ema = tf.train.ExponentialMovingAverage(decay=0.5)
update_train_loss_ema = ema.apply([model.loss])
train_loss_ema = ema.average(model.loss)
tf.scalar_summary('train_loss_ema', train_loss_ema)
train_accuracy = tf.placeholder(tf.float32, name='train_accuracy')
train_edit_dist = tf.placeholder(tf.float32, name='train_edit_dist')
test_accuracy = tf.placeholder(tf.float32, name='test_accuracy')
test_edit_dist = tf.placeholder(tf.float32, name='test_edit_dist')
values = [train_accuracy, train_edit_dist, test_accuracy, test_edit_dist]
tags = [value.op.name for value in values]
tf.scalar_summary('learning_rate', optimizer.learning_rate)
tf.scalar_summary(tags, tf.pack(values))
summary_op = tf.merge_all_summaries()
if os.path.exists(log_dir):
shutil.rmtree(log_dir)
summary_writer = tf.train.SummaryWriter(logdir=log_dir, graph=sess.graph)
saver = tf.train.Saver()
sess.run(tf.initialize_all_variables())
num_sweeps_visited = 0
start_time = time.time()
train_gen = data.sweep_generator(
[train_input_seqs, train_reset_seqs, train_label_seqs],
batch_size=batch_size, shuffle=True, num_sweeps=None)
while num_sweeps_visited <= optimizer.num_train_sweeps:
if num_sweeps_visited % num_sweeps_per_summary == 0:
train_prediction_seqs = models.predict(
sess, model, train_input_seqs, train_reset_seqs)
train_accuracy_, train_edit_dist_ = metrics.compute_metrics(
train_prediction_seqs, train_label_seqs)
test_prediction_seqs = models.predict(
sess, model, test_input_seqs, test_reset_seqs)
test_accuracy_, test_edit_dist_ = metrics.compute_metrics(
test_prediction_seqs, test_label_seqs)
summary = sess.run(summary_op,
feed_dict={train_accuracy: train_accuracy_,
train_edit_dist: train_edit_dist_,
test_accuracy: test_accuracy_,
test_edit_dist: test_edit_dist_})
summary_writer.add_summary(summary, global_step=num_sweeps_visited)
status_path = os.path.join(log_dir, 'status.txt')
with open(status_path, 'w') as f:
line = '%05.1f ' % ((time.time() - start_time)/60)
line += '%04d ' % num_sweeps_visited
line += '%.6f %08.3f ' % (train_accuracy_,
train_edit_dist_)
line += '%.6f %08.3f ' % (test_accuracy_,
test_edit_dist_)
print(line, file=f)
label_path = os.path.join(log_dir, 'test_label_seqs.pkl')
with open(label_path, 'w') as f:
cPickle.dump(test_label_seqs, f)
pred_path = os.path.join(log_dir, 'test_prediction_seqs.pkl')
with open(pred_path, 'w') as f:
cPickle.dump(test_prediction_seqs, f)
vis_filename = 'test_visualizations_%06d.png' % num_sweeps_visited
vis_path = os.path.join(log_dir, vis_filename)
fig, axes = data.visualize_predictions(test_prediction_seqs,
test_label_seqs,
model.target_size)
axes[0].set_title(line)
plt.tight_layout()
plt.savefig(vis_path)
plt.close(fig)
if num_sweeps_visited % num_sweeps_per_save == 0:
saver.save(sess, os.path.join(log_dir, 'model.ckpt'))
train_inputs, train_resets, train_labels = train_gen.next()
# We squeeze here because otherwise the targets would have shape
# [batch_size, duration, 1, num_classes].
train_targets = data.one_hot(train_labels, model.target_size)
train_targets = train_targets.squeeze(axis=2)
_, _, num_sweeps_visited = sess.run(
[optimizer.optimize_op,
update_train_loss_ema,
optimizer.num_sweeps_visited],
feed_dict={model.inputs: train_inputs,
model.resets: train_resets,
model.targets: train_targets,
model.training: True})
