def test(self, save_dir, test_data, model_file, batch_size, avg=False):
#with tf.get_default_graph().as_default():
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth = True
model = self.module.Model(batch_size, False) #get_model_with_placeholders(self.module, reuse=False)
if avg:
variable_averages = tf.train.ExponentialMovingAverage(0.999)
variables_to_restore = variable_averages.variables_to_restore()
saver = tf.compat.v1.train.Saver(variables_to_restore)
else:
saver = tf.compat.v1.train.Saver()
with tf.compat.v1.Session() as sess:
sess.run(tf.compat.v1.global_variables_initializer())
saver.restore(sess, model_file)
tflearn.is_training(False)
with tf.compat.v1.variable_scope("inference") as scope:
scope.reuse_variables()
scope.reuse_variables()
print(sess.run(tflearn.get_training_mode()))
dices = {}
if not os.path.exists(save_dir):
os.makedirs(save_dir)
for tup in test_data:
inferer = model.build_full_inferer()
dice = inferer(sess, tup, save_dir, model)
dices[tup[0]] = dice
print("Median: {}, Max: {}, Min: {}"
.format(np.median(list(dices.values()), axis=0),
np.max(list(dices.values()), axis=0),
np.min(list(dices.values()), axis=0)))
print(dices)
return(dices)
sess.close()
def get_input_layer(flags):
img_prep = tflearn.ImagePreprocessing()
img_prep.add_featurewise_zero_center()
img_prep.add_featurewise_stdnorm()
curr = input_data(shape=[None, flags.inres, flags.inres, flags.inchan],
name='input',
data_preprocessing=img_prep)
if flags.rts_aug:
w, h = curr.get_shape().as_list()[1:3]
a = -flags.rts_aug_ang + 2 * flags.rts_aug_ang * tf.random_uniform(
[flags.bs])
a *= np.pi / 180
# centralize rot/scale
y = ((w - 1) - (tf.cos(a) * (w - 1) - tf.sin(a) * (h - 1))) / 2.0
x = ((h - 1) - (tf.sin(a) * (w - 1) + tf.cos(a) * (h - 1))) / 2.0
transforms = tf.transpose(
tf.stack([
tf.cos(a),
tf.sin(a), x, -tf.sin(a),
tf.cos(a), y,
tf.zeros(flags.bs),
tf.zeros(flags.bs)
]))
return tf.cond(tflearn.get_training_mode(),
lambda: tf.contrib.image.transform(curr, transforms),
lambda: curr)
else:
return curr
def augmenetation_pts(incoming):
def aug(incoming):
aug_pt = tf.cast(transform.warp_points(
augFlow, incoming), tf.float32)
pt_mask = tf.cast(tf.reduce_all(
incoming >= 0, axis=-1, keep_dims=True), tf.float32)
return aug_pt * pt_mask - (1 - pt_mask)
return tf.cond(tflearn.get_training_mode(), lambda: aug(incoming), lambda: incoming)
def join(cols, drop_prob=.15):
if len(cols) == 1:
return cols[0]
with tf.variable_op_scope(cols, None, "Join"):
joined = tf.reduce_mean(cols, 0)
out = tf.cond(tflearn.get_training_mode(),
lambda: local_drop(cols, drop_prob), lambda: joined)
return joined
def dropout(incoming, keep_prob, noise_shape=None, name="Dropout"):
""" Dropout.
Outputs the input element scaled up by `1 / keep_prob`. The scaling is so
that the expected sum is unchanged.
By default, each element is kept or dropped independently. If noise_shape
is specified, it must be broadcastable to the shape of x, and only dimensions
with noise_shape[i] == shape(x)[i] will make independent decisions. For
example, if shape(x) = [k, l, m, n] and noise_shape = [k, 1, 1, n], each
batch and channel component will be kept independently and each row and column
will be kept or not kept together.
Arguments:
incoming : A `Tensor`. The incoming tensor.
keep_prob : A float representing the probability that each element
is kept.
noise_shape : A 1-D Tensor of type int32, representing the shape for
randomly generated keep/drop flags.
name : A name for this layer (optional).
References:
Dropout: A Simple Way to Prevent Neural Networks from Overfitting.
N. Srivastava, G. Hinton, A. Krizhevsky, I. Sutskever & R. Salakhutdinov,
(2014), Journal of Machine Learning Research, 5(Jun)(2), 1929-1958.
Links:
[https://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf]
(https://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf)
"""
with tf.name_scope(name) as scope:
inference = incoming
def apply_dropout():
if type(inference) in [list, np.array]:
for x in inference:
x = tf.nn.dropout(x, keep_prob, noise_shape)
return inference
else:
return tf.nn.dropout(inference, keep_prob, noise_shape)
is_training = tflearn.get_training_mode()
inference = tf.cond(is_training, apply_dropout, lambda: inference)
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
def dropout(incoming, keep_prob, noise_shape=None, name="Dropout"):
""" Dropout.
Outputs the input element scaled up by `1 / keep_prob`. The scaling is so
that the expected sum is unchanged.
By default, each element is kept or dropped independently. If noise_shape
is specified, it must be broadcastable to the shape of x, and only dimensions
with noise_shape[i] == shape(x)[i] will make independent decisions. For
example, if shape(x) = [k, l, m, n] and noise_shape = [k, 1, 1, n], each
batch and channel component will be kept independently and each row and column
will be kept or not kept together.
Arguments:
incoming : A `Tensor`. The incoming tensor.
keep_prob : A float representing the probability that each element
is kept.
noise_shape : A 1-D Tensor of type int32, representing the shape for
randomly generated keep/drop flags.
name : A name for this layer (optional).
References:
Dropout: A Simple Way to Prevent Neural Networks from Overfitting.
N. Srivastava, G. Hinton, A. Krizhevsky, I. Sutskever & R. Salakhutdinov,
(2014), Journal of Machine Learning Research, 5(Jun)(2), 1929-1958.
Links:
[https://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf]
(https://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf)
"""
with tf.name_scope(name) as scope:
inference = incoming
def apply_dropout():
if type(inference) in [list, np.array]:
for x in inference:
x = tf.nn.dropout(x, keep_prob, noise_shape)
return inference
else:
return tf.nn.dropout(inference, keep_prob, noise_shape)
is_training = tflearn.get_training_mode()
inference = tf.cond(is_training, apply_dropout, lambda: inference)
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
def build_model(self):
self._create_placeholders()
self._create_variables()
result = self.inputs
# forward layers
for layer_no in range(self.layers_qtty - 1):
layer_from = layer_no
layer_to = layer_no + 1
weights = getattr(self, self._get_w_name(layer_from, layer_to))
bias = self._get_bias_tensor(layer_to)
result = tf.sigmoid(tf.matmul(result, weights) + bias)
# bin encoding
layer_from = self.layers_qtty - 1
layer_to = self.layers_qtty
weights = getattr(self, self._get_w_name(layer_from, layer_to))
bias = self._get_bias_tensor(layer_to)
mat_mul_result = tf.matmul(result, weights) + bias
# add noise only in case of training
is_training = tflearn.get_training_mode()
mat_mul_result = tf.cond(is_training,
lambda: tf.add(mat_mul_result, self.noise),
lambda: mat_mul_result)
result = tf.sigmoid(mat_mul_result)
self.encoded_array = result
# backward layers
for layer_no in range(self.layers_qtty, self.layers_qtty * 2):
layer_from = layer_no
layer_to = layer_no + 1
weights = getattr(self, self._get_w_name(layer_from, layer_to))
bias = self._get_bias_tensor(layer_to)
result = tf.matmul(result, weights, transpose_b=True) + bias
if layer_to != (self.layers_qtty * 2):
result = tf.sigmoid(result)
self.reconstr = result
self.reconstr_prob = tf.sigmoid(result)
# define cost and optimizer
self.cost = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(self.reconstr,
self.inputs))
optimizer = tf.train.GradientDescentOptimizer(
self.params['learning_rate'])
self.minimization = optimizer.minimize(self.cost)
tf.scalar_summary("train_loss", self.cost)
self.summary = tf.merge_all_summaries()
def inference(inputs, n_class=1000, finetuning=False):
block_list = [10, 20, 9]
scale_list = [0.17, 0.10, 0.20]
print(inputs)
net = stem(inputs, scope='stem')
print(net)
# bloack A
for i in range(block_list[0]):
net = blockA(net, scale=scale_list[0], scope='A_' + str(i))
print(net)
# reduction A
net = reductionA(net)
# bloack B
for i in range(block_list[1]):
net = blockB(net, scale=scale_list[1], scope='B_' + str(i))
print(net)
# reduction B
net = reductionB(net)
print(net)
# bloack C
for i in range(block_list[2]):
net = blockB(net, scale=scale_list[2], scope='C_' + str(i))
print(net)
net = tflearn.global_avg_pool(net)
print(net)
keep_prob = tf.cond(tflearn.get_training_mode(), lambda: 1.0, lambda: 0.8)
net = tflearn.dropout(net, keep_prob=keep_prob)
with tf.variable_scope('FC', reuse=tf.AUTO_REUSE):
net = tflearn.fully_connected(net,
n_class,
weights_init='uniform_scaling',
regularizer='L2',
weight_decay=0.0001,
restore=(not finetuning))
print(net)
return net
def pt_regressor(layer_in, flags):
net, curr = pt_regressor_conv(layer_in, flags)
net['ptreg_in'] = layer_in
dims = curr.get_shape().as_list()
weights_init = 'zeros'
bias_init = tf.ones([1])
# 1x1 conv, no BN, no ReLU on final heatmap
net['ptreg_out'], curr = dup(
conv_2d(curr,
1,
1,
activation='linear',
weights_init=weights_init,
bias_init=bias_init,
padding=flags.pad,
name='ptreg_out'))
# take the centroid of the feature map
s = tf.shape(curr)
# compute xc, yc from -1 to 1
xc = tf.tile(tf.linspace(-1., 1., s[2])[np.newaxis, ...], (s[1], 1))
yc = tf.transpose(xc)
net['po_j'] = (
tf.reduce_sum(curr[..., 0] * xc[np.newaxis, ...], axis=(1, 2)) /
tf.reduce_sum(curr[..., 0], axis=(1, 2)))
net['po_i'] = (
tf.reduce_sum(curr[..., 0] * yc[np.newaxis, ...], axis=(1, 2)) /
tf.reduce_sum(curr[..., 0], axis=(1, 2)))
net['polar_origin'] = tf.stack([net['po_j'], net['po_i']], axis=1)
# origin augmentation
if flags.ptreg_aug > 0:
dim = layer_in.get_shape().as_list()[1]
shift = tf.cond(
tflearn.get_training_mode(),
lambda: 1. / dim * tf.random_uniform([flags.bs, 2],
minval=-flags.ptreg_aug,
maxval=flags.ptreg_aug),
lambda: tf.zeros([flags.bs, 2]))
net['polar_origin'] += shift
return net
def join(columns, coin):
"""Takes mean of the columns, applies drop path if
`tflearn.get_training_mode()` is True.
Args:
columns: columns of fractal block.
is_training: boolean in tensor form. Determines whether drop path
should be used.
coin: boolean in tensor form. Determines whether drop path is
local or global.
"""
if len(columns) == 1:
return columns[0]
with tf.variable_op_scope(columns, None, "Join"):
columns = tf.convert_to_tensor(columns)
columns = tf.cond(tflearn.get_training_mode(),
lambda: drop_path(columns, coin), lambda: columns)
out = tf.reduce_mean(columns, 0)
return out
def central_cut(net, block_size, shrink_factor):
output_shape = net.get_shape().as_list()
output_shape[1] = None
cut_size = tf.shape(net)[1] - tf.div(block_size, shrink_factor)
with tf.control_dependencies([
tf.cond(
tflearn.get_training_mode(), lambda: tf.assert_equal(
tf.mod(cut_size, 2), 0, name="cut_size_assert"),
lambda: tf.no_op()),
tf.assert_non_negative(cut_size)
]):
cut_size = tf.div(cut_size, 2)
net = tf.slice(net, [0, cut_size, 0],
[-1, tf.div(block_size, shrink_factor), -1],
name="Cutting")
net.set_shape(output_shape)
return net
def dropout(incoming, keep_prob, name="Dropout"):
""" Dropout.
Outputs the input element scaled up by `1 / keep_prob`. The scaling is so
that the expected sum is unchanged.
Arguments:
incoming : A `Tensor`. The incoming tensor.
keep_prob : A float representing the probability that each element
is kept.
name : A name for this layer (optional).
References:
Dropout: A Simple Way to Prevent Neural Networks from Overfitting.
N. Srivastava, G. Hinton, A. Krizhevsky, I. Sutskever & R. Salakhutdinov,
(2014), Journal of Machine Learning Research, 5(Jun)(2), 1929-1958.
Links:
[https://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf]
(https://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf)
"""
with tf.name_scope(name) as scope:
inference = incoming
def apply_dropout():
if type(inference) in [list, np.array]:
for x in inference:
x = tf.nn.dropout(x, keep_prob)
return inference
else:
return tf.nn.dropout(inference, keep_prob)
is_training = tflearn.get_training_mode()
inference = tf.cond(is_training, apply_dropout, lambda: inference)
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
def dropout(incoming, keep_prob, name="Dropout"):
""" Dropout.
Outputs the input element scaled up by `1 / keep_prob`. The scaling is so
that the expected sum is unchanged.
Arguments:
incoming : A `Tensor`. The incoming tensor.
keep_prob : A float representing the probability that each element
is kept.
name : A name for this layer (optional).
References:
Dropout: A Simple Way to Prevent Neural Networks from Overfitting.
N. Srivastava, G. Hinton, A. Krizhevsky, I. Sutskever & R. Salakhutdinov,
(2014), Journal of Machine Learning Research, 5(Jun)(2), 1929-1958.
Links:
[https://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf]
(https://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf)
"""
with tf.name_scope(name) as scope:
inference = incoming
def apply_dropout():
if type(inference) in [list, np.array]:
for x in inference:
x = tf.nn.dropout(x, keep_prob)
return inference
else:
return tf.nn.dropout(inference, keep_prob)
is_training = tflearn.get_training_mode()
inference = tf.cond(is_training, apply_dropout, lambda: inference)
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
def feats(self, save_dir, test_data, model_file, batch_size):
with tf.compat.v1.get_default_graph().as_default():
#config = tf.ConfigProto()
#config.gpu_options.allow_growth=True
model = self.module.Model(batch_size, False) #get_model_with_placeholders(self.module, reuse=False)
saver = tf.compat.v1.train.Saver()
with tf.compat.v1.Session() as sess:
sess.run(tf.compat.v1.global_variables_initializer())
saver.restore(sess, model_file)
tflearn.is_training(False)
#with tf.variable_scope("inference") as scope:
# scope.reuse_variables()
#print(sess.run(tf.get_variable('level7/trans4/W')))
#scope.reuse_variables()
print(sess.run(tflearn.get_training_mode()))
if not os.path.exists(save_dir):
os.makedirs(save_dir)
for tup in test_data:
feats_writer = model.build_feats_writer()
feats_writer(sess, tup, save_dir, model)
def join(columns,
coin):
"""Takes mean of the columns, applies drop path if
`tflearn.get_training_mode()` is True.
Args:
columns: columns of fractal block.
is_training: boolean in tensor form. Determines whether drop path
should be used.
coin: boolean in tensor form. Determines whether drop path is
local or global.
"""
if len(columns)==1:
return columns[0]
with tf.variable_op_scope(columns, None, "Join"):
columns = tf.convert_to_tensor(columns)
columns = tf.cond(tflearn.get_training_mode(),
lambda: drop_path(columns, coin),
lambda: columns)
out = tf.reduce_mean(columns, 0)
return out
# Hyper parameters for the one class Neural Network
v = 0.04
nu = 0.04
# Initialize rho
value = 0.0001
init = tf.constant_initializer(value)
rho = va.variable(name='rho', dtype=tf.float32, shape=[], initializer=init)
rcomputed = []
auc = []
sess = tf.Session()
sess.run(tf.initialize_all_variables())
print sess.run(tflearn.get_training_mode()) #False
tflearn.is_training(True, session=sess)
print sess.run(tflearn.get_training_mode()) #now True
X = data_train
D = X.shape[1]
nu = 0.04
# temp = np.random.normal(0, 1, K + K*D + 1)[-1]
temp = theta0[-1] * 1000000
# temp = tflearn.variables.get_value(rho, session=sess)
oneClassNN = oneClassNN(output_layer,
v,
def fractal_block(incoming,
filters,
ncols=3,
fsize=[3, 3],
joined=True,
reuse=False,
scope=None,
name="FractalBlock"):
Ws = [[] for _ in range(ncols)]
bs = [[] for _ in range(ncols)]
def conv_block(incoming, col):
with tf.variable_op_scope([incoming], None, "Column_{}".format(col)):
conv = tflearn.conv_2d(incoming,
filters,
fsize,
weights_init='xavier',
activation='linear')
net = tflearn.batch_normalization(conv)
net = tflearn.activation(net, 'relu')
Ws[col].append(conv.W)
bs[col].append(conv.b)
return net
def fractal_expand(incoming, col=0):
left = [conv_block(incoming, col)]
if (col == ncols - 1):
return left
right = join(fractal_expand(incoming, col + 1))
right = fractal_expand(right, col + 1)
out = left + right
return out
def together(name="Fractal"):
with tf.variable_op_scope([incoming], None, name) as scope:
out = fractal_expand(incoming)
net = join(out) if joined else out
return net
def random_col(cols, name='RandomColumn'):
with tf.variable_op_scope(cols, name):
col_idx = tf.random_uniform([], 0, len(cols), 'int32')
return tf.gather(cols, col_idx)
def seperated(name="Seperated"):
with tf.variable_op_scope([incoming], name) as scope:
sep = [incoming] * ncols
for col in range(ncols):
with tf.variable_op_scope([], None, "Column_{}".format(col)):
for idx, (W, b) in enumerate(zip(Ws[col], bs[col])):
with tf.variable_op_scope([incoming], None,
"ConvBlock") as scope:
conv = (tf.nn.conv2d(sep[col], W, [1, 1, 1, 1],
'SAME') + b)
conv = tflearn.batch_normalization(conv)
sep[col] = tf.nn.relu(conv)
return random_col(sep)
is_training = tflearn.get_training_mode()
with tf.variable_op_scope([incoming], scope, name, reuse=reuse) as scope:
fractal = together()
columns = seperated()
with tf.variable_op_scope([incoming], "DropPath"):
global_drop = tf.logical_and(is_training,
tf.random_uniform([]) > .5)
net = tf.cond(global_drop,
lambda: columns,
lambda: fractal,
name="DropPath")
return net
def batch_normalization(incoming, beta=0.0, gamma=1.0, epsilon=1e-5,
decay=0.9, stddev=0.002, trainable=True,
restore=True, reuse=False, scope=None,
name="BatchNormalization"):
""" Batch Normalization.
Normalize activations of the previous layer at each batch.
Arguments:
incoming: `Tensor`. Incoming Tensor.
beta: `float`. Default: 0.0.
gamma: `float`. Default: 1.0.
epsilon: `float`. Defalut: 1e-5.
decay: `float`. Default: 0.9.
stddev: `float`. Standard deviation for weights initialization.
trainable: `bool`. If True, weights will be trainable.
restore: `bool`. If True, this layer weights will be restored when
loading a model.
reuse: `bool`. If True and 'scope' is provided, this layer variables
will be reused (shared).
scope: `str`. Define this layer scope (optional). A scope can be
used to share variables between layers. Note that scope will
override name.
name: `str`. A name for this layer (optional).
References:
Batch Normalization: Accelerating Deep Network Training by Reducing
Internal Covariate Shif. Sergey Ioffe, Christian Szegedy. 2015.
Links:
[http://arxiv.org/pdf/1502.03167v3.pdf](http://arxiv.org/pdf/1502.03167v3.pdf)
"""
input_shape = utils.get_incoming_shape(incoming)
input_ndim = len(input_shape)
gamma_init = tf.random_normal_initializer(mean=gamma, stddev=stddev)
with tf.variable_op_scope([incoming], scope, name, reuse=reuse) as scope:
name = scope.name
beta = vs.variable('beta', shape=[input_shape[-1]],
initializer=tf.constant_initializer(beta),
trainable=trainable, restore=restore)
gamma = vs.variable('gamma', shape=[input_shape[-1]],
initializer=gamma_init, trainable=trainable,
restore=restore)
# Track per layer variables
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, beta)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, gamma)
if not restore:
tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, beta)
tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, gamma)
axis = list(range(input_ndim - 1))
moving_mean = vs.variable('moving_mean',
input_shape[-1:],
initializer=tf.zeros_initializer,
trainable=False,
restore=restore)
moving_variance = vs.variable('moving_variance',
input_shape[-1:],
initializer=tf.ones_initializer,
trainable=False,
restore=restore)
# Define a function to update mean and variance
def update_mean_var():
mean, variance = tf.nn.moments(incoming, axis)
update_moving_mean = moving_averages.assign_moving_average(
moving_mean, mean, decay)
update_moving_variance = moving_averages.assign_moving_average(
moving_variance, variance, decay)
with tf.control_dependencies(
[update_moving_mean, update_moving_variance]):
return tf.identity(mean), tf.identity(variance)
# Retrieve variable managing training mode
is_training = tflearn.get_training_mode()
mean, var = tf.python.control_flow_ops.cond(
is_training, update_mean_var, lambda: (moving_mean, moving_variance))
try:
inference = tf.nn.batch_normalization(
incoming, mean, var, beta, gamma, epsilon)
inference.set_shape(input_shape)
# Fix for old Tensorflow
except Exception as e:
inference = tf.nn.batch_norm_with_global_normalization(
incoming, mean, var, beta, gamma, epsilon,
scale_after_normalization=True,
)
inference.set_shape(input_shape)
# Add attributes for easy access
inference.scope = scope
inference.beta = beta
inference.gamma = gamma
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
cfg = json.load(fi)
FLAGS.checkpoint = cfg['checkpoint']
x, y = data_util.load_dataset_csv('dataset/{}/test.csv'.format(
FLAGS.dataset))
graph = tf.Graph()
with graph.as_default():
config = tflearn.config.init_graph(gpu_memory_fraction=FLAGS.mem)
cnn = net_class(1014,
num_classes=y.shape[1],
num_blocks=map(int, FLAGS.blocks.split(',')))
sess = tf.Session(config=config)
restore_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
restore_variables.remove(tflearn.get_training_mode())
saver = tf.train.Saver(restore_variables)
if FLAGS.checkpoint is None:
checkpoint_file = tf.train.latest_checkpoint(model_dir)
else:
checkpoint_file = '{}/model-{}'.format(model_dir, FLAGS.checkpoint)
#print(model_dir)
saver.restore(sess, checkpoint_file)
model = tflearn.DNN(cnn.train_op, session=sess)
if FLAGS.action == 'test':
print(model.evaluate(x, y, batch_size=FLAGS.batch_size))
elif FLAGS.action == 'notebook':
np.random.seed(10)
shuffle_indices = np.random.permutation(np.arange(len(y)))
def tflearn_OneClass_NN_linear(data_train, data_test, labels_train):
X = data_train
Y = labels_train
D = X.shape[1]
No_of_inputNodes = X.shape[1]
# Clear all the graph variables created in previous run and start fresh
tf.reset_default_graph()
# Define the network
input_layer = input_data(shape=[None,
No_of_inputNodes]) # input layer of size
np.random.seed(42)
theta0 = np.random.normal(0, 1, K + K * D + 1) * 0.0001
#theta0 = np.random.normal(0, 1, K + K*D + 1) # For linear
hidden_layer = fully_connected(
input_layer,
4,
bias=False,
activation='linear',
name="hiddenLayer_Weights",
weights_init="normal") # hidden layer of size 2
output_layer = fully_connected(
hidden_layer,
1,
bias=False,
activation='linear',
name="outputLayer_Weights",
weights_init="normal") # output layer of size 1
# Initialize rho
value = 0.01
init = tf.constant_initializer(value)
rho = va.variable(name='rho', dtype=tf.float32, shape=[], initializer=init)
rcomputed = []
auc = []
sess = tf.Session()
sess.run(tf.initialize_all_variables())
# print sess.run(tflearn.get_training_mode()) #False
tflearn.is_training(True, session=sess)
print sess.run(tflearn.get_training_mode()) #now True
temp = theta0[-1]
oneClassNN_Net = oneClassNN(output_layer,
v,
rho,
hidden_layer,
output_layer,
optimizer='sgd',
loss='OneClassNN_Loss',
learning_rate=1)
model = DNN(oneClassNN_Net, tensorboard_verbose=3)
model.set_weights(output_layer.W, theta0[0:K][:, np.newaxis])
model.set_weights(hidden_layer.W, np.reshape(theta0[K:K + K * D], (D, K)))
iterStep = 0
while (iterStep < 100):
print "Running Iteration :", iterStep
# Call the cost function
y_pred = model.predict(data_train) # Apply some ops
tflearn.is_training(False, session=sess)
y_pred_test = model.predict(data_test) # Apply some ops
tflearn.is_training(True, session=sess)
value = np.percentile(y_pred, v * 100)
tflearn.variables.set_value(rho, value, session=sess)
rStar = rho
model.fit(X, Y, n_epoch=2, show_metric=True, batch_size=100)
iterStep = iterStep + 1
rcomputed.append(rho)
temp = tflearn.variables.get_value(rho, session=sess)
# print "Rho",temp
# print "y_pred",y_pred
# print "y_predTest", y_pred_test
# g = lambda x: x
g = lambda x: 1 / (1 + tf.exp(-x))
def nnScore(X, w, V, g):
return tf.matmul(g((tf.matmul(X, w))), V)
# Format the datatype to suite the computation of nnscore
X = X.astype(np.float32)
X_test = data_test
X_test = X_test.astype(np.float32)
# assign the learnt weights
# wStar = hidden_layer.W
# VStar = output_layer.W
# Get weights values of fc2
wStar = model.get_weights(hidden_layer.W)
VStar = model.get_weights(output_layer.W)
# print "Hideen",wStar
# print VStar
train = nnScore(X, wStar, VStar, g)
test = nnScore(X_test, wStar, VStar, g)
# Access the value inside the train and test for plotting
# Create a new session and run the example
# sess = tf.Session()
# sess.run(tf.initialize_all_variables())
arrayTrain = train.eval(session=sess)
arrayTest = test.eval(session=sess)
# print "Train Array:",arrayTrain
# print "Test Array:",arrayTest
# plt.hist(arrayTrain-temp, bins = 25,label='Normal');
# plt.hist(arrayTest-temp, bins = 25, label='Anomalies');
# plt.legend(loc='upper right')
# plt.title('r = %1.6f- Sigmoid Activation ' % temp)
# plt.show()
pos_decisionScore = arrayTrain - temp
neg_decisionScore = arrayTest - temp
return [pos_decisionScore, neg_decisionScore]
def batch_normalization(incoming, beta=0.0, gamma=1.0, epsilon=1e-5,
decay=0.999, trainable=True, restore=True,
stddev=0.002, name="BatchNormalization"):
""" Batch Normalization.
Normalize activations of the previous layer at each batch.
Arguments:
incoming: `Tensor`. Incoming Tensor.
beta: `float`. Default: 0.0.
gamma: `float`. Default: 1.0.
epsilon: `float`. Defalut: 1e-5.
decay: `float`. Default: 0.999.
trainable: `bool`. If True, weights will be trainable.
restore: `bool`. If True, this layer weights will be restored when
loading a model.
stddev: `float`. Standard deviation for weights initialization.
name: `str`. A name for this layer (optional).
References:
Batch Normalization: Accelerating Deep Network Training by Reducing
Internal Covariate Shif. Sergey Ioffe, Christian Szegedy. 2015.
Links:
[http://arxiv.org/pdf/1502.03167v3.pdf](http://arxiv.org/pdf/1502.03167v3.pdf)
"""
input_shape = utils.get_incoming_shape(incoming)
input_ndim = len(input_shape)
gamma_init = tf.random_normal_initializer(mean=gamma, stddev=stddev)
with tf.name_scope(name) as scope:
beta = vs.variable(scope + 'beta', shape=[input_shape[-1]],
initializer=tf.constant_initializer(beta),
trainable=trainable, restore=restore)
gamma = vs.variable(scope + 'gamma', shape=[input_shape[-1]],
initializer=gamma_init, trainable=trainable,
restore=restore)
# Track per layer variables
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + scope, beta)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + scope, gamma)
if not restore:
tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, beta)
tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, gamma)
ema = tf.train.ExponentialMovingAverage(decay=decay)
axis = [i for i in range(input_ndim - 1)]
if len(axis) < 1: axis = [0]
batch_mean, batch_var = tf.nn.moments(incoming, axis,
name='moments')
ema_apply_op = ema.apply([batch_mean, batch_var])
ema_mean, ema_var = ema.average(batch_mean), ema.average(
batch_var)
def update_mean_var():
with tf.control_dependencies([ema_apply_op]):
return tf.identity(ema_mean), tf.identity(ema_var)
is_training = tflearn.get_training_mode()
mean, var = tf.python.control_flow_ops.cond(
is_training, update_mean_var, lambda: (ema_mean, ema_var))
try:
inference = tf.nn.batch_normalization(
incoming, mean, var, beta, gamma, epsilon)
# Fix for old Tensorflow
except Exception as e:
inference = tf.nn.batch_norm_with_global_normalization(
incoming, mean, var, beta, gamma, epsilon,
scale_after_normalization=True,
)
# Add attributes for easy access
inference.scope = scope
inference.beta = beta
inference.gamma = gamma
return inference
def _build_network(self, layers):
#restructuring input tensor
#network2 = tf.transpose(self.input_tensor, [0, 2, 3, 1])
network = tf.transpose(self.input_tensor, [0, 2, 3, 1])
#network, volume = tf.split(network2, [4, 1], 3)
# volume = network2[:, :, :, 4:5]
#volume = network2[:, :, :, 0:6]
# [batch, assets, window, features]
decay = 0.999
epsilon = 1e-3
#scale = tf.Variable(tf.ones([1, network.get_shape()[1], 1, network.get_shape()[-1]]))
#beta = tf.Variable(tf.ones([1, network.get_shape()[1], 1, network.get_shape()[-1]]))
#pop_mean = tf.Variable(tf.zeros([1, network.get_shape()[1], 1, network.get_shape()[-1]]), trainable=False)
#pop_var = tf.Variable(tf.ones([1, network.get_shape()[1], 1, network.get_shape()[-1]]), trainable=False)
#is_training = tflearn.get_training_mode()
#if is_training:
# batch_mean, batch_var = tf.nn.moments(network, [0, 2], keep_dims=True)
# train_mean = tf.assign(pop_mean, pop_mean * decay + batch_mean * (1 - decay))
# train_var = tf.assign(pop_var, pop_var * decay + batch_var * (1 - decay))
# with tf.control_dependencies([train_mean, train_var]):
# network = tf.nn.batch_normalization(network, batch_mean, batch_var, beta, scale, epsilon)
#else:
# network = tf.nn.batch_normalization(network, pop_mean, pop_var, beta, scale, epsilon)
#else:
network = network / network[:, :, -1, 0, None, None]
#volume = volume / (volume[:, :, -1, 0, None, None]+epsilon)
"""
"""
#tf.truediv(network[:, :, :, :-1, None], network[:, :, -1, None, :-1, None])
for layer_number, layer in enumerate(layers):
if layer["type"] == "Volume_LSTM":
volume = tf.transpose(volume, [0, 2, 3, 1])
# [batch, window, features, assets]
resultlist = []
reuse = False
for i in range(self._rows):
result = tflearn.layers.lstm(volume[:, :, :, i],
1,
dropout=(0.8, 0.8),
reuse=reuse,
scope="lstm")
reuse = True
resultlist.append(result)
volume = tf.stack(resultlist)
volume = tf.transpose(volume, [1, 0, 2])
volume = tf.reshape(volume, [-1, self._rows, 1, 1])
elif layer["type"] == "Volume_Conv":
width = volume.get_shape()[2]
volume = tflearn.layers.conv_2d(volume,
1, [1, width], [1, 1],
"valid",
"relu",
regularizer="L2",
weight_decay=5e-09)
elif layer["type"] == "DenseLayer":
network = tflearn.layers.core.fully_connected(
network,
int(layer["neuron_number"]),
layer["activation_function"],
regularizer=layer["regularizer"],
weight_decay=layer["weight_decay"])
elif layer["type"] == "DropOut":
network = tflearn.layers.core.dropout(
network, layer["keep_probability"])
#conv2d over window with output: [batch, assets, new window, filter number]
elif layer["type"] == "EIIE_Dense":
width = network.get_shape()[2]
network = tflearn.layers.conv_2d(
network,
int(layer["filter_number"]), [1, width], [1, 1],
"valid",
layer["activation_function"],
regularizer=layer["regularizer"],
weight_decay=layer["weight_decay"])
#Normalize activations of the previous layer at each batch based on Sergey Ioffe, Christian Szegedy. 2015
elif layer["type"] == "Batch_Normalization2":
scale2 = tf.Variable(
tf.ones([
1,
network.get_shape()[1], 1,
network.get_shape()[-1]
]))
beta2 = tf.Variable(
tf.ones([
1,
network.get_shape()[1], 1,
network.get_shape()[-1]
]))
pop_mean2 = tf.Variable(tf.zeros(
[1, network.get_shape()[1], 1,
network.get_shape()[-1]]),
trainable=False)
pop_var2 = tf.Variable(tf.ones(
[1, network.get_shape()[1], 1,
network.get_shape()[-1]]),
trainable=False)
is_training = tflearn.get_training_mode()
if is_training:
batch_mean2, batch_var2 = tf.nn.moments(network, [0, 2],
keep_dims=True)
train_mean2 = tf.assign(
pop_mean2,
pop_mean2 * decay + batch_mean2 * (1 - decay))
train_var2 = tf.assign(
pop_var2, pop_var2 * decay + batch_var2 * (1 - decay))
with tf.control_dependencies([train_mean2, train_var2]):
network = tf.nn.batch_normalization(
network, batch_mean2, batch_var2, beta2, scale2,
epsilon)
else:
network = tf.nn.batch_normalization(
network, pop_mean2, pop_var2, beta2, scale2, epsilon)
# Normalize activations of the previous layer at each batch based on Sergey Ioffe, Christian Szegedy. 2015
elif layer["type"] == "Batch_Normalization3":
scale3 = tf.Variable(
tf.ones([
1,
network.get_shape()[1], 1,
network.get_shape()[-1]
]))
beta3 = tf.Variable(
tf.ones([
1,
network.get_shape()[1], 1,
network.get_shape()[-1]
]))
pop_mean3 = tf.Variable(tf.zeros(
[1, network.get_shape()[1], 1,
network.get_shape()[-1]]),
trainable=False)
pop_var3 = tf.Variable(tf.ones(
[1, network.get_shape()[1], 1,
network.get_shape()[-1]]),
trainable=False)
is_training = tflearn.get_training_mode()
if is_training:
batch_mean3, batch_var3 = tf.nn.moments(network, [0, 2],
keep_dims=True)
train_mean3 = tf.assign(
pop_mean3,
pop_mean3 * decay + batch_mean3 * (1 - decay))
train_var3 = tf.assign(
pop_var3, pop_var3 * decay + batch_var3 * (1 - decay))
with tf.control_dependencies([train_mean3, train_var3]):
network = tf.nn.batch_normalization(
network, batch_mean3, batch_var3, beta3, scale3,
epsilon)
else:
network = tf.nn.batch_normalization(
network, pop_mean3, pop_var3, beta3, scale3, epsilon)
#conv2d over features with output: [batch, new features, new window, filter number]
elif layer["type"] == "ConvLayer_over_features":
network = tf.transpose(network, [0, 3, 2, 1])
# [batch, features, window, assets]
network = tflearn.layers.conv_2d(
network,
int(layer["filter_number"]),
allint(layer["filter_shape"]),
allint(layer["strides"]),
layer["padding"],
layer["activation_function"],
regularizer=layer["regularizer"],
weight_decay=layer["weight_decay"])
elif layer["type"] == "ConvLayer":
network = tflearn.layers.conv_2d(
network,
int(layer["filter_number"]),
allint(layer["filter_shape"]),
allint(layer["strides"]),
layer["padding"],
layer["activation_function"],
regularizer=layer["regularizer"],
weight_decay=layer["weight_decay"])
elif layer["type"] == "MaxPooling":
network = tflearn.layers.conv.max_pool_2d(
network, layer["strides"])
elif layer["type"] == "AveragePooling":
network = tflearn.layers.conv.avg_pool_2d(
network, layer["strides"])
elif layer["type"] == "LocalResponseNormalization":
network = tflearn.layers.normalization.local_response_normalization(
network)
elif layer["type"] == "FullyCon_WithW":
network = tflearn.flatten(network)
network = tf.concat([network, self.previous_w], axis=1)
network = tflearn.fully_connected(
network,
self._rows + 1,
activation="relu",
regularizer=layer["regularizer"],
weight_decay=layer["weight_decay"])
elif layer["type"] == "EIIE_Output":
width = network.get_shape()[2]
network = tflearn.layers.conv_2d(
network,
1, [1, width],
padding="valid",
regularizer=layer["regularizer"],
weight_decay=layer["weight_decay"])
network = network[:, :, 0, 0]
btc_bias = tf.ones((self.input_num, 1))
network = tf.concat([btc_bias, network], 1)
network = tflearn.layers.core.activation(network,
activation="softmax")
elif layer["type"] == "Output_WithW":
network = tflearn.fully_connected(
network,
self._rows + 1,
activation="softmax",
regularizer=layer["regularizer"],
weight_decay=layer["weight_decay"])
elif layer["type"] == "EIIE_Output_WithW":
width = network.get_shape()[2]
#window length
height = network.get_shape()[1]
#asset length
features = network.get_shape()[3]
#feature length
network = tf.reshape(
network,
[self.input_num,
int(height), 1,
int(width * features)])
w = tf.reshape(self.previous_w, [-1, int(height), 1, 1])
#volume = tf.reshape(volume, [-1, int(height), 1, 1])
#network = tf.concat([network, w, volume], axis=3)
network = tf.concat([network, w], axis=3)
#network = tf.concat([volume, w], axis=2)
network = tflearn.layers.conv_2d(
network,
1, [1, 1],
padding="valid",
regularizer=layer["regularizer"],
weight_decay=layer["weight_decay"])
network = network[:, :, 0, 0]
btc_bias = tf.zeros((self.input_num, 1))
network = tf.concat([btc_bias, network], 1)
self.voting = network
network = tflearn.layers.core.activation(network,
activation="softmax")
elif layer["type"] == "EIIE_LSTM" or\
layer["type"] == "EIIE_RNN":
network = tf.transpose(network, [0, 2, 3, 1])
#[batch, window, features, assets]
resultlist = []
reuse = False
for i in range(self._rows):
if i > 0:
reuse = True
if layer["type"] == "EIIE_LSTM":
result = tflearn.layers.lstm(
network[:, :, :, i],
int(layer["neuron_number"]),
dropout=layer["dropouts"],
scope="lstm" + str(layer_number),
reuse=reuse)
else:
result = tflearn.layers.simple_rnn(
network[:, :, :, i],
int(layer["neuron_number"]),
dropout=layer["dropouts"],
scope="rnn" + str(layer_number),
reuse=reuse)
resultlist.append(result)
network = tf.stack(resultlist)
network = tf.transpose(network, [1, 0, 2])
network = tf.reshape(
network, [-1, self._rows, 1,
int(layer["neuron_number"])])
else:
raise ValueError("the layer {} not supported.".format(
layer["type"]))
return network
def __init__(self, is_training, timeNeurons, timeLayers, noteNeurons,
noteLayers, dropout):
#this iterations defines how long the network will train for
iterations = 2000
with tf.Session() as sess:
# you can't reg variables to define network behavior, tf provides options for boolean logic though
is_training = tflearn.get_training_mode()
with tf.name_scope('inputs'):
batch = tf.placeholder(
tf.float32,
[None, None, 128, 2]) # (batch, time, notes, 2)
modelInput = tf.placeholder(
tf.float32, [None, None, 80]
) # subtract the last layer we don't need its output (time-1, batch*notes, vector len)
# Tensorflow needs 3d array so modelInput [[vector],[vector],[vector]->128*batch],[[vector],[vector],[vector]->128*batch]
# ------------------------------------time--------------------------------------
with tf.variable_scope("time"):
# LSTMCell makes a whole layer, in this case with 300 neurons, and timeStack is both of the layers
timeStack = tf.contrib.rnn.MultiRNNCell([
LSTMCell(timeNeurons[0], state_is_tuple=True)
for _ in range(timeLayers)
],
state_is_tuple=True)
#we pass our layers to dynamic_rnn and it loops through our modelInput and trains the weights
timeOutputs, _ = tf.nn.dynamic_rnn(timeStack,
modelInput,
dtype=tf.float32,
time_major=True)
# timeOutputs is the outputs at the last timestep, current of shape (time-1, batch*notes, vector len)
# In order to now loop throguh the notes in the note layer we need to get (note,time*batch,hiddens)
# first reshape to (time,batch,notes,hiddens) -> (notes, batch, time, hiddens) -> (note, time*batch,hiddens)
with tf.name_scope('reshaping'):
# timeFin = tf.reshape(timeOutputs, [batch_len-1,batch_width,128,300])
timeFin = tf.reshape(
timeOutputs,
[tf.shape(modelInput)[0],
tf.shape(batch)[0], 128, 300])
timeFin = tf.transpose(timeFin, [2, 1, 0, 3])
# timeFin = tf.reshape(timeFin, [128,batch_width*(batch_len-1),300])
timeFin = tf.reshape(
timeFin,
[128,
tf.shape(modelInput)[0] * tf.shape(batch)[0], 300])
actual_note = tf.cond(is_training, lambda: trainingInputs(batch),
lambda: predInputs(batch))
# we take the timeFin and smoosh it with the actual notes played along the 2 axis
# this means that the actual notes played are added with the 300 hiddens we are passing in
# (notes,batch*time,hiddens+batchs)
noteLayer_input = tf.concat([timeFin, actual_note], 2)
# This is the start of the note layer, we are passing (note, batch*time, hiddens)
with tf.variable_scope("note"):
noteStack = tf.contrib.rnn.MultiRNNCell([
LSTMCell(noteNeurons[i], state_is_tuple=True)
for i in range(noteLayers)
],
state_is_tuple=True)
noteOutputs, _ = tf.nn.dynamic_rnn(noteStack,
noteLayer_input,
dtype=tf.float32,
time_major=False)
# we now pass noteOutputs to a sigmoid layer
# noteOutputs is (note, batch*time, hiddens) -> (note, batch*time, 2)
# 2 represents [playProb,ArticProb]
sig_layer = nn_layer(noteOutputs, 50, 2, layer_name="sigmoid")
# NoteFin takes in the sigmoid layer (note, batch*time, 2) and reshapes it to (note, batch, time, 2)
# then transpose to be (batch,time,notes,2), which matches the shape of our batch inputs
noteFin = tf.reshape(sig_layer,
[128, batch_width, (batch_len - 1), 2])
noteFin = tf.transpose(noteFin, [1, 2, 0, 3])
# actual_note are the notes played at the next time step, we used this in addition to the time layer to pass into the note layer
# We now need this same input to test what comes out of the note layer
# here we are masking out the articulation prob
actualPlayProb = actual_note[:, :, 0:1]
# we mask out the articulation prob like we did for the actual_note
playProb = sig_layer[:, :, 0:1]
# the mask has all 1s for the play prob and 1s for the articProb if those notes where actually being played in the input batch
# we can multiple this by what we guess to get a more acc representation of our model predictions
actual_note_padded = tf.expand_dims(batch[:, 1:, :, 0], 3)
mask = tf.concat([
tf.ones_like(actual_note_padded, optimize=True),
actual_note_padded
],
axis=3)
# sometimes the cross entropy will go to 0 if epsilon is too small or there is no epsilon
eps = tf.constant(1.0e-7)
# cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=playProb,labels=actualPlayProb))
# this the cross entropy function
percentages = mask * tf.log(2 * noteFin * batch[:, 1:] - noteFin -
batch[:, 1:] + 1 + eps)
cost = tf.negative(tf.reduce_sum(percentages))
# optimizer - goes and adjusts our defined weights in backprop - trying to reduce cost
train_step = tf.train.RMSPropOptimizer(0.01).minimize(cost)
# train_step = tf.train.AdadeltaOptimizer(learning_rate=0.1, epsilon=1e-6).minimize(cost)
#saver lets us take snapshots of the network, sess run is a tf struct that needs to be called
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
train_writer = tf.summary.FileWriter('~.', sess.graph)
# training results is used for debugging/ checking our cost function
# f = open('training_results.txt', 'w')
# now we actually start training
tflearn.is_training(True)
song = numpy.array(numpy.zeros([1, 128, 2]))
for i in range(iterations):
# this grabs random sections in our song dictionary that we then train the network on
inputBatch, inputModelInput = getModelInputs()
# print(inputBatch)
# print(inputModelInput)
# looks at the cost function as the networks training
if i % 20 == 1:
train_accuracy = cost.eval(feed_dict={
batch: inputBatch,
modelInput: inputModelInput
})
print("step %d, training cost %g" % (i, train_accuracy))
# writing to our debug file
# f.write("step %d, training cost %g\n"%(i, train_accuracy))
# this is were the magic happens, this takes our song section and feeds it into the network
train_step.run(feed_dict={
batch: inputBatch,
modelInput: inputModelInput
})
# DEBUGGING PURPOSES DO NOT REMOVE
# if i > (iterations-100):
# an = sess.run([sig_layer],feed_dict={batch: inputBatch, modelInput:inputModelInput})
#
# result = tf.round(an[0]).eval()
# result = tf.transpose(result, [1,0,2]).eval()
# for k in range(128):
# if result[0,k,0] == 0:
# result[0,k,1] = 0
# print("======")
# print(result)
# print("___________")
# song = numpy.append(song, result, axis=0)
merged = tf.summary.merge_all()
train_writer.add_summary(merged, i)
# At this point the network has been trained and all the weights are set
# We now need to actually start generation
tflearn.is_training(False)
print("=======================================================")
# we need to input both the batch adn the model input, however because we are now generating
# we no longer use the batch inside the model -- hence the need for the tf.learn.is_training
startBatchInput = numpy.array(numpy.zeros([1, 1, 128, 2]))
preinput = numpy.array(numpy.zeros([1, 128, 2])).tolist()
startModelInput = numpy.array(batchToVectors(preinput))
prev = startBatchInput
# range will define how long the song will be
for j in range(500):
# we feed the input into the network, but instead of going all the way to the end with the cost function
# we only go until the sig_layer and take that output to be our notes that get played
result = sess.run([sig_layer],
feed_dict={
batch: startBatchInput,
modelInput: startModelInput
})
startBatchInput = prev
# print(result)
# we round the % values to being played or not played / 0 or 1
# note under 50% are not played, notes above 50% do get played
result = tf.round(result[0]).eval()
result = tf.transpose(result, [1, 0, 2]).eval()
# this is taking care of the articProb
for k in range(128):
if result[0, k, 0] == 0:
result[0, k, 1] = 0
# at this point our result no fits the structure of our statematrices and we can append and generate the next timestep
song = numpy.append(song, result, axis=0)
prev = numpy.reshape(result, [1, 1, 128, 2])
startModelInput = numpy.array([
stateToInputVectorArray(j, state)
for _, state in enumerate(result.tolist())
])
# print(batch)
# we now start the process of converting back to python 2.7 to construct our midi
matDict = dict()
matDict["test"] = song.tolist()
dataJSON = open('dataJSON.json', 'w')
json.dump(matDict, dataJSON)
dataJSON.close()
saver.save(sess, './train_model_checkpoint/export-model')
sess.close()
def model_fn(net, X_len, max_reach, block_size, out_classes, batch_size, dtype,
**kwargs):
"""
Args:
net -> Input tensor shaped (batch_size, max_reach + block_size + max_reach, 3)
Returns:
logits -> Unscaled logits tensor in time_major form, (block_size, batch_size, out_classes)
"""
with tf.name_scope("model"):
print("model in", net.get_shape())
for block in range(1, 4):
with tf.variable_scope("block%d" % block):
for layer in range(1, 20 + 1):
with tf.variable_scope('layer_%d' % layer):
res = net
for sublayer in [1, 2]:
res = batch_normalization(res,
scope='bn_%d' % sublayer)
res = tf.nn.relu(res)
res = conv_1d(
res,
64,
3,
scope="conv_1d_%d" % sublayer,
weights_init=variance_scaling_initializer(
dtype=dtype))
k = tf.get_variable(
"k",
initializer=tf.constant_initializer(1.0),
shape=[])
net = tf.nn.relu(k) * res + net
net = max_pool_1d(net, 2)
cut_size = tf.shape(net)[1] - tf.div(block_size, 8)
with tf.control_dependencies([
tf.cond(
tflearn.get_training_mode(), lambda: tf.assert_equal(
tf.mod(cut_size, 2), 0, name="cut_size_assert"),
lambda: tf.no_op())
]):
cut_size = tf.div(cut_size, 2)
net = tf.slice(net, [0, cut_size, 0],
[-1, tf.div(block_size, 8), -1],
name="Cutting")
print("after slice", net.get_shape())
net = tf.transpose(net, [1, 0, 2], name="Shift_to_time_major")
state_size = 64
outputs = net
print("outputs", outputs.get_shape())
with tf.variable_scope("Output"):
outputs = tf.reshape(outputs,
[block_size // 8 * batch_size, state_size],
name="flatten")
W = tf.get_variable("W", shape=[state_size, out_classes])
b = tf.get_variable("b", shape=[out_classes])
outputs = tf.matmul(outputs, W) + b
logits = tf.reshape(outputs,
[block_size // 8, batch_size, out_classes],
name="logits")
print("model out", logits.get_shape())
return {
'logits': logits,
'init_state': tf.constant(0),
'final_state': tf.constant(0),
}
def batch_normalization(incoming,
beta=0.0,
gamma=1.0,
epsilon=1e-5,
decay=0.9,
stddev=0.002,
trainable=True,
restore=True,
reuse=False,
scope=None,
name="BatchNormalization"):
""" Batch Normalization.
Normalize activations of the previous layer at each batch.
Arguments:
incoming: `Tensor`. Incoming Tensor.
beta: `float`. Default: 0.0.
gamma: `float`. Default: 1.0.
epsilon: `float`. Defalut: 1e-5.
decay: `float`. Default: 0.9.
stddev: `float`. Standard deviation for weights initialization.
trainable: `bool`. If True, weights will be trainable.
restore: `bool`. If True, this layer weights will be restored when
loading a model.
reuse: `bool`. If True and 'scope' is provided, this layer variables
will be reused (shared).
scope: `str`. Define this layer scope (optional). A scope can be
used to share variables between layers. Note that scope will
override name.
name: `str`. A name for this layer (optional).
References:
Batch Normalization: Accelerating Deep Network Training by Reducing
Internal Covariate Shif. Sergey Ioffe, Christian Szegedy. 2015.
Links:
[http://arxiv.org/pdf/1502.03167v3.pdf](http://arxiv.org/pdf/1502.03167v3.pdf)
"""
input_shape = utils.get_incoming_shape(incoming)
input_ndim = len(input_shape)
gamma_init = tf.random_normal_initializer(mean=gamma, stddev=stddev)
# Variable Scope fix for older TF
try:
vscope = tf.variable_scope(scope,
name=name,
values=[incoming],
reuse=reuse)
except Exception:
vscope = tf.variable_op_scope([incoming], scope, name, reuse=reuse)
with vscope as scope:
name = scope.name
beta = vs.variable('beta',
shape=[input_shape[-1]],
initializer=tf.constant_initializer(beta),
trainable=trainable,
restore=restore)
gamma = vs.variable('gamma',
shape=[input_shape[-1]],
initializer=gamma_init,
trainable=trainable,
restore=restore)
# Track per layer variables
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, beta)
tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, gamma)
if not restore:
tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, beta)
tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, gamma)
axis = list(range(input_ndim - 1))
moving_mean = vs.variable('moving_mean',
input_shape[-1:],
initializer=tf.zeros_initializer,
trainable=False,
restore=restore)
moving_variance = vs.variable('moving_variance',
input_shape[-1:],
initializer=tf.ones_initializer,
trainable=False,
restore=restore)
# Define a function to update mean and variance
def update_mean_var():
mean, variance = tf.nn.moments(incoming, axis)
update_moving_mean = moving_averages.assign_moving_average(
moving_mean, mean, decay)
update_moving_variance = moving_averages.assign_moving_average(
moving_variance, variance, decay)
with tf.control_dependencies(
[update_moving_mean, update_moving_variance]):
return tf.identity(mean), tf.identity(variance)
# Retrieve variable managing training mode
is_training = tflearn.get_training_mode()
mean, var = tf.cond(is_training, update_mean_var, lambda:
(moving_mean, moving_variance))
try:
inference = tf.nn.batch_normalization(incoming, mean, var, beta,
gamma, epsilon)
inference.set_shape(input_shape)
# Fix for old Tensorflow
except Exception as e:
inference = tf.nn.batch_norm_with_global_normalization(
incoming,
mean,
var,
beta,
gamma,
epsilon,
scale_after_normalization=True,
)
inference.set_shape(input_shape)
# Add attributes for easy access
inference.scope = scope
inference.beta = beta
inference.gamma = gamma
# Track output tensor.
tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference)
return inference
def augmentation(x):
return tf.cond(tflearn.get_training_mode(),
lambda: self.reconstruction([x, augFlow]),
lambda: x)
def __init__(self, is_training, timeNeurons, timeLayers, noteNeurons, noteLayers, dropout):
iterations = 5000;
with tf.Session() as sess:
is_training = tflearn.get_training_mode()
with tf.name_scope('inputs'):
batch = tf.placeholder(tf.float32, [None, None, 128, 2]) # (batch, time, notes, 2)
modelInput = tf.placeholder(tf.float32, [None, None, 80]) # subtract the last layer we don't need its output (time-1, batch*notes, vector len)
# Tensorflow needs 3d array so modelInput [[vector],[vector],[vector]->128*batch],[[vector],[vector],[vector]->128*batch]
# ------------------------------------time--------------------------------------
with tf.variable_scope("time"):
# LSTMCell makes a whole layer, in this case with 300 neurons, and timeStack is both of the layers
timeStack = tf.contrib.rnn.MultiRNNCell([LSTMCell(timeNeurons[0],state_is_tuple=True) for _ in range(timeLayers)], state_is_tuple=True)
#we pass our layers to dynamic_rnn and it loops through our modelInput and trains the weights
timeOutputs, _ = tf.nn.dynamic_rnn(timeStack, modelInput, dtype=tf.float32, time_major=True)
# timeOutputs is the outputs at the last timestep, current of shape (time-1, batch*notes, vector len)
# In order to now loop throguh the notes in the note layer we need to get (note,time*batch,hiddens)
# first reshape to (time,batch,notes,hiddens) -> (notes, batch, time, hiddens) -> (note, time*batch,hiddens)
with tf.name_scope('reshaping'):
# timeFin = tf.reshape(timeOutputs, [batch_len-1,batch_width,128,300])
timeFin = tf.reshape(timeOutputs, [tf.shape(modelInput)[0],tf.shape(batch)[0],128,300])
timeFin = tf.transpose(timeFin, [2,1,0,3])
# timeFin = tf.reshape(timeFin, [128,batch_width*(batch_len-1),300])
timeFin = tf.reshape(timeFin, [128,tf.shape(modelInput)[0]*tf.shape(batch)[0],300])
actual_note = tf.cond(is_training, lambda: trainingInputs(batch), lambda: predInputs(batch))
# we take the timeFin and smoosh it with the actual notes played along the 2 axis
# this means that the actual notes played are added with the 300 hiddens we are passing in
# (notes,batch*time,hiddens+batchs)
noteLayer_input = tf.concat([timeFin,actual_note],2)
# This is the start of the note layer, we are passing (note, batch*time, hiddens)
with tf.variable_scope("note"):
noteStack = tf.contrib.rnn.MultiRNNCell([LSTMCell(noteNeurons[i],state_is_tuple=True) for i in range(noteLayers)], state_is_tuple=True)
noteOutputs, _ = tf.nn.dynamic_rnn(noteStack, noteLayer_input, dtype=tf.float32, time_major=False)
# we now pass noteOutputs to a sigmoid layer
# noteOutputs is (note, batch*time, hiddens) -> (note, batch*time, 2)
# 2 represents [playProb,ArticProb]
sig_layer = nn_layer(noteOutputs, 50 , 2 , layer_name="sigmoid")
# NoteFin takes in the sigmoid layer (note, batch*time, 2) and reshapes it to (note, batch, time, 2)
# then transpose to be (batch,time,notes,2), which matches the shape of our batch inputs
noteFin = tf.reshape(sig_layer, [128,batch_width,(batch_len-1),2])
noteFin= tf.transpose(noteFin, [1,2,0,3])
# actual_note are the notes played at the next time step, we used this in addition to the time layer to pass into the note layer
# We now need this same input to test what comes out of the note layer
# here we are masking out the articulation prob
actualPlayProb = actual_note[:,:,0:1]
# we mask out the articulation prob like we did for the actual_note
playProb = sig_layer[:,:,0:1]
# the mask has all 1s for the play prob and 1s for the articProb if those notes where actually being played in the input batch
# we can multiple this by what we guess to get a more acc representation of our model predictions
actual_note_padded = tf.expand_dims(batch[:,1:,:,0],3)
# mask = tf.concat([tf.ones_like(actual_note_padded, optimize=True),actual_note_padded],axis=3)
# sometimes the cross entropy will go to 0 if epsilon is too small or there is no epsilon
eps = tf.constant(1.0e-7)
# cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=playProb,labels=actualPlayProb))
# this the cross entropy function
# percentages = mask * tf.log( 2 * noteFin * batch[:,1:] - noteFin - batch[:,1:] + 1 + eps )
percentages = tf.log( 2 * noteFin * batch[:,1:] - noteFin - batch[:,1:] + 1 + eps )
cost = tf.negative(tf.reduce_sum(percentages))
# optimizer
train_step = tf.train.RMSPropOptimizer(0.01).minimize(cost)
# train_step = tf.train.AdadeltaOptimizer(learning_rate=0.1, epsilon=1e-6).minimize(cost)
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
train_writer = tf.summary.FileWriter('~.', sess.graph)
f = open('training_results.txt', 'w')
tflearn.is_training(True)
for i in range(iterations):
inputBatch, inputModelInput = getModelInputs()
if i % 50 == 1:
train_accuracy = cost.eval(feed_dict={batch: inputBatch, modelInput:inputModelInput})
print("step %d, training cost %g"%(i, train_accuracy))
f.write("step %d, training cost %g\n"%(i, train_accuracy))
# train_step.run(feed_dict={batch: inputBatch, modelInput:inputModelInput})
train_step.run({batch: inputBatch, modelInput:inputModelInput})
# merged = tf.summary.merge_all()
# print(merged)
# train_writer.add_summary(tf.summary.merge_all(),i)
tflearn.is_training(False)
f.close()
print("=======================================================")
song = numpy.array(numpy.zeros([1,128,2]))
startBatchInput = numpy.array(numpy.zeros([1,1,128,2]))
preinput = numpy.array(numpy.zeros([1,128,2])).tolist()
startModelInput = numpy.array(batchToVectors(preinput))
prev = startBatchInput
for j in range(750):
result = sess.run([sig_layer],feed_dict={batch: startBatchInput, modelInput:startModelInput})[0]
startBatchInput = prev
# print(result)
# result = tf.nn.softmax(result)
result = tf.round(result)
result = tf.transpose(result, [1,0,2]).eval()
# print(result.shape)
for k in range(128):
if result[0,k,0] == 0:
result[0,k,1] = 0
startModelInput = numpy.array([stateToInputVectorArray(j,state) for _,state in enumerate(result.tolist())])
song = numpy.append(song, result, axis=0)
prev = numpy.reshape(result, [1,1,128,2])
# startModelInput = numpy.array([stateToInputVectorArray(j,state) for _,state in enumerate(result.tolist())])
# startModelInput = numpy.array(batchToVectors(result))
# print(batch)
matDict = dict()
matDict["test"] = song.tolist()
# dataJSON = open('StateMatrixData/dataJSON.json', 'w')
# json.dump(matDict, dataJSON)
# dataJSON.close()
dataFile = open('StateMatrixData/newSongDict.txt', 'wb')
pickle.dump(matDict, dataFile, pickle.HIGHEST_PROTOCOL)
dataFile.close()
saver.save(sess,'./train_model_checkpoint/export-model')
sess.close()
