def test(self, save_dir, test_data, model_file, batch_size, avg=False): #with tf.get_default_graph().as_default(): config = tf.compat.v1.ConfigProto() config.gpu_options.allow_growth = True model = self.module.Model(batch_size, False) #get_model_with_placeholders(self.module, reuse=False) if avg: variable_averages = tf.train.ExponentialMovingAverage(0.999) variables_to_restore = variable_averages.variables_to_restore() saver = tf.compat.v1.train.Saver(variables_to_restore) else: saver = tf.compat.v1.train.Saver() with tf.compat.v1.Session() as sess: sess.run(tf.compat.v1.global_variables_initializer()) saver.restore(sess, model_file) tflearn.is_training(False) with tf.compat.v1.variable_scope("inference") as scope: scope.reuse_variables() scope.reuse_variables() print(sess.run(tflearn.get_training_mode())) dices = {} if not os.path.exists(save_dir): os.makedirs(save_dir) for tup in test_data: inferer = model.build_full_inferer() dice = inferer(sess, tup, save_dir, model) dices[tup[0]] = dice print("Median: {}, Max: {}, Min: {}" .format(np.median(list(dices.values()), axis=0), np.max(list(dices.values()), axis=0), np.min(list(dices.values()), axis=0))) print(dices) return(dices) sess.close()
def get_input_layer(flags): img_prep = tflearn.ImagePreprocessing() img_prep.add_featurewise_zero_center() img_prep.add_featurewise_stdnorm() curr = input_data(shape=[None, flags.inres, flags.inres, flags.inchan], name='input', data_preprocessing=img_prep) if flags.rts_aug: w, h = curr.get_shape().as_list()[1:3] a = -flags.rts_aug_ang + 2 * flags.rts_aug_ang * tf.random_uniform( [flags.bs]) a *= np.pi / 180 # centralize rot/scale y = ((w - 1) - (tf.cos(a) * (w - 1) - tf.sin(a) * (h - 1))) / 2.0 x = ((h - 1) - (tf.sin(a) * (w - 1) + tf.cos(a) * (h - 1))) / 2.0 transforms = tf.transpose( tf.stack([ tf.cos(a), tf.sin(a), x, -tf.sin(a), tf.cos(a), y, tf.zeros(flags.bs), tf.zeros(flags.bs) ])) return tf.cond(tflearn.get_training_mode(), lambda: tf.contrib.image.transform(curr, transforms), lambda: curr) else: return curr
def augmenetation_pts(incoming): def aug(incoming): aug_pt = tf.cast(transform.warp_points( augFlow, incoming), tf.float32) pt_mask = tf.cast(tf.reduce_all( incoming >= 0, axis=-1, keep_dims=True), tf.float32) return aug_pt * pt_mask - (1 - pt_mask) return tf.cond(tflearn.get_training_mode(), lambda: aug(incoming), lambda: incoming)
def join(cols, drop_prob=.15): if len(cols) == 1: return cols[0] with tf.variable_op_scope(cols, None, "Join"): joined = tf.reduce_mean(cols, 0) out = tf.cond(tflearn.get_training_mode(), lambda: local_drop(cols, drop_prob), lambda: joined) return joined
def dropout(incoming, keep_prob, noise_shape=None, name="Dropout"): """ Dropout. Outputs the input element scaled up by `1 / keep_prob`. The scaling is so that the expected sum is unchanged. By default, each element is kept or dropped independently. If noise_shape is specified, it must be broadcastable to the shape of x, and only dimensions with noise_shape[i] == shape(x)[i] will make independent decisions. For example, if shape(x) = [k, l, m, n] and noise_shape = [k, 1, 1, n], each batch and channel component will be kept independently and each row and column will be kept or not kept together. Arguments: incoming : A `Tensor`. The incoming tensor. keep_prob : A float representing the probability that each element is kept. noise_shape : A 1-D Tensor of type int32, representing the shape for randomly generated keep/drop flags. name : A name for this layer (optional). References: Dropout: A Simple Way to Prevent Neural Networks from Overfitting. N. Srivastava, G. Hinton, A. Krizhevsky, I. Sutskever & R. Salakhutdinov, (2014), Journal of Machine Learning Research, 5(Jun)(2), 1929-1958. Links: [https://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf] (https://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf) """ with tf.name_scope(name) as scope: inference = incoming def apply_dropout(): if type(inference) in [list, np.array]: for x in inference: x = tf.nn.dropout(x, keep_prob, noise_shape) return inference else: return tf.nn.dropout(inference, keep_prob, noise_shape) is_training = tflearn.get_training_mode() inference = tf.cond(is_training, apply_dropout, lambda: inference) # Track output tensor. tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference) return inference
def build_model(self): self._create_placeholders() self._create_variables() result = self.inputs # forward layers for layer_no in range(self.layers_qtty - 1): layer_from = layer_no layer_to = layer_no + 1 weights = getattr(self, self._get_w_name(layer_from, layer_to)) bias = self._get_bias_tensor(layer_to) result = tf.sigmoid(tf.matmul(result, weights) + bias) # bin encoding layer_from = self.layers_qtty - 1 layer_to = self.layers_qtty weights = getattr(self, self._get_w_name(layer_from, layer_to)) bias = self._get_bias_tensor(layer_to) mat_mul_result = tf.matmul(result, weights) + bias # add noise only in case of training is_training = tflearn.get_training_mode() mat_mul_result = tf.cond(is_training, lambda: tf.add(mat_mul_result, self.noise), lambda: mat_mul_result) result = tf.sigmoid(mat_mul_result) self.encoded_array = result # backward layers for layer_no in range(self.layers_qtty, self.layers_qtty * 2): layer_from = layer_no layer_to = layer_no + 1 weights = getattr(self, self._get_w_name(layer_from, layer_to)) bias = self._get_bias_tensor(layer_to) result = tf.matmul(result, weights, transpose_b=True) + bias if layer_to != (self.layers_qtty * 2): result = tf.sigmoid(result) self.reconstr = result self.reconstr_prob = tf.sigmoid(result) # define cost and optimizer self.cost = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(self.reconstr, self.inputs)) optimizer = tf.train.GradientDescentOptimizer( self.params['learning_rate']) self.minimization = optimizer.minimize(self.cost) tf.scalar_summary("train_loss", self.cost) self.summary = tf.merge_all_summaries()
def inference(inputs, n_class=1000, finetuning=False): block_list = [10, 20, 9] scale_list = [0.17, 0.10, 0.20] print(inputs) net = stem(inputs, scope='stem') print(net) # bloack A for i in range(block_list[0]): net = blockA(net, scale=scale_list[0], scope='A_' + str(i)) print(net) # reduction A net = reductionA(net) # bloack B for i in range(block_list[1]): net = blockB(net, scale=scale_list[1], scope='B_' + str(i)) print(net) # reduction B net = reductionB(net) print(net) # bloack C for i in range(block_list[2]): net = blockB(net, scale=scale_list[2], scope='C_' + str(i)) print(net) net = tflearn.global_avg_pool(net) print(net) keep_prob = tf.cond(tflearn.get_training_mode(), lambda: 1.0, lambda: 0.8) net = tflearn.dropout(net, keep_prob=keep_prob) with tf.variable_scope('FC', reuse=tf.AUTO_REUSE): net = tflearn.fully_connected(net, n_class, weights_init='uniform_scaling', regularizer='L2', weight_decay=0.0001, restore=(not finetuning)) print(net) return net
def pt_regressor(layer_in, flags): net, curr = pt_regressor_conv(layer_in, flags) net['ptreg_in'] = layer_in dims = curr.get_shape().as_list() weights_init = 'zeros' bias_init = tf.ones([1]) # 1x1 conv, no BN, no ReLU on final heatmap net['ptreg_out'], curr = dup( conv_2d(curr, 1, 1, activation='linear', weights_init=weights_init, bias_init=bias_init, padding=flags.pad, name='ptreg_out')) # take the centroid of the feature map s = tf.shape(curr) # compute xc, yc from -1 to 1 xc = tf.tile(tf.linspace(-1., 1., s[2])[np.newaxis, ...], (s[1], 1)) yc = tf.transpose(xc) net['po_j'] = ( tf.reduce_sum(curr[..., 0] * xc[np.newaxis, ...], axis=(1, 2)) / tf.reduce_sum(curr[..., 0], axis=(1, 2))) net['po_i'] = ( tf.reduce_sum(curr[..., 0] * yc[np.newaxis, ...], axis=(1, 2)) / tf.reduce_sum(curr[..., 0], axis=(1, 2))) net['polar_origin'] = tf.stack([net['po_j'], net['po_i']], axis=1) # origin augmentation if flags.ptreg_aug > 0: dim = layer_in.get_shape().as_list()[1] shift = tf.cond( tflearn.get_training_mode(), lambda: 1. / dim * tf.random_uniform([flags.bs, 2], minval=-flags.ptreg_aug, maxval=flags.ptreg_aug), lambda: tf.zeros([flags.bs, 2])) net['polar_origin'] += shift return net
def join(columns, coin): """Takes mean of the columns, applies drop path if `tflearn.get_training_mode()` is True. Args: columns: columns of fractal block. is_training: boolean in tensor form. Determines whether drop path should be used. coin: boolean in tensor form. Determines whether drop path is local or global. """ if len(columns) == 1: return columns[0] with tf.variable_op_scope(columns, None, "Join"): columns = tf.convert_to_tensor(columns) columns = tf.cond(tflearn.get_training_mode(), lambda: drop_path(columns, coin), lambda: columns) out = tf.reduce_mean(columns, 0) return out
def central_cut(net, block_size, shrink_factor): output_shape = net.get_shape().as_list() output_shape[1] = None cut_size = tf.shape(net)[1] - tf.div(block_size, shrink_factor) with tf.control_dependencies([ tf.cond( tflearn.get_training_mode(), lambda: tf.assert_equal( tf.mod(cut_size, 2), 0, name="cut_size_assert"), lambda: tf.no_op()), tf.assert_non_negative(cut_size) ]): cut_size = tf.div(cut_size, 2) net = tf.slice(net, [0, cut_size, 0], [-1, tf.div(block_size, shrink_factor), -1], name="Cutting") net.set_shape(output_shape) return net
def dropout(incoming, keep_prob, name="Dropout"): """ Dropout. Outputs the input element scaled up by `1 / keep_prob`. The scaling is so that the expected sum is unchanged. Arguments: incoming : A `Tensor`. The incoming tensor. keep_prob : A float representing the probability that each element is kept. name : A name for this layer (optional). References: Dropout: A Simple Way to Prevent Neural Networks from Overfitting. N. Srivastava, G. Hinton, A. Krizhevsky, I. Sutskever & R. Salakhutdinov, (2014), Journal of Machine Learning Research, 5(Jun)(2), 1929-1958. Links: [https://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf] (https://www.cs.toronto.edu/~hinton/absps/JMLRdropout.pdf) """ with tf.name_scope(name) as scope: inference = incoming def apply_dropout(): if type(inference) in [list, np.array]: for x in inference: x = tf.nn.dropout(x, keep_prob) return inference else: return tf.nn.dropout(inference, keep_prob) is_training = tflearn.get_training_mode() inference = tf.cond(is_training, apply_dropout, lambda: inference) # Track output tensor. tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference) return inference
def feats(self, save_dir, test_data, model_file, batch_size): with tf.compat.v1.get_default_graph().as_default(): #config = tf.ConfigProto() #config.gpu_options.allow_growth=True model = self.module.Model(batch_size, False) #get_model_with_placeholders(self.module, reuse=False) saver = tf.compat.v1.train.Saver() with tf.compat.v1.Session() as sess: sess.run(tf.compat.v1.global_variables_initializer()) saver.restore(sess, model_file) tflearn.is_training(False) #with tf.variable_scope("inference") as scope: # scope.reuse_variables() #print(sess.run(tf.get_variable('level7/trans4/W'))) #scope.reuse_variables() print(sess.run(tflearn.get_training_mode())) if not os.path.exists(save_dir): os.makedirs(save_dir) for tup in test_data: feats_writer = model.build_feats_writer() feats_writer(sess, tup, save_dir, model)
def join(columns, coin): """Takes mean of the columns, applies drop path if `tflearn.get_training_mode()` is True. Args: columns: columns of fractal block. is_training: boolean in tensor form. Determines whether drop path should be used. coin: boolean in tensor form. Determines whether drop path is local or global. """ if len(columns)==1: return columns[0] with tf.variable_op_scope(columns, None, "Join"): columns = tf.convert_to_tensor(columns) columns = tf.cond(tflearn.get_training_mode(), lambda: drop_path(columns, coin), lambda: columns) out = tf.reduce_mean(columns, 0) return out
# Hyper parameters for the one class Neural Network v = 0.04 nu = 0.04 # Initialize rho value = 0.0001 init = tf.constant_initializer(value) rho = va.variable(name='rho', dtype=tf.float32, shape=[], initializer=init) rcomputed = [] auc = [] sess = tf.Session() sess.run(tf.initialize_all_variables()) print sess.run(tflearn.get_training_mode()) #False tflearn.is_training(True, session=sess) print sess.run(tflearn.get_training_mode()) #now True X = data_train D = X.shape[1] nu = 0.04 # temp = np.random.normal(0, 1, K + K*D + 1)[-1] temp = theta0[-1] * 1000000 # temp = tflearn.variables.get_value(rho, session=sess) oneClassNN = oneClassNN(output_layer, v,
def fractal_block(incoming, filters, ncols=3, fsize=[3, 3], joined=True, reuse=False, scope=None, name="FractalBlock"): Ws = [[] for _ in range(ncols)] bs = [[] for _ in range(ncols)] def conv_block(incoming, col): with tf.variable_op_scope([incoming], None, "Column_{}".format(col)): conv = tflearn.conv_2d(incoming, filters, fsize, weights_init='xavier', activation='linear') net = tflearn.batch_normalization(conv) net = tflearn.activation(net, 'relu') Ws[col].append(conv.W) bs[col].append(conv.b) return net def fractal_expand(incoming, col=0): left = [conv_block(incoming, col)] if (col == ncols - 1): return left right = join(fractal_expand(incoming, col + 1)) right = fractal_expand(right, col + 1) out = left + right return out def together(name="Fractal"): with tf.variable_op_scope([incoming], None, name) as scope: out = fractal_expand(incoming) net = join(out) if joined else out return net def random_col(cols, name='RandomColumn'): with tf.variable_op_scope(cols, name): col_idx = tf.random_uniform([], 0, len(cols), 'int32') return tf.gather(cols, col_idx) def seperated(name="Seperated"): with tf.variable_op_scope([incoming], name) as scope: sep = [incoming] * ncols for col in range(ncols): with tf.variable_op_scope([], None, "Column_{}".format(col)): for idx, (W, b) in enumerate(zip(Ws[col], bs[col])): with tf.variable_op_scope([incoming], None, "ConvBlock") as scope: conv = (tf.nn.conv2d(sep[col], W, [1, 1, 1, 1], 'SAME') + b) conv = tflearn.batch_normalization(conv) sep[col] = tf.nn.relu(conv) return random_col(sep) is_training = tflearn.get_training_mode() with tf.variable_op_scope([incoming], scope, name, reuse=reuse) as scope: fractal = together() columns = seperated() with tf.variable_op_scope([incoming], "DropPath"): global_drop = tf.logical_and(is_training, tf.random_uniform([]) > .5) net = tf.cond(global_drop, lambda: columns, lambda: fractal, name="DropPath") return net
def batch_normalization(incoming, beta=0.0, gamma=1.0, epsilon=1e-5, decay=0.9, stddev=0.002, trainable=True, restore=True, reuse=False, scope=None, name="BatchNormalization"): """ Batch Normalization. Normalize activations of the previous layer at each batch. Arguments: incoming: `Tensor`. Incoming Tensor. beta: `float`. Default: 0.0. gamma: `float`. Default: 1.0. epsilon: `float`. Defalut: 1e-5. decay: `float`. Default: 0.9. stddev: `float`. Standard deviation for weights initialization. trainable: `bool`. If True, weights will be trainable. restore: `bool`. If True, this layer weights will be restored when loading a model. reuse: `bool`. If True and 'scope' is provided, this layer variables will be reused (shared). scope: `str`. Define this layer scope (optional). A scope can be used to share variables between layers. Note that scope will override name. name: `str`. A name for this layer (optional). References: Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shif. Sergey Ioffe, Christian Szegedy. 2015. Links: [http://arxiv.org/pdf/1502.03167v3.pdf](http://arxiv.org/pdf/1502.03167v3.pdf) """ input_shape = utils.get_incoming_shape(incoming) input_ndim = len(input_shape) gamma_init = tf.random_normal_initializer(mean=gamma, stddev=stddev) with tf.variable_op_scope([incoming], scope, name, reuse=reuse) as scope: name = scope.name beta = vs.variable('beta', shape=[input_shape[-1]], initializer=tf.constant_initializer(beta), trainable=trainable, restore=restore) gamma = vs.variable('gamma', shape=[input_shape[-1]], initializer=gamma_init, trainable=trainable, restore=restore) # Track per layer variables tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, beta) tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, gamma) if not restore: tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, beta) tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, gamma) axis = list(range(input_ndim - 1)) moving_mean = vs.variable('moving_mean', input_shape[-1:], initializer=tf.zeros_initializer, trainable=False, restore=restore) moving_variance = vs.variable('moving_variance', input_shape[-1:], initializer=tf.ones_initializer, trainable=False, restore=restore) # Define a function to update mean and variance def update_mean_var(): mean, variance = tf.nn.moments(incoming, axis) update_moving_mean = moving_averages.assign_moving_average( moving_mean, mean, decay) update_moving_variance = moving_averages.assign_moving_average( moving_variance, variance, decay) with tf.control_dependencies( [update_moving_mean, update_moving_variance]): return tf.identity(mean), tf.identity(variance) # Retrieve variable managing training mode is_training = tflearn.get_training_mode() mean, var = tf.python.control_flow_ops.cond( is_training, update_mean_var, lambda: (moving_mean, moving_variance)) try: inference = tf.nn.batch_normalization( incoming, mean, var, beta, gamma, epsilon) inference.set_shape(input_shape) # Fix for old Tensorflow except Exception as e: inference = tf.nn.batch_norm_with_global_normalization( incoming, mean, var, beta, gamma, epsilon, scale_after_normalization=True, ) inference.set_shape(input_shape) # Add attributes for easy access inference.scope = scope inference.beta = beta inference.gamma = gamma # Track output tensor. tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference) return inference
cfg = json.load(fi) FLAGS.checkpoint = cfg['checkpoint'] x, y = data_util.load_dataset_csv('dataset/{}/test.csv'.format( FLAGS.dataset)) graph = tf.Graph() with graph.as_default(): config = tflearn.config.init_graph(gpu_memory_fraction=FLAGS.mem) cnn = net_class(1014, num_classes=y.shape[1], num_blocks=map(int, FLAGS.blocks.split(','))) sess = tf.Session(config=config) restore_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES) restore_variables.remove(tflearn.get_training_mode()) saver = tf.train.Saver(restore_variables) if FLAGS.checkpoint is None: checkpoint_file = tf.train.latest_checkpoint(model_dir) else: checkpoint_file = '{}/model-{}'.format(model_dir, FLAGS.checkpoint) #print(model_dir) saver.restore(sess, checkpoint_file) model = tflearn.DNN(cnn.train_op, session=sess) if FLAGS.action == 'test': print(model.evaluate(x, y, batch_size=FLAGS.batch_size)) elif FLAGS.action == 'notebook': np.random.seed(10) shuffle_indices = np.random.permutation(np.arange(len(y)))
def tflearn_OneClass_NN_linear(data_train, data_test, labels_train): X = data_train Y = labels_train D = X.shape[1] No_of_inputNodes = X.shape[1] # Clear all the graph variables created in previous run and start fresh tf.reset_default_graph() # Define the network input_layer = input_data(shape=[None, No_of_inputNodes]) # input layer of size np.random.seed(42) theta0 = np.random.normal(0, 1, K + K * D + 1) * 0.0001 #theta0 = np.random.normal(0, 1, K + K*D + 1) # For linear hidden_layer = fully_connected( input_layer, 4, bias=False, activation='linear', name="hiddenLayer_Weights", weights_init="normal") # hidden layer of size 2 output_layer = fully_connected( hidden_layer, 1, bias=False, activation='linear', name="outputLayer_Weights", weights_init="normal") # output layer of size 1 # Initialize rho value = 0.01 init = tf.constant_initializer(value) rho = va.variable(name='rho', dtype=tf.float32, shape=[], initializer=init) rcomputed = [] auc = [] sess = tf.Session() sess.run(tf.initialize_all_variables()) # print sess.run(tflearn.get_training_mode()) #False tflearn.is_training(True, session=sess) print sess.run(tflearn.get_training_mode()) #now True temp = theta0[-1] oneClassNN_Net = oneClassNN(output_layer, v, rho, hidden_layer, output_layer, optimizer='sgd', loss='OneClassNN_Loss', learning_rate=1) model = DNN(oneClassNN_Net, tensorboard_verbose=3) model.set_weights(output_layer.W, theta0[0:K][:, np.newaxis]) model.set_weights(hidden_layer.W, np.reshape(theta0[K:K + K * D], (D, K))) iterStep = 0 while (iterStep < 100): print "Running Iteration :", iterStep # Call the cost function y_pred = model.predict(data_train) # Apply some ops tflearn.is_training(False, session=sess) y_pred_test = model.predict(data_test) # Apply some ops tflearn.is_training(True, session=sess) value = np.percentile(y_pred, v * 100) tflearn.variables.set_value(rho, value, session=sess) rStar = rho model.fit(X, Y, n_epoch=2, show_metric=True, batch_size=100) iterStep = iterStep + 1 rcomputed.append(rho) temp = tflearn.variables.get_value(rho, session=sess) # print "Rho",temp # print "y_pred",y_pred # print "y_predTest", y_pred_test # g = lambda x: x g = lambda x: 1 / (1 + tf.exp(-x)) def nnScore(X, w, V, g): return tf.matmul(g((tf.matmul(X, w))), V) # Format the datatype to suite the computation of nnscore X = X.astype(np.float32) X_test = data_test X_test = X_test.astype(np.float32) # assign the learnt weights # wStar = hidden_layer.W # VStar = output_layer.W # Get weights values of fc2 wStar = model.get_weights(hidden_layer.W) VStar = model.get_weights(output_layer.W) # print "Hideen",wStar # print VStar train = nnScore(X, wStar, VStar, g) test = nnScore(X_test, wStar, VStar, g) # Access the value inside the train and test for plotting # Create a new session and run the example # sess = tf.Session() # sess.run(tf.initialize_all_variables()) arrayTrain = train.eval(session=sess) arrayTest = test.eval(session=sess) # print "Train Array:",arrayTrain # print "Test Array:",arrayTest # plt.hist(arrayTrain-temp, bins = 25,label='Normal'); # plt.hist(arrayTest-temp, bins = 25, label='Anomalies'); # plt.legend(loc='upper right') # plt.title('r = %1.6f- Sigmoid Activation ' % temp) # plt.show() pos_decisionScore = arrayTrain - temp neg_decisionScore = arrayTest - temp return [pos_decisionScore, neg_decisionScore]
def batch_normalization(incoming, beta=0.0, gamma=1.0, epsilon=1e-5, decay=0.999, trainable=True, restore=True, stddev=0.002, name="BatchNormalization"): """ Batch Normalization. Normalize activations of the previous layer at each batch. Arguments: incoming: `Tensor`. Incoming Tensor. beta: `float`. Default: 0.0. gamma: `float`. Default: 1.0. epsilon: `float`. Defalut: 1e-5. decay: `float`. Default: 0.999. trainable: `bool`. If True, weights will be trainable. restore: `bool`. If True, this layer weights will be restored when loading a model. stddev: `float`. Standard deviation for weights initialization. name: `str`. A name for this layer (optional). References: Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shif. Sergey Ioffe, Christian Szegedy. 2015. Links: [http://arxiv.org/pdf/1502.03167v3.pdf](http://arxiv.org/pdf/1502.03167v3.pdf) """ input_shape = utils.get_incoming_shape(incoming) input_ndim = len(input_shape) gamma_init = tf.random_normal_initializer(mean=gamma, stddev=stddev) with tf.name_scope(name) as scope: beta = vs.variable(scope + 'beta', shape=[input_shape[-1]], initializer=tf.constant_initializer(beta), trainable=trainable, restore=restore) gamma = vs.variable(scope + 'gamma', shape=[input_shape[-1]], initializer=gamma_init, trainable=trainable, restore=restore) # Track per layer variables tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + scope, beta) tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + scope, gamma) if not restore: tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, beta) tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, gamma) ema = tf.train.ExponentialMovingAverage(decay=decay) axis = [i for i in range(input_ndim - 1)] if len(axis) < 1: axis = [0] batch_mean, batch_var = tf.nn.moments(incoming, axis, name='moments') ema_apply_op = ema.apply([batch_mean, batch_var]) ema_mean, ema_var = ema.average(batch_mean), ema.average( batch_var) def update_mean_var(): with tf.control_dependencies([ema_apply_op]): return tf.identity(ema_mean), tf.identity(ema_var) is_training = tflearn.get_training_mode() mean, var = tf.python.control_flow_ops.cond( is_training, update_mean_var, lambda: (ema_mean, ema_var)) try: inference = tf.nn.batch_normalization( incoming, mean, var, beta, gamma, epsilon) # Fix for old Tensorflow except Exception as e: inference = tf.nn.batch_norm_with_global_normalization( incoming, mean, var, beta, gamma, epsilon, scale_after_normalization=True, ) # Add attributes for easy access inference.scope = scope inference.beta = beta inference.gamma = gamma return inference
def _build_network(self, layers): #restructuring input tensor #network2 = tf.transpose(self.input_tensor, [0, 2, 3, 1]) network = tf.transpose(self.input_tensor, [0, 2, 3, 1]) #network, volume = tf.split(network2, [4, 1], 3) # volume = network2[:, :, :, 4:5] #volume = network2[:, :, :, 0:6] # [batch, assets, window, features] decay = 0.999 epsilon = 1e-3 #scale = tf.Variable(tf.ones([1, network.get_shape()[1], 1, network.get_shape()[-1]])) #beta = tf.Variable(tf.ones([1, network.get_shape()[1], 1, network.get_shape()[-1]])) #pop_mean = tf.Variable(tf.zeros([1, network.get_shape()[1], 1, network.get_shape()[-1]]), trainable=False) #pop_var = tf.Variable(tf.ones([1, network.get_shape()[1], 1, network.get_shape()[-1]]), trainable=False) #is_training = tflearn.get_training_mode() #if is_training: # batch_mean, batch_var = tf.nn.moments(network, [0, 2], keep_dims=True) # train_mean = tf.assign(pop_mean, pop_mean * decay + batch_mean * (1 - decay)) # train_var = tf.assign(pop_var, pop_var * decay + batch_var * (1 - decay)) # with tf.control_dependencies([train_mean, train_var]): # network = tf.nn.batch_normalization(network, batch_mean, batch_var, beta, scale, epsilon) #else: # network = tf.nn.batch_normalization(network, pop_mean, pop_var, beta, scale, epsilon) #else: network = network / network[:, :, -1, 0, None, None] #volume = volume / (volume[:, :, -1, 0, None, None]+epsilon) """ """ #tf.truediv(network[:, :, :, :-1, None], network[:, :, -1, None, :-1, None]) for layer_number, layer in enumerate(layers): if layer["type"] == "Volume_LSTM": volume = tf.transpose(volume, [0, 2, 3, 1]) # [batch, window, features, assets] resultlist = [] reuse = False for i in range(self._rows): result = tflearn.layers.lstm(volume[:, :, :, i], 1, dropout=(0.8, 0.8), reuse=reuse, scope="lstm") reuse = True resultlist.append(result) volume = tf.stack(resultlist) volume = tf.transpose(volume, [1, 0, 2]) volume = tf.reshape(volume, [-1, self._rows, 1, 1]) elif layer["type"] == "Volume_Conv": width = volume.get_shape()[2] volume = tflearn.layers.conv_2d(volume, 1, [1, width], [1, 1], "valid", "relu", regularizer="L2", weight_decay=5e-09) elif layer["type"] == "DenseLayer": network = tflearn.layers.core.fully_connected( network, int(layer["neuron_number"]), layer["activation_function"], regularizer=layer["regularizer"], weight_decay=layer["weight_decay"]) elif layer["type"] == "DropOut": network = tflearn.layers.core.dropout( network, layer["keep_probability"]) #conv2d over window with output: [batch, assets, new window, filter number] elif layer["type"] == "EIIE_Dense": width = network.get_shape()[2] network = tflearn.layers.conv_2d( network, int(layer["filter_number"]), [1, width], [1, 1], "valid", layer["activation_function"], regularizer=layer["regularizer"], weight_decay=layer["weight_decay"]) #Normalize activations of the previous layer at each batch based on Sergey Ioffe, Christian Szegedy. 2015 elif layer["type"] == "Batch_Normalization2": scale2 = tf.Variable( tf.ones([ 1, network.get_shape()[1], 1, network.get_shape()[-1] ])) beta2 = tf.Variable( tf.ones([ 1, network.get_shape()[1], 1, network.get_shape()[-1] ])) pop_mean2 = tf.Variable(tf.zeros( [1, network.get_shape()[1], 1, network.get_shape()[-1]]), trainable=False) pop_var2 = tf.Variable(tf.ones( [1, network.get_shape()[1], 1, network.get_shape()[-1]]), trainable=False) is_training = tflearn.get_training_mode() if is_training: batch_mean2, batch_var2 = tf.nn.moments(network, [0, 2], keep_dims=True) train_mean2 = tf.assign( pop_mean2, pop_mean2 * decay + batch_mean2 * (1 - decay)) train_var2 = tf.assign( pop_var2, pop_var2 * decay + batch_var2 * (1 - decay)) with tf.control_dependencies([train_mean2, train_var2]): network = tf.nn.batch_normalization( network, batch_mean2, batch_var2, beta2, scale2, epsilon) else: network = tf.nn.batch_normalization( network, pop_mean2, pop_var2, beta2, scale2, epsilon) # Normalize activations of the previous layer at each batch based on Sergey Ioffe, Christian Szegedy. 2015 elif layer["type"] == "Batch_Normalization3": scale3 = tf.Variable( tf.ones([ 1, network.get_shape()[1], 1, network.get_shape()[-1] ])) beta3 = tf.Variable( tf.ones([ 1, network.get_shape()[1], 1, network.get_shape()[-1] ])) pop_mean3 = tf.Variable(tf.zeros( [1, network.get_shape()[1], 1, network.get_shape()[-1]]), trainable=False) pop_var3 = tf.Variable(tf.ones( [1, network.get_shape()[1], 1, network.get_shape()[-1]]), trainable=False) is_training = tflearn.get_training_mode() if is_training: batch_mean3, batch_var3 = tf.nn.moments(network, [0, 2], keep_dims=True) train_mean3 = tf.assign( pop_mean3, pop_mean3 * decay + batch_mean3 * (1 - decay)) train_var3 = tf.assign( pop_var3, pop_var3 * decay + batch_var3 * (1 - decay)) with tf.control_dependencies([train_mean3, train_var3]): network = tf.nn.batch_normalization( network, batch_mean3, batch_var3, beta3, scale3, epsilon) else: network = tf.nn.batch_normalization( network, pop_mean3, pop_var3, beta3, scale3, epsilon) #conv2d over features with output: [batch, new features, new window, filter number] elif layer["type"] == "ConvLayer_over_features": network = tf.transpose(network, [0, 3, 2, 1]) # [batch, features, window, assets] network = tflearn.layers.conv_2d( network, int(layer["filter_number"]), allint(layer["filter_shape"]), allint(layer["strides"]), layer["padding"], layer["activation_function"], regularizer=layer["regularizer"], weight_decay=layer["weight_decay"]) elif layer["type"] == "ConvLayer": network = tflearn.layers.conv_2d( network, int(layer["filter_number"]), allint(layer["filter_shape"]), allint(layer["strides"]), layer["padding"], layer["activation_function"], regularizer=layer["regularizer"], weight_decay=layer["weight_decay"]) elif layer["type"] == "MaxPooling": network = tflearn.layers.conv.max_pool_2d( network, layer["strides"]) elif layer["type"] == "AveragePooling": network = tflearn.layers.conv.avg_pool_2d( network, layer["strides"]) elif layer["type"] == "LocalResponseNormalization": network = tflearn.layers.normalization.local_response_normalization( network) elif layer["type"] == "FullyCon_WithW": network = tflearn.flatten(network) network = tf.concat([network, self.previous_w], axis=1) network = tflearn.fully_connected( network, self._rows + 1, activation="relu", regularizer=layer["regularizer"], weight_decay=layer["weight_decay"]) elif layer["type"] == "EIIE_Output": width = network.get_shape()[2] network = tflearn.layers.conv_2d( network, 1, [1, width], padding="valid", regularizer=layer["regularizer"], weight_decay=layer["weight_decay"]) network = network[:, :, 0, 0] btc_bias = tf.ones((self.input_num, 1)) network = tf.concat([btc_bias, network], 1) network = tflearn.layers.core.activation(network, activation="softmax") elif layer["type"] == "Output_WithW": network = tflearn.fully_connected( network, self._rows + 1, activation="softmax", regularizer=layer["regularizer"], weight_decay=layer["weight_decay"]) elif layer["type"] == "EIIE_Output_WithW": width = network.get_shape()[2] #window length height = network.get_shape()[1] #asset length features = network.get_shape()[3] #feature length network = tf.reshape( network, [self.input_num, int(height), 1, int(width * features)]) w = tf.reshape(self.previous_w, [-1, int(height), 1, 1]) #volume = tf.reshape(volume, [-1, int(height), 1, 1]) #network = tf.concat([network, w, volume], axis=3) network = tf.concat([network, w], axis=3) #network = tf.concat([volume, w], axis=2) network = tflearn.layers.conv_2d( network, 1, [1, 1], padding="valid", regularizer=layer["regularizer"], weight_decay=layer["weight_decay"]) network = network[:, :, 0, 0] btc_bias = tf.zeros((self.input_num, 1)) network = tf.concat([btc_bias, network], 1) self.voting = network network = tflearn.layers.core.activation(network, activation="softmax") elif layer["type"] == "EIIE_LSTM" or\ layer["type"] == "EIIE_RNN": network = tf.transpose(network, [0, 2, 3, 1]) #[batch, window, features, assets] resultlist = [] reuse = False for i in range(self._rows): if i > 0: reuse = True if layer["type"] == "EIIE_LSTM": result = tflearn.layers.lstm( network[:, :, :, i], int(layer["neuron_number"]), dropout=layer["dropouts"], scope="lstm" + str(layer_number), reuse=reuse) else: result = tflearn.layers.simple_rnn( network[:, :, :, i], int(layer["neuron_number"]), dropout=layer["dropouts"], scope="rnn" + str(layer_number), reuse=reuse) resultlist.append(result) network = tf.stack(resultlist) network = tf.transpose(network, [1, 0, 2]) network = tf.reshape( network, [-1, self._rows, 1, int(layer["neuron_number"])]) else: raise ValueError("the layer {} not supported.".format( layer["type"])) return network
def __init__(self, is_training, timeNeurons, timeLayers, noteNeurons, noteLayers, dropout): #this iterations defines how long the network will train for iterations = 2000 with tf.Session() as sess: # you can't reg variables to define network behavior, tf provides options for boolean logic though is_training = tflearn.get_training_mode() with tf.name_scope('inputs'): batch = tf.placeholder( tf.float32, [None, None, 128, 2]) # (batch, time, notes, 2) modelInput = tf.placeholder( tf.float32, [None, None, 80] ) # subtract the last layer we don't need its output (time-1, batch*notes, vector len) # Tensorflow needs 3d array so modelInput [[vector],[vector],[vector]->128*batch],[[vector],[vector],[vector]->128*batch] # ------------------------------------time-------------------------------------- with tf.variable_scope("time"): # LSTMCell makes a whole layer, in this case with 300 neurons, and timeStack is both of the layers timeStack = tf.contrib.rnn.MultiRNNCell([ LSTMCell(timeNeurons[0], state_is_tuple=True) for _ in range(timeLayers) ], state_is_tuple=True) #we pass our layers to dynamic_rnn and it loops through our modelInput and trains the weights timeOutputs, _ = tf.nn.dynamic_rnn(timeStack, modelInput, dtype=tf.float32, time_major=True) # timeOutputs is the outputs at the last timestep, current of shape (time-1, batch*notes, vector len) # In order to now loop throguh the notes in the note layer we need to get (note,time*batch,hiddens) # first reshape to (time,batch,notes,hiddens) -> (notes, batch, time, hiddens) -> (note, time*batch,hiddens) with tf.name_scope('reshaping'): # timeFin = tf.reshape(timeOutputs, [batch_len-1,batch_width,128,300]) timeFin = tf.reshape( timeOutputs, [tf.shape(modelInput)[0], tf.shape(batch)[0], 128, 300]) timeFin = tf.transpose(timeFin, [2, 1, 0, 3]) # timeFin = tf.reshape(timeFin, [128,batch_width*(batch_len-1),300]) timeFin = tf.reshape( timeFin, [128, tf.shape(modelInput)[0] * tf.shape(batch)[0], 300]) actual_note = tf.cond(is_training, lambda: trainingInputs(batch), lambda: predInputs(batch)) # we take the timeFin and smoosh it with the actual notes played along the 2 axis # this means that the actual notes played are added with the 300 hiddens we are passing in # (notes,batch*time,hiddens+batchs) noteLayer_input = tf.concat([timeFin, actual_note], 2) # This is the start of the note layer, we are passing (note, batch*time, hiddens) with tf.variable_scope("note"): noteStack = tf.contrib.rnn.MultiRNNCell([ LSTMCell(noteNeurons[i], state_is_tuple=True) for i in range(noteLayers) ], state_is_tuple=True) noteOutputs, _ = tf.nn.dynamic_rnn(noteStack, noteLayer_input, dtype=tf.float32, time_major=False) # we now pass noteOutputs to a sigmoid layer # noteOutputs is (note, batch*time, hiddens) -> (note, batch*time, 2) # 2 represents [playProb,ArticProb] sig_layer = nn_layer(noteOutputs, 50, 2, layer_name="sigmoid") # NoteFin takes in the sigmoid layer (note, batch*time, 2) and reshapes it to (note, batch, time, 2) # then transpose to be (batch,time,notes,2), which matches the shape of our batch inputs noteFin = tf.reshape(sig_layer, [128, batch_width, (batch_len - 1), 2]) noteFin = tf.transpose(noteFin, [1, 2, 0, 3]) # actual_note are the notes played at the next time step, we used this in addition to the time layer to pass into the note layer # We now need this same input to test what comes out of the note layer # here we are masking out the articulation prob actualPlayProb = actual_note[:, :, 0:1] # we mask out the articulation prob like we did for the actual_note playProb = sig_layer[:, :, 0:1] # the mask has all 1s for the play prob and 1s for the articProb if those notes where actually being played in the input batch # we can multiple this by what we guess to get a more acc representation of our model predictions actual_note_padded = tf.expand_dims(batch[:, 1:, :, 0], 3) mask = tf.concat([ tf.ones_like(actual_note_padded, optimize=True), actual_note_padded ], axis=3) # sometimes the cross entropy will go to 0 if epsilon is too small or there is no epsilon eps = tf.constant(1.0e-7) # cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=playProb,labels=actualPlayProb)) # this the cross entropy function percentages = mask * tf.log(2 * noteFin * batch[:, 1:] - noteFin - batch[:, 1:] + 1 + eps) cost = tf.negative(tf.reduce_sum(percentages)) # optimizer - goes and adjusts our defined weights in backprop - trying to reduce cost train_step = tf.train.RMSPropOptimizer(0.01).minimize(cost) # train_step = tf.train.AdadeltaOptimizer(learning_rate=0.1, epsilon=1e-6).minimize(cost) #saver lets us take snapshots of the network, sess run is a tf struct that needs to be called saver = tf.train.Saver() sess.run(tf.global_variables_initializer()) train_writer = tf.summary.FileWriter('~.', sess.graph) # training results is used for debugging/ checking our cost function # f = open('training_results.txt', 'w') # now we actually start training tflearn.is_training(True) song = numpy.array(numpy.zeros([1, 128, 2])) for i in range(iterations): # this grabs random sections in our song dictionary that we then train the network on inputBatch, inputModelInput = getModelInputs() # print(inputBatch) # print(inputModelInput) # looks at the cost function as the networks training if i % 20 == 1: train_accuracy = cost.eval(feed_dict={ batch: inputBatch, modelInput: inputModelInput }) print("step %d, training cost %g" % (i, train_accuracy)) # writing to our debug file # f.write("step %d, training cost %g\n"%(i, train_accuracy)) # this is were the magic happens, this takes our song section and feeds it into the network train_step.run(feed_dict={ batch: inputBatch, modelInput: inputModelInput }) # DEBUGGING PURPOSES DO NOT REMOVE # if i > (iterations-100): # an = sess.run([sig_layer],feed_dict={batch: inputBatch, modelInput:inputModelInput}) # # result = tf.round(an[0]).eval() # result = tf.transpose(result, [1,0,2]).eval() # for k in range(128): # if result[0,k,0] == 0: # result[0,k,1] = 0 # print("======") # print(result) # print("___________") # song = numpy.append(song, result, axis=0) merged = tf.summary.merge_all() train_writer.add_summary(merged, i) # At this point the network has been trained and all the weights are set # We now need to actually start generation tflearn.is_training(False) print("=======================================================") # we need to input both the batch adn the model input, however because we are now generating # we no longer use the batch inside the model -- hence the need for the tf.learn.is_training startBatchInput = numpy.array(numpy.zeros([1, 1, 128, 2])) preinput = numpy.array(numpy.zeros([1, 128, 2])).tolist() startModelInput = numpy.array(batchToVectors(preinput)) prev = startBatchInput # range will define how long the song will be for j in range(500): # we feed the input into the network, but instead of going all the way to the end with the cost function # we only go until the sig_layer and take that output to be our notes that get played result = sess.run([sig_layer], feed_dict={ batch: startBatchInput, modelInput: startModelInput }) startBatchInput = prev # print(result) # we round the % values to being played or not played / 0 or 1 # note under 50% are not played, notes above 50% do get played result = tf.round(result[0]).eval() result = tf.transpose(result, [1, 0, 2]).eval() # this is taking care of the articProb for k in range(128): if result[0, k, 0] == 0: result[0, k, 1] = 0 # at this point our result no fits the structure of our statematrices and we can append and generate the next timestep song = numpy.append(song, result, axis=0) prev = numpy.reshape(result, [1, 1, 128, 2]) startModelInput = numpy.array([ stateToInputVectorArray(j, state) for _, state in enumerate(result.tolist()) ]) # print(batch) # we now start the process of converting back to python 2.7 to construct our midi matDict = dict() matDict["test"] = song.tolist() dataJSON = open('dataJSON.json', 'w') json.dump(matDict, dataJSON) dataJSON.close() saver.save(sess, './train_model_checkpoint/export-model') sess.close()
def model_fn(net, X_len, max_reach, block_size, out_classes, batch_size, dtype, **kwargs): """ Args: net -> Input tensor shaped (batch_size, max_reach + block_size + max_reach, 3) Returns: logits -> Unscaled logits tensor in time_major form, (block_size, batch_size, out_classes) """ with tf.name_scope("model"): print("model in", net.get_shape()) for block in range(1, 4): with tf.variable_scope("block%d" % block): for layer in range(1, 20 + 1): with tf.variable_scope('layer_%d' % layer): res = net for sublayer in [1, 2]: res = batch_normalization(res, scope='bn_%d' % sublayer) res = tf.nn.relu(res) res = conv_1d( res, 64, 3, scope="conv_1d_%d" % sublayer, weights_init=variance_scaling_initializer( dtype=dtype)) k = tf.get_variable( "k", initializer=tf.constant_initializer(1.0), shape=[]) net = tf.nn.relu(k) * res + net net = max_pool_1d(net, 2) cut_size = tf.shape(net)[1] - tf.div(block_size, 8) with tf.control_dependencies([ tf.cond( tflearn.get_training_mode(), lambda: tf.assert_equal( tf.mod(cut_size, 2), 0, name="cut_size_assert"), lambda: tf.no_op()) ]): cut_size = tf.div(cut_size, 2) net = tf.slice(net, [0, cut_size, 0], [-1, tf.div(block_size, 8), -1], name="Cutting") print("after slice", net.get_shape()) net = tf.transpose(net, [1, 0, 2], name="Shift_to_time_major") state_size = 64 outputs = net print("outputs", outputs.get_shape()) with tf.variable_scope("Output"): outputs = tf.reshape(outputs, [block_size // 8 * batch_size, state_size], name="flatten") W = tf.get_variable("W", shape=[state_size, out_classes]) b = tf.get_variable("b", shape=[out_classes]) outputs = tf.matmul(outputs, W) + b logits = tf.reshape(outputs, [block_size // 8, batch_size, out_classes], name="logits") print("model out", logits.get_shape()) return { 'logits': logits, 'init_state': tf.constant(0), 'final_state': tf.constant(0), }
def batch_normalization(incoming, beta=0.0, gamma=1.0, epsilon=1e-5, decay=0.9, stddev=0.002, trainable=True, restore=True, reuse=False, scope=None, name="BatchNormalization"): """ Batch Normalization. Normalize activations of the previous layer at each batch. Arguments: incoming: `Tensor`. Incoming Tensor. beta: `float`. Default: 0.0. gamma: `float`. Default: 1.0. epsilon: `float`. Defalut: 1e-5. decay: `float`. Default: 0.9. stddev: `float`. Standard deviation for weights initialization. trainable: `bool`. If True, weights will be trainable. restore: `bool`. If True, this layer weights will be restored when loading a model. reuse: `bool`. If True and 'scope' is provided, this layer variables will be reused (shared). scope: `str`. Define this layer scope (optional). A scope can be used to share variables between layers. Note that scope will override name. name: `str`. A name for this layer (optional). References: Batch Normalization: Accelerating Deep Network Training by Reducing Internal Covariate Shif. Sergey Ioffe, Christian Szegedy. 2015. Links: [http://arxiv.org/pdf/1502.03167v3.pdf](http://arxiv.org/pdf/1502.03167v3.pdf) """ input_shape = utils.get_incoming_shape(incoming) input_ndim = len(input_shape) gamma_init = tf.random_normal_initializer(mean=gamma, stddev=stddev) # Variable Scope fix for older TF try: vscope = tf.variable_scope(scope, name=name, values=[incoming], reuse=reuse) except Exception: vscope = tf.variable_op_scope([incoming], scope, name, reuse=reuse) with vscope as scope: name = scope.name beta = vs.variable('beta', shape=[input_shape[-1]], initializer=tf.constant_initializer(beta), trainable=trainable, restore=restore) gamma = vs.variable('gamma', shape=[input_shape[-1]], initializer=gamma_init, trainable=trainable, restore=restore) # Track per layer variables tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, beta) tf.add_to_collection(tf.GraphKeys.LAYER_VARIABLES + '/' + name, gamma) if not restore: tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, beta) tf.add_to_collection(tf.GraphKeys.EXCL_RESTORE_VARS, gamma) axis = list(range(input_ndim - 1)) moving_mean = vs.variable('moving_mean', input_shape[-1:], initializer=tf.zeros_initializer, trainable=False, restore=restore) moving_variance = vs.variable('moving_variance', input_shape[-1:], initializer=tf.ones_initializer, trainable=False, restore=restore) # Define a function to update mean and variance def update_mean_var(): mean, variance = tf.nn.moments(incoming, axis) update_moving_mean = moving_averages.assign_moving_average( moving_mean, mean, decay) update_moving_variance = moving_averages.assign_moving_average( moving_variance, variance, decay) with tf.control_dependencies( [update_moving_mean, update_moving_variance]): return tf.identity(mean), tf.identity(variance) # Retrieve variable managing training mode is_training = tflearn.get_training_mode() mean, var = tf.cond(is_training, update_mean_var, lambda: (moving_mean, moving_variance)) try: inference = tf.nn.batch_normalization(incoming, mean, var, beta, gamma, epsilon) inference.set_shape(input_shape) # Fix for old Tensorflow except Exception as e: inference = tf.nn.batch_norm_with_global_normalization( incoming, mean, var, beta, gamma, epsilon, scale_after_normalization=True, ) inference.set_shape(input_shape) # Add attributes for easy access inference.scope = scope inference.beta = beta inference.gamma = gamma # Track output tensor. tf.add_to_collection(tf.GraphKeys.LAYER_TENSOR + '/' + name, inference) return inference
def augmentation(x): return tf.cond(tflearn.get_training_mode(), lambda: self.reconstruction([x, augFlow]), lambda: x)
def __init__(self, is_training, timeNeurons, timeLayers, noteNeurons, noteLayers, dropout): iterations = 5000; with tf.Session() as sess: is_training = tflearn.get_training_mode() with tf.name_scope('inputs'): batch = tf.placeholder(tf.float32, [None, None, 128, 2]) # (batch, time, notes, 2) modelInput = tf.placeholder(tf.float32, [None, None, 80]) # subtract the last layer we don't need its output (time-1, batch*notes, vector len) # Tensorflow needs 3d array so modelInput [[vector],[vector],[vector]->128*batch],[[vector],[vector],[vector]->128*batch] # ------------------------------------time-------------------------------------- with tf.variable_scope("time"): # LSTMCell makes a whole layer, in this case with 300 neurons, and timeStack is both of the layers timeStack = tf.contrib.rnn.MultiRNNCell([LSTMCell(timeNeurons[0],state_is_tuple=True) for _ in range(timeLayers)], state_is_tuple=True) #we pass our layers to dynamic_rnn and it loops through our modelInput and trains the weights timeOutputs, _ = tf.nn.dynamic_rnn(timeStack, modelInput, dtype=tf.float32, time_major=True) # timeOutputs is the outputs at the last timestep, current of shape (time-1, batch*notes, vector len) # In order to now loop throguh the notes in the note layer we need to get (note,time*batch,hiddens) # first reshape to (time,batch,notes,hiddens) -> (notes, batch, time, hiddens) -> (note, time*batch,hiddens) with tf.name_scope('reshaping'): # timeFin = tf.reshape(timeOutputs, [batch_len-1,batch_width,128,300]) timeFin = tf.reshape(timeOutputs, [tf.shape(modelInput)[0],tf.shape(batch)[0],128,300]) timeFin = tf.transpose(timeFin, [2,1,0,3]) # timeFin = tf.reshape(timeFin, [128,batch_width*(batch_len-1),300]) timeFin = tf.reshape(timeFin, [128,tf.shape(modelInput)[0]*tf.shape(batch)[0],300]) actual_note = tf.cond(is_training, lambda: trainingInputs(batch), lambda: predInputs(batch)) # we take the timeFin and smoosh it with the actual notes played along the 2 axis # this means that the actual notes played are added with the 300 hiddens we are passing in # (notes,batch*time,hiddens+batchs) noteLayer_input = tf.concat([timeFin,actual_note],2) # This is the start of the note layer, we are passing (note, batch*time, hiddens) with tf.variable_scope("note"): noteStack = tf.contrib.rnn.MultiRNNCell([LSTMCell(noteNeurons[i],state_is_tuple=True) for i in range(noteLayers)], state_is_tuple=True) noteOutputs, _ = tf.nn.dynamic_rnn(noteStack, noteLayer_input, dtype=tf.float32, time_major=False) # we now pass noteOutputs to a sigmoid layer # noteOutputs is (note, batch*time, hiddens) -> (note, batch*time, 2) # 2 represents [playProb,ArticProb] sig_layer = nn_layer(noteOutputs, 50 , 2 , layer_name="sigmoid") # NoteFin takes in the sigmoid layer (note, batch*time, 2) and reshapes it to (note, batch, time, 2) # then transpose to be (batch,time,notes,2), which matches the shape of our batch inputs noteFin = tf.reshape(sig_layer, [128,batch_width,(batch_len-1),2]) noteFin= tf.transpose(noteFin, [1,2,0,3]) # actual_note are the notes played at the next time step, we used this in addition to the time layer to pass into the note layer # We now need this same input to test what comes out of the note layer # here we are masking out the articulation prob actualPlayProb = actual_note[:,:,0:1] # we mask out the articulation prob like we did for the actual_note playProb = sig_layer[:,:,0:1] # the mask has all 1s for the play prob and 1s for the articProb if those notes where actually being played in the input batch # we can multiple this by what we guess to get a more acc representation of our model predictions actual_note_padded = tf.expand_dims(batch[:,1:,:,0],3) # mask = tf.concat([tf.ones_like(actual_note_padded, optimize=True),actual_note_padded],axis=3) # sometimes the cross entropy will go to 0 if epsilon is too small or there is no epsilon eps = tf.constant(1.0e-7) # cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=playProb,labels=actualPlayProb)) # this the cross entropy function # percentages = mask * tf.log( 2 * noteFin * batch[:,1:] - noteFin - batch[:,1:] + 1 + eps ) percentages = tf.log( 2 * noteFin * batch[:,1:] - noteFin - batch[:,1:] + 1 + eps ) cost = tf.negative(tf.reduce_sum(percentages)) # optimizer train_step = tf.train.RMSPropOptimizer(0.01).minimize(cost) # train_step = tf.train.AdadeltaOptimizer(learning_rate=0.1, epsilon=1e-6).minimize(cost) saver = tf.train.Saver() sess.run(tf.global_variables_initializer()) train_writer = tf.summary.FileWriter('~.', sess.graph) f = open('training_results.txt', 'w') tflearn.is_training(True) for i in range(iterations): inputBatch, inputModelInput = getModelInputs() if i % 50 == 1: train_accuracy = cost.eval(feed_dict={batch: inputBatch, modelInput:inputModelInput}) print("step %d, training cost %g"%(i, train_accuracy)) f.write("step %d, training cost %g\n"%(i, train_accuracy)) # train_step.run(feed_dict={batch: inputBatch, modelInput:inputModelInput}) train_step.run({batch: inputBatch, modelInput:inputModelInput}) # merged = tf.summary.merge_all() # print(merged) # train_writer.add_summary(tf.summary.merge_all(),i) tflearn.is_training(False) f.close() print("=======================================================") song = numpy.array(numpy.zeros([1,128,2])) startBatchInput = numpy.array(numpy.zeros([1,1,128,2])) preinput = numpy.array(numpy.zeros([1,128,2])).tolist() startModelInput = numpy.array(batchToVectors(preinput)) prev = startBatchInput for j in range(750): result = sess.run([sig_layer],feed_dict={batch: startBatchInput, modelInput:startModelInput})[0] startBatchInput = prev # print(result) # result = tf.nn.softmax(result) result = tf.round(result) result = tf.transpose(result, [1,0,2]).eval() # print(result.shape) for k in range(128): if result[0,k,0] == 0: result[0,k,1] = 0 startModelInput = numpy.array([stateToInputVectorArray(j,state) for _,state in enumerate(result.tolist())]) song = numpy.append(song, result, axis=0) prev = numpy.reshape(result, [1,1,128,2]) # startModelInput = numpy.array([stateToInputVectorArray(j,state) for _,state in enumerate(result.tolist())]) # startModelInput = numpy.array(batchToVectors(result)) # print(batch) matDict = dict() matDict["test"] = song.tolist() # dataJSON = open('StateMatrixData/dataJSON.json', 'w') # json.dump(matDict, dataJSON) # dataJSON.close() dataFile = open('StateMatrixData/newSongDict.txt', 'wb') pickle.dump(matDict, dataFile, pickle.HIGHEST_PROTOCOL) dataFile.close() saver.save(sess,'./train_model_checkpoint/export-model') sess.close()