def _autoencoder(self): updates = [] layers = self._layers[:-1] for i, layer in enumerate(layers): a = layer.output # [3000, 784] u = self._layers[i + 1].output kernel = layer.kernel temporary_shape = utility.transpose_shape(kernel) # [1000, 784] if layer.use_bias: temporary_shape[0] += 1 kernel = tf.concat((kernel, layer.bias[None, ...]), axis=0) temporary_kernel = tf.get_variable( 'temporal_{}'.format(i), temporary_shape, dtype=tf.float64, initializer=tf.contrib.layers.xavier_initializer(), trainable=False) u, _ = mf.semi_nmf( a=a, u=u, v=temporary_kernel, use_tf=True, use_bias=layer.use_bias, num_iters=1, first_nneg=True, ) # Not use activation (ReLU) if not layer.activation: _, v = mf.semi_nmf( a=u, u=a, v=kernel, use_tf=True, use_bias=layer.use_bias, num_iters=1, first_nneg=True, ) # Use activation (ReLU) # else utility.get_op_name(layer.activation) == 'Relu': else: _, v = mf.nonlin_semi_nmf( a=u, u=a, v=kernel, use_tf=True, use_bias=layer.use_bias, num_calc_v=0, num_calc_u=1, first_nneg=True, ) if layer.use_bias: v, bias = utility.split_v_bias(v) updates.append(layer.bias.assign(bias)) updates.append(layer.kernel.assign(v)) return tf.group(*updates)
def test_tf_vanilla_semi_nmf(self): a = np.random.uniform(-1., 1., size=(100, 100)) u = np.random.uniform(0., 1., size=(100, 300)) v = np.random.uniform(-1., 1., size=(300, 100)) old_loss = np_frobenius_norm(a, u @ v) # [1000, 500] a_ph = tf.placeholder(tf.float64, shape=a.shape) # [1000, 201] u_ph = tf.placeholder(tf.float64, shape=u.shape) # [200, 500] v_ph = tf.placeholder(tf.float64, shape=v.shape) tf_u, tf_v = semi_nmf(a_ph, u_ph, v_ph, use_tf=True, use_bias=False) tf_loss = frobenius_norm(a_ph, tf.matmul(tf_u, tf_v)) init = tf.global_variables_initializer() with tf.Session() as sess: init.run() start_time = time.time() _u, _v, new_loss = sess.run([tf_u, tf_v, tf_loss], feed_dict={a_ph: a, u_ph: u, v_ph: v}) assert np.min(_u) > 0, np.min(_u) end_time = time.time() duration = end_time - start_time assert a.shape == (_u @ _v).shape assert new_loss < old_loss, "new loss should be less than old loss." print('\n[TensorFlow]Solve semi-NMF\n\t' 'old loss {0}\n\t' 'new loss {1}\n\t' 'process duration {2}'.format(old_loss, new_loss, duration)) print_format('TensorFlow', 'semi-NMF', a, u, v, old_loss, new_loss, duration)
def test_tf_biased_semi_nmf(self): auv = sio.loadmat(mat_file) a, u, v = auv['a'], auv['u'], auv['v'] bias_v = np.vstack((v, np.ones((1, v.shape[1])))) old_loss = np_frobenius_norm(a, u @ v) a_ph = tf.placeholder(tf.float64, shape=a.shape) u_ph = tf.placeholder(tf.float64, shape=u.shape) bias_v_ph = tf.placeholder(tf.float64, shape=bias_v.shape) tf_bias_u, tf_v = semi_nmf(a_ph, u_ph, bias_v_ph, use_bias=True, use_tf=True) init = tf.global_variables_initializer() with tf.Session() as sess: init.run() start_time = time.time() _u, _bias_v = sess.run([tf_bias_u, tf_v], feed_dict={a_ph: a, u_ph: u, bias_v_ph: bias_v}) end_time = time.time() duration = end_time - start_time _bias_u = np.hstack((_u, np.ones((_u.shape[0], 1)))) new_loss = np_frobenius_norm(a, _bias_u @ _bias_v) assert a.shape == (_bias_u @ _bias_v).shape assert new_loss < old_loss, "new loss should be less than old loss." print_format('TensorFlow', 'biased semi-NMF', a, _bias_u, _bias_v, old_loss, new_loss, duration)
def test_simplest_factorize(self): print() model = benchmark_model.build_tf_model() ops = utility.get_train_ops() layers = utility.zip_layer(inputs=model.inputs, ops=ops) hidden = layers[-1].output last_weights = layers[-1].kernel tf_u, tf_v = semi_nmf(model.labels, hidden, last_weights, use_tf=True, use_bias=False, num_iters=3) _old_local_loss = losses.frobenius_norm(model.labels, hidden @ last_weights) _new_local_loss = losses.frobenius_norm(model.labels, tf_u @ tf_v) x, y = benchmark_model.build_data(batch_size, label_size) init = tf.global_variables_initializer() with self.test_session() as sess: sess.run(init) old_local_loss, new_local_loss = sess.run( [_old_local_loss, _new_local_loss], feed_dict={ model.inputs: x, model.labels: y, }) self.assertGreater(old_local_loss, new_local_loss) print("old {} new {}".format(old_local_loss, new_local_loss))
def test_tf_vanilla_semi_nmf(self): auv = sio.loadmat(mat_file) a, u, v = auv['a'], auv['u'], auv['v'] old_loss = np_frobenius_norm(a, u @ v) # [1000, 500] a_ph = tf.placeholder(tf.float64, shape=a.shape) # [1000, 201] u_ph = tf.placeholder(tf.float64, shape=u.shape) # [200, 500] v_ph = tf.placeholder(tf.float64, shape=v.shape) tf_u, tf_v = semi_nmf(a_ph, u_ph, v_ph, use_tf=True, use_bias=False) tf_loss = frobenius_norm(a_ph, tf.matmul(tf_u, tf_v)) init = tf.global_variables_initializer() with tf.Session() as sess: init.run() start_time = time.time() _u, _v, new_loss = sess.run([tf_u, tf_v, tf_loss], feed_dict={ a_ph: a, u_ph: u, v_ph: v }) end_time = time.time() duration = end_time - start_time assert a.shape == (_u @ _v).shape assert new_loss < old_loss, "new loss should be less than old loss." print('\n[TensorFlow]Solve semi-NMF\n\t' 'old loss {0}\n\t' 'new loss {1}\n\t' 'process duration {2}'.format(old_loss, new_loss, duration))
def minimize(self, loss=None): """Construct the control dependencies for calculating neural net optimized. Returns: tf.no_op. The import """ self._init(loss) if self._use_autoencoder: self._autoencoder() a = self.labels updates = [] # Reverse layers = self._layers[::-1] for i, layer in enumerate(layers): _u = layer.output v = layer.kernel # Check whether u is a tensor or not. # that is Recurrent output if it have dim more than 3. if _u.shape.ndims >= 3 and not isinstance(layer.recurrent, tf.Variable): u = _u[:, -1, :] else: u = _u if isinstance(layer.recurrent, tf.Variable): v = tf.concat((layer.kernel, layer.recurrent), axis=0) if layer.use_bias: v = tf.concat((v, layer.bias[None, ...]), axis=0) # Not use activation (ReLU) if not layer.activation: u, v = mf.semi_nmf(a=a, u=u, v=v, use_tf=True, use_bias=layer.use_bias, num_iters=1, first_nneg=True, ) # Use activation (ReLU) else: u, v = mf.nonlin_semi_nmf(a=a, u=u, v=v, use_tf=True, use_bias=layer.use_bias, num_calc_v=1, num_calc_u=1, first_nneg=True, ) if layer.use_bias: v, bias = utility.split_v_bias(v) updates.append(layer.bias.assign(bias)) updates.append(layer.kernel.assign(v)) a = tf.identity(_u) return tf.group(*updates)
def test_check_shape(self): print() a = tf.random_uniform((100, 50)) u = tf.random_uniform((100, 25)) v = tf.random_uniform((25, 50)) tf_u, tf_v = mf.semi_nmf(a, u, v, use_tf=True) print(tf_u, tf_v) assert u.shape == tf_u.shape assert v.shape == tf_v.shape
def minimize(self, loss=None): """Construct the control dependencies for calculating neural net optimized. Returns: tf.no_op. The import """ self._init(loss) if self._use_autoencoder: self._autoencoder() a = self.labels updates = [] # Reverse layers = self._layers[::-1] for i, layer in enumerate(layers): u = layer.output v = layer.kernel if layer.use_bias: v = tf.concat((v, layer.bias[None, ...]), axis=0) # Not use activation (ReLU) if not layer.activation: u, v = mf.semi_nmf( a=a, u=u, v=v, use_tf=True, use_bias=layer.use_bias, num_iters=1, first_nneg=True, ) # Use activation (ReLU) else: u, v = mf.nonlin_semi_nmf( a=a, u=u, v=v, use_tf=True, use_bias=layer.use_bias, num_calc_v=1, num_calc_u=1, first_nneg=True, ) if layer.use_bias: v, bias = utility.split_v_bias(v) updates.append(layer.bias.assign(bias)) updates.append(layer.kernel.assign(v)) a = tf.identity(u) return tf.group(*updates)
def test_np_u_neg_matlab_semi_nmf(self): a = np.random.uniform(size=(100, 100)) u = np.random.uniform(-1., 1., size=(100, 50)) v = np.random.uniform(0., 1., size=(50, 100)) old_loss = np_frobenius_norm(a, u @ v) start_time = time.time() u, v = semi_nmf(a, u, v, use_bias=False, data_format=False) assert np.min(u) < 0, np.min(u) assert np.min(v) > 0, np.min(v) end_time = time.time() duration = end_time - start_time new_loss = np_frobenius_norm(a, u @ v) assert a.shape == (u @ v).shape assert new_loss < old_loss, "new loss should be less than old loss." print_format('Numpy', 'semi-NMF', a, u, v, old_loss, new_loss, duration)
def test_np_vanilla_semi_nmf(self): auv = sio.loadmat(mat_file) a, u, v = auv['a'], auv['u'], auv['v'] old_loss = np_frobenius_norm(a, u @ v) start_time = time.time() u, v = semi_nmf(a, u, v, use_bias=False) end_time = time.time() duration = end_time - start_time new_loss = np_frobenius_norm(a, u @ v) assert a.shape == (u @ v).shape assert new_loss < old_loss, "new loss should be less than old loss." print('\n[Numpy]Solve semi-NMF\n\t' 'old loss {0}\n\t' 'new loss {1}\n\t' 'process duration {2}'.format(old_loss, new_loss, duration))
def test_np_biased_semi_nmf(self): a = np.random.uniform(-1., 1., size=(100, 100)) u = np.random.uniform(0., 1., size=(100, 300)) v = np.random.uniform(-1., 1., size=(300, 100)) old_loss = np_frobenius_norm(a, u @ v) bias_v = np.vstack((v, np.ones((1, v.shape[1])))) start_time = time.time() u, bias_v = semi_nmf(a, u, bias_v, use_bias=True) assert np.min(u) > 0, np.min(u) end_time = time.time() duration = end_time - start_time bias_u = np.hstack((u, np.ones((u.shape[0], 1)))) new_loss = np_frobenius_norm(a, bias_u @ bias_v) assert a.shape == (bias_u @ bias_v).shape assert new_loss < old_loss, "new loss should be less than old loss." print_format('Numpy', 'biased semi-NMF', a, bias_u, bias_v, old_loss, new_loss, duration)
def test_np_biased_semi_nmf(self): auv = sio.loadmat(mat_file) a, u, v = auv['a'], auv['u'], auv['v'] old_loss = np_frobenius_norm(a, u @ v) u = np.hstack((u, np.ones((u.shape[0], 1)))) start_time = time.time() u, v = semi_nmf(a, u, v, use_bias=True) end_time = time.time() duration = end_time - start_time bias_v = np.vstack((v, np.ones((1, v.shape[1])))) new_loss = np_frobenius_norm(a, u @ bias_v) assert a.shape == (u @ bias_v).shape assert new_loss < old_loss, "new loss should be less than old loss." print('\n[Numpy]Solve biased semi-NMF\n\t' 'old loss {0}\n\t' 'new loss {1}\n\t' 'process duration {2}'.format(old_loss, new_loss, duration))
def minimize(self, loss=None, pretrain=False): """Construct the control dependencies for calculating neural net optimized. Returns: tf.no_op. The import """ self._init(loss) # pre-train with auto encoder. pretrain_op = self._autoencoder() if pretrain else tf.no_op() a = self.labels updates = [] # Reverse layers = self._layers[::-1] for i, layer in enumerate(layers): u = layer.output v = layer.kernel if layer.use_bias: v = tf.concat((v, layer.bias[None, ...]), axis=0) # Not use activation (ReLU) if not layer.activation: u, v = mf.semi_nmf( a=a, u=u, v=v, use_tf=True, use_bias=layer.use_bias, num_iters=1, first_nneg=True, ) # Use activation (ReLU) elif utility.get_op_name(layer.activation) == 'Relu': u, v = mf.nonlin_semi_nmf( a=a, u=u, v=v, use_tf=True, use_bias=layer.use_bias, num_calc_v=1, num_calc_u=1, first_nneg=True, ) # Use Softmax elif utility.get_op_name(layer.activation) == 'Softmax': print('used softmax!!') u, v = mf.softmax_nmf( a=a, u=u, v=v, use_tf=True, use_bias=layer.use_bias, ) if layer.use_bias: v, bias = utility.split_v_bias(v) updates.append(layer.bias.assign(bias)) updates.append(layer.kernel.assign(v)) a = tf.identity(u) return AttrDict(ae=pretrain_op, nmf=tf.group(*updates))