def _setup(self, x, activation=ph.lrelu): widget_list = list() for layer in self._layers: widget_list.append(layer) widget_list.append(activation) widget_list += [ph.flatten, self._fc, tf.nn.tanh] y = ph.setup(x, widget_list) return y
def _setup(self, x, dropout=None, activation=ph.swish): s = activation y = ph.setup(x, [ self._c1, s, dropout, self._c2, s, self._p1, self._c3, s, dropout, self._c4, s, self._p2, self._c5, s, dropout, self._c6, s, dropout, self._c7, s, self._p3, self._c8, s, dropout, self._c9, s, dropout, self._c10, s, self._p4, self._c11, s, dropout, self._c12, s, dropout, self._c13, s, self._p5, dropout, ph.flatten, self._h1, s, dropout, self._h2, s, dropout, self._h3 ]) return y
def _setup(self, x, activation=ph.lrelu, dropout=None): widget_list = [ dropout, self._fc, activation, (tf.reshape, {'shape': (-1, *self._layers[-1].input_size)}) ] for layer in reversed(self._layers): widget_list.append(layer) widget_list.append(activation) widget_list[-1] = tf.nn.tanh y = ph.setup(x, widget_list) return y
def _setup(self, x, name='out'): return ph.setup( x - self._mean, [ self._conv1_1, (tf.nn.relu, 'map1_1'), # 1 self._conv1_2, (tf.nn.relu, 'map1_2'), self._pool1, # 2 self._conv2_1, (tf.nn.relu, 'map2_1'), # 3 self._conv2_2, (tf.nn.relu, 'map2_2'), self._pool2, # 4 self._conv3_1, (tf.nn.relu, 'map3_1'), # 5 self._conv3_2, (tf.nn.relu, 'map3_2'), # 6 self._conv3_3, (tf.nn.relu, 'map3_3'), # 7 self._conv3_4, (tf.nn.relu, 'map3_4'), self._pool3, # 8 self._conv4_1, (tf.nn.relu, 'map4_1'), # 9 self._conv4_2, (tf.nn.relu, 'map4_2'), # 10 self._conv4_3, (tf.nn.relu, 'map4_3'), # 11 self._conv4_4, (tf.nn.relu, 'map4_4'), self._pool4, # 12 self._conv5_1, (tf.nn.relu, 'map5_1'), # 13 self._conv5_2, (tf.nn.relu, 'map5_2'), # 14 self._conv5_3, (tf.nn.relu, 'map5_3'), # 15 self._conv5_4, (tf.nn.relu, 'map5_4'), self._pool5, # 16 ph.ops.flatten, self._fc6, (tf.nn.relu, 'h6'), # 17 self._fc7, (tf.nn.relu, 'h7'), # 18 self._fc8, (tf.nn.softmax, name) ] # 19 )
def _setup(self, x): return ph.setup( x - self._mean, [ self._conv_1, self._activation, self._lrn, self._pool_1, 'map_1', self._conv_2, self._activation, self._lrn, self._pool_2, 'map_2', self._conv_3, self._activation, 'map_3', self._conv_4, self._activation, 'map_4', self._conv_5, self._activation, self._pool_5, 'map_5', ph.ops.flatten, 'feature_5', self._dense_6, self._activation, 'feature_6', self._dense_7, self._activation, 'feature_7', self._dense_8, tf.nn.softmax ] )
def _setup(self, x): h = ph.setup( x, [self._conv1, tf.nn.relu, self._lrn, self._pool1, self._conv2, tf.nn.relu, self._lrn, self._pool2, self._conv3, tf.nn.relu, self._conv4, tf.nn.relu, self._conv5, tf.nn.relu, self._pool5, ph.ops.flatten, self._fc6, tf.nn.relu, self._fc7, tf.nn.relu] ) y = self._fc8.setup(h) y = tf.nn.softmax(y) return y, h
def _setup(self, x): batch_size, seq_len, voc_size = tf.shape(x) y = ph.setup(tf.reshape(x, (batch_size, seq_len, voc_size, 1)), [ self._conv1, self._pool1, ph.ops.lrelu, self._conv2, self._pool2, ph.ops.lrelu, self._conv3, self._pool3, ph.ops.lrelu, ph.ops.flatten, self._dense1, ph.ops.lrelu, self._dense2, tf.nn.sigmoid, ]) return y
def _build(self): x = ph.placeholder('x', shape=(None, self._input_size), dtype=ph.float) hidden_layer = ph.Linear('hidden_layer', input_size=self._input_size, output_size=self._hidden_size) out_layer = ph.Linear('out_layer', input_size=self._hidden_size, output_size=self._num_classes) dropout = ph.Dropout('dropout') y = ph.setup( x, [hidden_layer, ph.ops.lrelu, dropout, out_layer, tf.nn.softmax]) label = tf.argmax(y, axis=1) self.predict = ph.Step(inputs=x, outputs=(label, y), givens={dropout.keep_prob: 1.0}) true_label = ph.placeholder('true_label', shape=(None, ), dtype=ph.int) target = tf.one_hot(true_label, self._num_classes) loss = ph.ops.cross_entropy(target, y) loss = tf.reduce_mean(loss) var_list = self.get_trainable_variables() reg = ph.reg.L2Regularizer(1e-6) reg.setup(var_list) grad_list = [ tf.clip_by_value(grad, -10, 10) for grad in tf.gradients(loss + reg.get_loss(), var_list) ] lr = ph.train.ExponentialDecayedValue('lr_train', 1e-4, num_loops=2e4, min_value=1e-6) update = tf.train.AdamOptimizer(lr.value).apply_gradients( zip(grad_list, var_list)) self.train = ph.Step(inputs=(x, true_label), outputs=loss, updates=(update, lr.update_op), givens={dropout.keep_prob: self._keep_prob})
def _build(self): input_image = tf.placeholder(shape=(None, 784), dtype=tf.float32, name='input_image') hidden_layer = ph.Linear('hidden_layer', 784, self._hidden_size) output_layer = ph.Linear('output_layer', self._hidden_size, 10) y = ph.setup(input_image, [hidden_layer, ph.ops.lrelu, output_layer, tf.nn.softmax]) label = tf.argmax(y, 1) input_label = tf.placeholder(shape=(None, ), dtype=tf.int64, name='input_label') y_ = tf.one_hot(input_label, 10, dtype=tf.float32) loss = ph.ops.cross_entropy(y_, y) loss = tf.reduce_mean(loss) self.train = ph.Step(inputs=(input_image, input_label), outputs=loss, updates=tf.train.RMSPropOptimizer( 1e-4, 0.9, 0.9).minimize(loss)) self.predict = ph.Step(inputs=input_image, outputs=label)
def _setup(self, x): h = ph.setup(x, [ self._conv1, tf.nn.relu, self._lrn, self._pool1, self._conv2, tf.nn.relu, self._lrn, self._pool2, self._conv3, tf.nn.relu, self._conv4, tf.nn.relu, self._conv5, tf.nn.relu, self._pool5, ph.flatten, self._fc6, tf.nn.relu, self._fc7, tf.nn.relu ]) y = self._fc8.setup(h) y = tf.nn.softmax(y) # conv1 = self._conv1.setup(x) # conv1 = tf.nn.relu(conv1) # norm1 = self._lrn(conv1, radius=2, alpha=1e-5, beta=0.75, name='norm1') # pool1 = self._pool1.setup(norm1) # # # conv2 = self._conv2.setup(pool1) # conv2 = tf.nn.relu(conv2) # norm2 = self._lrn(conv2, radius=2, alpha=1e-5, beta=0.75, name='norm2') # pool2 = self._pool2.setup(norm2) # # # conv3 = self._conv3.setup(pool2) # conv3 = tf.nn.relu(conv3) # conv4 = self._conv4.setup(conv3) # conv4 = tf.nn.relu(conv4) # conv5 = self._conv5.setup(conv4) # conv5 = tf.nn.relu(conv5) # pool5 = self._pool5.setup(conv5) # # # flattened = ph.flatten(pool5) # fc6 = self._fc6.setup(flattened) # fc6 = tf.nn.relu(fc6) # # dropout6 = tf.nn.dropout(fc6, self._keep_prob) # # # fc7 = self._fc7.setup(fc6) # fc7 = tf.nn.relu(fc7) # # dropout7 = tf.nn.dropout(fc7, self._keep_prob) # # # fc8 = self._fc8.setup(fc7) # y = tf.nn.softmax(fc8) return y, h
def _build(self): input_image = ph.placeholder('input_image', (None, alexnet.HEIGHT, alexnet.WIDTH, 3), ph.float) encoder = alexnet.AlexNet('encoder', ph.ops.swish) dropout = ph.Dropout('dropout') dense = ph.Linear('dense', encoder['dense_7'].output_size, self._hidden_size) output_layer = ph.Linear('output_layer', dense.output_size, self._num_classes + 1) encoder.setup(input_image) y = ph.setup( encoder['feature_7'], [dense, ph.ops.swish, dropout, output_layer, tf.nn.softmax]) label = tf.argmax(y, axis=1) self.predict = ph.Step(inputs=input_image, outputs=(label, y), givens={dropout.keep_prob: 1.0}) input_label = ph.placeholder('input_label', (None, ), ph.int) y_target = tf.one_hot(input_label, self._num_classes + 1) loss = -ph.ops.log_likelihood(y_target, y) loss = tf.reduce_mean(loss) ################################################################################ # pre-train ################################################################################ vars_new = [ *dense.get_trainable_variables(), *output_layer.get_trainable_variables() ] reg = ph.reg.L2Regularizer(self._reg) reg.setup(vars_new) lr = ph.train.ExponentialDecayedValue('lr_1', init_value=self._learning_rate_1, num_loops=self._num_loops_1, min_value=self._learning_rate_1 / 10) update_1 = tf.train.AdamOptimizer(lr.value).apply_gradients([ (tf.clip_by_value(g, -self._grad_clip, self._grad_clip), v) for g, v in zip(tf.gradients(loss + reg.get_loss(), vars_new), vars_new) if g is not None ]) # with tf.control_dependencies([update_1]): # update_2 = ph.train.L2Regularizer(self._reg).apply(vars_new) self.train = ph.Step(inputs=(input_image, input_label), outputs=(loss, lr.variable), updates=update_1, givens={dropout.keep_prob: self._keep_prob}) ################################################################################ # fine tune ################################################################################ vars_all = self.get_trainable_variables() reg = ph.reg.L2Regularizer(self._reg) reg.setup(vars_all) lr = ph.train.ExponentialDecayedValue('lr_2', init_value=self._learning_rate_2, num_loops=self._num_loops_2, min_value=self._learning_rate_2 / 10) update_1 = tf.train.AdamOptimizer(lr.value).apply_gradients([ (tf.clip_by_value(g, -self._grad_clip, self._grad_clip), v) for g, v in zip(tf.gradients(loss + reg.get_loss(), vars_all), vars_all) if g is not None ]) # with tf.control_dependencies([update_1]): # update_2 = ph.train.L2Regularizer(self._reg).apply(vars_all) self.fine_tune = ph.Step(inputs=(input_image, input_label), outputs=(loss, lr.variable), updates=update_1, givens={dropout.keep_prob: self._keep_prob})
def _build(self): shared = Embedding('shared', self._wemb_size, 500, act) specific = Embedding('specific', self._wemb_size, 500) gate = ph.Gate('gate', (500, 500), 500) lin = ph.Linear('lin', 500, 1000) out = ph.Linear('out', 1000, 2) stat = VectorStat('stat') drop = ph.Dropout('drop') # seq = ph.placeholder('seq', (None, None, self._wemb_size)) h1, states1 = shared.setup(seq) stat.setup(tf.reshape(seq, (-1, self._wemb_size), name='flat_seq')) stat.setup(tf.reshape(states1, (-1, 500), name='flat_states')) h2, _ = specific.setup(seq) g = gate.setup(h1, h2) h = g * h1 + (1.0 - g) * h2 y_pred = ph.setup(h, [drop, lin, ph.lrelu, drop, out, tf.nn.sigmoid]) y_pred_ = ph.setup(h1, [drop, lin, ph.lrelu, drop, out, tf.nn.sigmoid]) y_pred__ = ph.setup(h1, [drop, lin, ph.lrelu, drop, out, tf.nn.sigmoid]) label_pred = tf.argmax(y_pred, 1) label = ph.placeholder('label', (None, 2)) loss = tf.reduce_mean((y_pred - label)**2, axis=1) loss += tf.reduce_mean((y_pred_ - label)**2, axis=1) loss += tf.reduce_mean((y_pred__ - label)**2, axis=1) loss_sum = tf.reduce_sum(loss) loss_mean = tf.reduce_mean(loss) # correct = tf.cast(tf.equal(label_pred, tf.argmax(label, 1)), ph.D_TYPE) correct_pos = correct * label[:, 1] correct_neg = correct * label[:, 0] hit_pos = tf.reduce_sum(correct_pos) hit_neg = tf.reduce_sum(correct_neg) pred_pos = tf.reduce_sum(label_pred) pred_neg = tf.reduce_sum(1 - label_pred) error = tf.reduce_sum(1 - correct) # reg = ph.Regularizer() reg.add_l1(self.get_trainable_variables()) # optimizer = MaskGrad(tf.train.RMSPropOptimizer(1e-4, 0.8, 0.9)) self._optimizer = optimizer optimizer.add_mask(shared.cell.wz) optimizer.add_mask(shared.cell.wr) optimizer.add_mask(shared.cell.wh) optimizer.add_mask(shared.cell.uz) optimizer.add_mask(shared.cell.ur) optimizer.add_mask(shared.cell.uh) # self._add_train_slot(inputs=(seq, label), outputs={ 'Loss': loss_mean, 'Norm': tf.norm(self.specific.cell.uz, 1) }, updates=(optimizer.minimize(loss_mean + reg.get_loss(2e-7)), stat.updates), givens={drop.keep_prob: 0.5}) self._add_validate_slot(inputs=(seq, label), outputs={ 'Loss': loss_sum, 'hit_pos': hit_pos * 100, 'hit_neg': hit_neg * 100, 'pred_pos': pred_pos * 100, 'pred_neg': pred_neg * 100, 'Error': error * 100, }, givens={drop.keep_prob: 1.0})
def _loop(prev_state, x): if self._emb_layer is None: x = ph.setup(x, [self._emb_layer, activation]) state = self._cell.setup(x, prev_state) return state