def _setup(self, x, activation=ph.lrelu):
widget_list = list()
for layer in self._layers:
widget_list.append(layer)
widget_list.append(activation)
widget_list += [ph.flatten, self._fc, tf.nn.tanh]
y = ph.setup(x, widget_list)
return y
def _setup(self, x, dropout=None, activation=ph.swish):
s = activation
y = ph.setup(x, [
self._c1, s, dropout, self._c2, s, self._p1, self._c3, s, dropout,
self._c4, s, self._p2, self._c5, s, dropout, self._c6, s, dropout,
self._c7, s, self._p3, self._c8, s, dropout, self._c9, s, dropout,
self._c10, s, self._p4, self._c11, s, dropout, self._c12, s,
dropout, self._c13, s, self._p5, dropout, ph.flatten, self._h1, s,
dropout, self._h2, s, dropout, self._h3
])
return y
def _setup(self, x, activation=ph.lrelu, dropout=None):
widget_list = [
dropout,
self._fc, activation,
(tf.reshape, {'shape': (-1, *self._layers[-1].input_size)})
]
for layer in reversed(self._layers):
widget_list.append(layer)
widget_list.append(activation)
widget_list[-1] = tf.nn.tanh
y = ph.setup(x, widget_list)
return y
def _setup(self, x, name='out'):
return ph.setup(
x - self._mean,
[
self._conv1_1,
(tf.nn.relu, 'map1_1'), # 1
self._conv1_2,
(tf.nn.relu, 'map1_2'),
self._pool1, # 2
self._conv2_1,
(tf.nn.relu, 'map2_1'), # 3
self._conv2_2,
(tf.nn.relu, 'map2_2'),
self._pool2, # 4
self._conv3_1,
(tf.nn.relu, 'map3_1'), # 5
self._conv3_2,
(tf.nn.relu, 'map3_2'), # 6
self._conv3_3,
(tf.nn.relu, 'map3_3'), # 7
self._conv3_4,
(tf.nn.relu, 'map3_4'),
self._pool3, # 8
self._conv4_1,
(tf.nn.relu, 'map4_1'), # 9
self._conv4_2,
(tf.nn.relu, 'map4_2'), # 10
self._conv4_3,
(tf.nn.relu, 'map4_3'), # 11
self._conv4_4,
(tf.nn.relu, 'map4_4'),
self._pool4, # 12
self._conv5_1,
(tf.nn.relu, 'map5_1'), # 13
self._conv5_2,
(tf.nn.relu, 'map5_2'), # 14
self._conv5_3,
(tf.nn.relu, 'map5_3'), # 15
self._conv5_4,
(tf.nn.relu, 'map5_4'),
self._pool5, # 16
ph.ops.flatten,
self._fc6,
(tf.nn.relu, 'h6'), # 17
self._fc7,
(tf.nn.relu, 'h7'), # 18
self._fc8,
(tf.nn.softmax, name)
] # 19
)
def _setup(self, x):
return ph.setup(
x - self._mean, [
self._conv_1, self._activation, self._lrn, self._pool_1, 'map_1',
self._conv_2, self._activation, self._lrn, self._pool_2, 'map_2',
self._conv_3, self._activation, 'map_3',
self._conv_4, self._activation, 'map_4',
self._conv_5, self._activation, self._pool_5, 'map_5',
ph.ops.flatten, 'feature_5',
self._dense_6, self._activation, 'feature_6',
self._dense_7, self._activation, 'feature_7',
self._dense_8, tf.nn.softmax
]
)
def _setup(self, x):
h = ph.setup(
x,
[self._conv1, tf.nn.relu, self._lrn, self._pool1,
self._conv2, tf.nn.relu, self._lrn, self._pool2,
self._conv3, tf.nn.relu,
self._conv4, tf.nn.relu,
self._conv5, tf.nn.relu, self._pool5,
ph.ops.flatten,
self._fc6, tf.nn.relu,
self._fc7, tf.nn.relu]
)
y = self._fc8.setup(h)
y = tf.nn.softmax(y)
return y, h
def _setup(self, x):
batch_size, seq_len, voc_size = tf.shape(x)
y = ph.setup(tf.reshape(x, (batch_size, seq_len, voc_size, 1)), [
self._conv1,
self._pool1,
ph.ops.lrelu,
self._conv2,
self._pool2,
ph.ops.lrelu,
self._conv3,
self._pool3,
ph.ops.lrelu,
ph.ops.flatten,
self._dense1,
ph.ops.lrelu,
self._dense2,
tf.nn.sigmoid,
])
return y
def _build(self):
x = ph.placeholder('x', shape=(None, self._input_size), dtype=ph.float)
hidden_layer = ph.Linear('hidden_layer',
input_size=self._input_size,
output_size=self._hidden_size)
out_layer = ph.Linear('out_layer',
input_size=self._hidden_size,
output_size=self._num_classes)
dropout = ph.Dropout('dropout')
y = ph.setup(
x, [hidden_layer, ph.ops.lrelu, dropout, out_layer, tf.nn.softmax])
label = tf.argmax(y, axis=1)
self.predict = ph.Step(inputs=x,
outputs=(label, y),
givens={dropout.keep_prob: 1.0})
true_label = ph.placeholder('true_label', shape=(None, ), dtype=ph.int)
target = tf.one_hot(true_label, self._num_classes)
loss = ph.ops.cross_entropy(target, y)
loss = tf.reduce_mean(loss)
var_list = self.get_trainable_variables()
reg = ph.reg.L2Regularizer(1e-6)
reg.setup(var_list)
grad_list = [
tf.clip_by_value(grad, -10, 10)
for grad in tf.gradients(loss + reg.get_loss(), var_list)
]
lr = ph.train.ExponentialDecayedValue('lr_train',
1e-4,
num_loops=2e4,
min_value=1e-6)
update = tf.train.AdamOptimizer(lr.value).apply_gradients(
zip(grad_list, var_list))
self.train = ph.Step(inputs=(x, true_label),
outputs=loss,
updates=(update, lr.update_op),
givens={dropout.keep_prob: self._keep_prob})
def _build(self):
input_image = tf.placeholder(shape=(None, 784),
dtype=tf.float32,
name='input_image')
hidden_layer = ph.Linear('hidden_layer', 784, self._hidden_size)
output_layer = ph.Linear('output_layer', self._hidden_size, 10)
y = ph.setup(input_image,
[hidden_layer, ph.ops.lrelu, output_layer, tf.nn.softmax])
label = tf.argmax(y, 1)
input_label = tf.placeholder(shape=(None, ),
dtype=tf.int64,
name='input_label')
y_ = tf.one_hot(input_label, 10, dtype=tf.float32)
loss = ph.ops.cross_entropy(y_, y)
loss = tf.reduce_mean(loss)
self.train = ph.Step(inputs=(input_image, input_label),
outputs=loss,
updates=tf.train.RMSPropOptimizer(
1e-4, 0.9, 0.9).minimize(loss))
self.predict = ph.Step(inputs=input_image, outputs=label)
def _setup(self, x):
h = ph.setup(x, [
self._conv1, tf.nn.relu, self._lrn, self._pool1, self._conv2,
tf.nn.relu, self._lrn, self._pool2, self._conv3, tf.nn.relu,
self._conv4, tf.nn.relu, self._conv5, tf.nn.relu, self._pool5,
ph.flatten, self._fc6, tf.nn.relu, self._fc7, tf.nn.relu
])
y = self._fc8.setup(h)
y = tf.nn.softmax(y)
# conv1 = self._conv1.setup(x)
# conv1 = tf.nn.relu(conv1)
# norm1 = self._lrn(conv1, radius=2, alpha=1e-5, beta=0.75, name='norm1')
# pool1 = self._pool1.setup(norm1)
# #
# conv2 = self._conv2.setup(pool1)
# conv2 = tf.nn.relu(conv2)
# norm2 = self._lrn(conv2, radius=2, alpha=1e-5, beta=0.75, name='norm2')
# pool2 = self._pool2.setup(norm2)
# #
# conv3 = self._conv3.setup(pool2)
# conv3 = tf.nn.relu(conv3)
# conv4 = self._conv4.setup(conv3)
# conv4 = tf.nn.relu(conv4)
# conv5 = self._conv5.setup(conv4)
# conv5 = tf.nn.relu(conv5)
# pool5 = self._pool5.setup(conv5)
# #
# flattened = ph.flatten(pool5)
# fc6 = self._fc6.setup(flattened)
# fc6 = tf.nn.relu(fc6)
# # dropout6 = tf.nn.dropout(fc6, self._keep_prob)
# #
# fc7 = self._fc7.setup(fc6)
# fc7 = tf.nn.relu(fc7)
# # dropout7 = tf.nn.dropout(fc7, self._keep_prob)
# #
# fc8 = self._fc8.setup(fc7)
# y = tf.nn.softmax(fc8)
return y, h
def _build(self):
input_image = ph.placeholder('input_image',
(None, alexnet.HEIGHT, alexnet.WIDTH, 3),
ph.float)
encoder = alexnet.AlexNet('encoder', ph.ops.swish)
dropout = ph.Dropout('dropout')
dense = ph.Linear('dense', encoder['dense_7'].output_size,
self._hidden_size)
output_layer = ph.Linear('output_layer', dense.output_size,
self._num_classes + 1)
encoder.setup(input_image)
y = ph.setup(
encoder['feature_7'],
[dense, ph.ops.swish, dropout, output_layer, tf.nn.softmax])
label = tf.argmax(y, axis=1)
self.predict = ph.Step(inputs=input_image,
outputs=(label, y),
givens={dropout.keep_prob: 1.0})
input_label = ph.placeholder('input_label', (None, ), ph.int)
y_target = tf.one_hot(input_label, self._num_classes + 1)
loss = -ph.ops.log_likelihood(y_target, y)
loss = tf.reduce_mean(loss)
################################################################################
# pre-train
################################################################################
vars_new = [
*dense.get_trainable_variables(),
*output_layer.get_trainable_variables()
]
reg = ph.reg.L2Regularizer(self._reg)
reg.setup(vars_new)
lr = ph.train.ExponentialDecayedValue('lr_1',
init_value=self._learning_rate_1,
num_loops=self._num_loops_1,
min_value=self._learning_rate_1 /
10)
update_1 = tf.train.AdamOptimizer(lr.value).apply_gradients([
(tf.clip_by_value(g, -self._grad_clip, self._grad_clip), v)
for g, v in zip(tf.gradients(loss +
reg.get_loss(), vars_new), vars_new)
if g is not None
])
# with tf.control_dependencies([update_1]):
# update_2 = ph.train.L2Regularizer(self._reg).apply(vars_new)
self.train = ph.Step(inputs=(input_image, input_label),
outputs=(loss, lr.variable),
updates=update_1,
givens={dropout.keep_prob: self._keep_prob})
################################################################################
# fine tune
################################################################################
vars_all = self.get_trainable_variables()
reg = ph.reg.L2Regularizer(self._reg)
reg.setup(vars_all)
lr = ph.train.ExponentialDecayedValue('lr_2',
init_value=self._learning_rate_2,
num_loops=self._num_loops_2,
min_value=self._learning_rate_2 /
10)
update_1 = tf.train.AdamOptimizer(lr.value).apply_gradients([
(tf.clip_by_value(g, -self._grad_clip, self._grad_clip), v)
for g, v in zip(tf.gradients(loss +
reg.get_loss(), vars_all), vars_all)
if g is not None
])
# with tf.control_dependencies([update_1]):
# update_2 = ph.train.L2Regularizer(self._reg).apply(vars_all)
self.fine_tune = ph.Step(inputs=(input_image, input_label),
outputs=(loss, lr.variable),
updates=update_1,
givens={dropout.keep_prob: self._keep_prob})
def _build(self):
shared = Embedding('shared', self._wemb_size, 500, act)
specific = Embedding('specific', self._wemb_size, 500)
gate = ph.Gate('gate', (500, 500), 500)
lin = ph.Linear('lin', 500, 1000)
out = ph.Linear('out', 1000, 2)
stat = VectorStat('stat')
drop = ph.Dropout('drop')
#
seq = ph.placeholder('seq', (None, None, self._wemb_size))
h1, states1 = shared.setup(seq)
stat.setup(tf.reshape(seq, (-1, self._wemb_size), name='flat_seq'))
stat.setup(tf.reshape(states1, (-1, 500), name='flat_states'))
h2, _ = specific.setup(seq)
g = gate.setup(h1, h2)
h = g * h1 + (1.0 - g) * h2
y_pred = ph.setup(h, [drop, lin, ph.lrelu, drop, out, tf.nn.sigmoid])
y_pred_ = ph.setup(h1, [drop, lin, ph.lrelu, drop, out, tf.nn.sigmoid])
y_pred__ = ph.setup(h1,
[drop, lin, ph.lrelu, drop, out, tf.nn.sigmoid])
label_pred = tf.argmax(y_pred, 1)
label = ph.placeholder('label', (None, 2))
loss = tf.reduce_mean((y_pred - label)**2, axis=1)
loss += tf.reduce_mean((y_pred_ - label)**2, axis=1)
loss += tf.reduce_mean((y_pred__ - label)**2, axis=1)
loss_sum = tf.reduce_sum(loss)
loss_mean = tf.reduce_mean(loss)
#
correct = tf.cast(tf.equal(label_pred, tf.argmax(label, 1)), ph.D_TYPE)
correct_pos = correct * label[:, 1]
correct_neg = correct * label[:, 0]
hit_pos = tf.reduce_sum(correct_pos)
hit_neg = tf.reduce_sum(correct_neg)
pred_pos = tf.reduce_sum(label_pred)
pred_neg = tf.reduce_sum(1 - label_pred)
error = tf.reduce_sum(1 - correct)
#
reg = ph.Regularizer()
reg.add_l1(self.get_trainable_variables())
#
optimizer = MaskGrad(tf.train.RMSPropOptimizer(1e-4, 0.8, 0.9))
self._optimizer = optimizer
optimizer.add_mask(shared.cell.wz)
optimizer.add_mask(shared.cell.wr)
optimizer.add_mask(shared.cell.wh)
optimizer.add_mask(shared.cell.uz)
optimizer.add_mask(shared.cell.ur)
optimizer.add_mask(shared.cell.uh)
#
self._add_train_slot(inputs=(seq, label),
outputs={
'Loss': loss_mean,
'Norm': tf.norm(self.specific.cell.uz, 1)
},
updates=(optimizer.minimize(loss_mean +
reg.get_loss(2e-7)),
stat.updates),
givens={drop.keep_prob: 0.5})
self._add_validate_slot(inputs=(seq, label),
outputs={
'Loss': loss_sum,
'hit_pos': hit_pos * 100,
'hit_neg': hit_neg * 100,
'pred_pos': pred_pos * 100,
'pred_neg': pred_neg * 100,
'Error': error * 100,
},
givens={drop.keep_prob: 1.0})
def _loop(prev_state, x):
if self._emb_layer is None:
x = ph.setup(x, [self._emb_layer, activation])
state = self._cell.setup(x, prev_state)
return state
