def __init__(self,
input_dim=None,
output_dim=1,
init_path=None,
opt_algo='gd',
learning_rate=1e-2,
l2_weight=0,
random_seed=None):
init_vars = [('w', [input_dim, output_dim], 'tnormal', dtype),
('b', [output_dim], 'zero', dtype)]
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = tf.sparse_placeholder(dtype)
self.y = tf.placeholder(dtype)
self.vars = train_util.init_var_map(init_vars, init_path)
w = self.vars['w']
b = self.vars['b']
logits = tf.sparse_tensor_dense_matmul(self.X, w) + b
self.y_prob = tf.sigmoid(logits)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=self.y)) + \
l2_weight * tf.nn.l2_loss(w)
self.optimizer = train_util.get_optimizer(opt_algo, learning_rate,
self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def __init__(self,
input_dim=None,
output_dim=1,
factor_order=10,
init_path=None,
opt_algo='gd',
learning_rate=1e-2,
l2_w=0,
l2_v=0,
random_seed=None):
init_vars = [('w', [input_dim, output_dim], 'tnormal', dtype),
('v', [input_dim, factor_order], 'tnormal', dtype),
('b', [output_dim], 'zero', dtype)]
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = tf.sparse_placeholder(dtype)
self.y = tf.placeholder(dtype)
self.vars = train_util.init_var_map(init_vars, init_path)
w = self.vars['w']
v = self.vars['v']
b = self.vars['b']
X_square = tf.SparseTensor(self.X.indices,
tf.square(self.X.values), self.X.shape)
p = 0.5 * tf.reshape(
tf.reduce_sum(
tf.square(tf.sparse_tensor_dense_matmul(self.X, v)) -
tf.sparse_tensor_dense_matmul(X_square, tf.square(v)), 1),
[-1, output_dim])
logits = tf.sparse_tensor_dense_matmul(self.X, w) + b + p
self.y_prob = tf.sigmoid(logits)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=self.y)) + \
l2_w * tf.nn.l2_loss(w) + \
l2_v * tf.nn.l2_loss(v)
self.optimizer = train_util.get_optimizer(opt_algo, learning_rate,
self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def __init__(self,
layer_sizes=None,
layer_acts=None,
layer_keeps=None,
layer_l2=None,
kernel_l2=None,
init_path=None,
opt_algo='gd',
learning_rate=1e-2,
random_seed=None):
init_vars = []
num_inputs = len(layer_sizes[0])
factor_order = layer_sizes[1]
for i in range(num_inputs):
layer_input = layer_sizes[0][i]
layer_output = factor_order
init_vars.append(('w0_%d' % i, [layer_input,
layer_output], 'tnormal', dtype))
init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype))
init_vars.append(('w1', [num_inputs * factor_order,
layer_sizes[2]], 'tnormal', dtype))
init_vars.append(('k1', [factor_order * factor_order,
layer_sizes[2]], 'tnormal', dtype))
init_vars.append(('b1', [layer_sizes[2]], 'zero', dtype))
for i in range(2, len(layer_sizes) - 1):
layer_input = layer_sizes[i]
layer_output = layer_sizes[i + 1]
init_vars.append((
'w%d' % i,
[layer_input, layer_output],
'tnormal',
))
init_vars.append(('b%d' % i, [layer_output], 'zero', dtype))
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype)
for i in range(num_inputs)] #one hot feat per field
self.y = tf.placeholder(dtype)
self.vars = train_util.init_var_map(init_vars, init_path)
w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)
] #one hot to embedding transformer
b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)]
l = tf.nn.dropout(
train_util.activate(
tf.concat(
[ #concat to #field * emb_dim 39 * 10
tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) +
b0[i] # transform to embedding [#field, emb_dim] [39, 10]
for i in range(num_inputs)
],
1),
layer_acts[0]),
layer_keeps[0])
w1 = self.vars['w1']
k1 = self.vars['k1']
b1 = self.vars['b1']
z = tf.reduce_sum(tf.reshape(l, [-1, num_inputs, factor_order]),
1) # sum all embed of embed dim -> (emb_dim,)
p = tf.reshape(
tf.matmul(tf.reshape(z, [-1, factor_order, 1]),
tf.reshape(z, [-1, 1, factor_order])),
[-1, factor_order * factor_order])
l = tf.nn.dropout(
train_util.activate(
tf.matmul(l, w1) + tf.matmul(p, k1) + b1, layer_acts[1]),
layer_keeps[1])
for i in range(2, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
l = tf.nn.dropout(
train_util.activate(tf.matmul(l, wi) + bi, layer_acts[i]),
layer_keeps[i])
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l,
labels=self.y))
if layer_l2 is not None:
for i in range(num_inputs):
self.loss += layer_l2[0] * tf.nn.l2_loss(w0[i])
for i in range(1, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
# bi = self.vars['b%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
if kernel_l2 is not None:
self.loss += kernel_l2 * tf.nn.l2_loss(k1)
self.optimizer = train_util.get_optimizer(opt_algo, learning_rate,
self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def __init__(self,
layer_sizes,
embd_size=50,
opt_algo="gd",
reg_rate=0.05,
learning_rate=0.03,
random_seed=0,
log_exp_product=False,
init_path=None):
feature_sizes = layer_sizes[0]
init_vars = []
init_vars.append(('b0', [feature_sizes[0],
1], 'zero', dtype)) #zero no problem?
init_vars.append(('b1', [feature_sizes[1], 1], 'zero', dtype))
init_vars.append(('bg', [1, 1], 'zero', dtype))
init_vars.append(('w0', [feature_sizes[0],
embd_size], 'tnormal', dtype))
init_vars.append(('w1', [feature_sizes[1],
embd_size], 'tnormal', dtype))
if log_exp_product:
init_vars.append(('w2', [2 * embd_size, 1], 'tnormal', dtype))
self.graph = tf.Graph()
with self.graph.as_default():
self.X = [
tf.sparse_placeholder(dtype),
tf.sparse_placeholder(dtype)
]
self.y = tf.placeholder(dtype)
self.vars = train_util.init_var_map(init_vars, init_path=init_path)
w0 = self.vars['w0']
w1 = self.vars['w1']
b0 = self.vars['b0']
b1 = self.vars['b1']
bg = self.vars['bg']
embd0 = tf.sparse_tensor_dense_matmul(self.X[0], w0)
embd1 = tf.sparse_tensor_dense_matmul(self.X[1], w1)
bias0 = tf.sparse_tensor_dense_matmul(self.X[0], b0)
bias1 = tf.sparse_tensor_dense_matmul(self.X[1], b1)
if log_exp_product:
w2 = self.vars['w2']
self.y_prob = tf.exp(
tf.matmul(tf.concat([embd0, embd1], 1), w2))
else:
self.y_prob = tf.reshape(
tf.reduce_sum(tf.multiply(embd0, embd1), 1), [-1, 1])
print self.y_prob
self.y_prob = tf.add(self.y_prob, bg)
print self.y_prob
self.y_prob = tf.add(self.y_prob, bias0)
print self.y_prob
self.y_prob = tf.add(self.y_prob, bias1)
print self.y_prob
regularizer = tf.add(tf.nn.l2_loss(embd0),
tf.nn.l2_loss(embd1),
name='svd_regularizer')
self.loss = tf.sqrt(tf.nn.l2_loss(tf.subtract(self.y_prob,
self.y)))
penalty = tf.constant(reg_rate,
dtype=tf.float32,
shape=[],
name="l2")
self.loss = tf.add(self.loss, tf.multiply(regularizer, penalty))
self.optimizer = train_util.get_optimizer(opt_algo, learning_rate,
self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def __init__(self,
layer_sizes=None,
layer_acts=None,
layer_keeps=None,
short_cuts=None,
layer_l2=None,
kernel_l2=None,
init_path=None,
opt_algo='gd',
learning_rate=1e-2,
random_seed=None,
p_mode=1,
add_svd_score=False,
init_with_normal_one=True,
drop_embedding_layer=False,
add_u_auto_rec=False,
add_i_auto_rec=False,
prev_item_vec_cnt=0,
add_user_product_bias=True,
init_with_user_product_one=False):
init_vars = []
num_inputs = len(layer_sizes[0])
factor_order = layer_sizes[1]
for i in range(num_inputs):
layer_input = layer_sizes[0][i]
layer_output = factor_order
init_vars.append(('w0_%d' % i, [layer_input,
layer_output], 'tnormal', dtype))
init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype))
#add score bias for fields
user_item_fields = [0, 1]
for i in user_item_fields:
field_size = layer_sizes[0][i]
init_vars.append(('field_score_b_%d' % i, [field_size,
1], 'tnormal', dtype))
init_vars.append(('w1', [num_inputs * factor_order,
layer_sizes[2]], 'tnormal', dtype))
init_type = (
'normal_one' if not init_with_user_product_one else
"tnormal_user_product_one") if init_with_normal_one else 'tnormal'
if p_mode == 0:
init_vars.append(('k1', [num_inputs,
layer_sizes[2]], init_type, dtype))
elif p_mode == 1:
init_vars.append(('k1', [num_inputs * num_inputs,
layer_sizes[2]], init_type, dtype))
else:
init_vars.append(('k1', [num_inputs,
layer_sizes[2]], init_type, dtype))
init_vars.append(('b1', [layer_sizes[2]], 'zero', dtype))
for i in range(2, len(layer_sizes) - 1):
layer_input = layer_sizes[i]
layer_output = layer_sizes[i + 1]
init_vars.append(
('w%d' % i, [layer_input, layer_output
], ('normal_one' if not init_with_user_product_one
else "hidden_tnormal_user_product_one")
if init_with_normal_one else 'tnormal',
dtype)) #critical modify tnormal_zero to normal_one
init_vars.append(('b%d' % i, [layer_output], 'zero', dtype))
#AutoRec
user_cnt = layer_sizes[0][0]
item_cnt = layer_sizes[0][1]
if add_u_auto_rec:
scale = math.sqrt(6.0 / (user_cnt + layer_sizes[1]))
WU0 = tf.Variable(
tf.random_uniform([user_cnt, layer_sizes[1]], -scale, scale))
bU0 = tf.Variable(
tf.random_uniform([layer_sizes[1]], -scale, scale))
WU1 = tf.Variable(
tf.random_uniform([layer_sizes[1], user_cnt], -scale, scale))
bU = tf.Variable(tf.random_uniform([user_cnt], -scale, scale))
if add_i_auto_rec:
scale = math.sqrt(6.0 / (item_cnt + layer_sizes[1]))
WI0 = tf.Variable(
tf.random_uniform([item_cnt, layer_sizes[1]], -scale, scale))
bI0 = tf.Variable(
tf.random_uniform([layer_sizes[1]], -scale, scale))
WI1 = tf.Variable(
tf.random_uniform([layer_sizes[1], item_cnt], -scale, scale))
bI = tf.Variable(tf.random_uniform([item_cnt], -scale, scale))
self.graph = tf.Graph()
with self.graph.as_default():
layers = []
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.vars = train_util.init_var_map(init_vars, init_path)
w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)]
b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)]
#transform to embedding
embd_vecs = [
tf.div(
tf.sparse_tensor_dense_matmul(self.X[i], w0[i]),
tf.clip_by_value(
tf.reshape(tf.sparse_reduce_sum(self.X[i], 1),
(-1, 1)), 1.0, 999999.0)) + b0[i]
for i in range(num_inputs - prev_item_vec_cnt)
]
#embd_vecs = [tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) + b0[i] for i in range(num_inputs-prev_item_vec_cnt)]
if prev_item_vec_cnt != 0:
embd_vecs.extend([
tf.div(
tf.sparse_tensor_dense_matmul(self.X[i], w0[i]),
tf.clip_by_value(
tf.reshape(tf.sparse_reduce_sum(self.X[i], 1),
(-1, 1)), 1.0, 999999.0) + b0[i])
for i in range(num_inputs)[-prev_item_vec_cnt:]
])
l = tf.nn.dropout(
train_util.activate(tf.concat(embd_vecs, 1), layer_acts[0]),
layer_keeps[0])
layers.append(l)
w1 = self.vars['w1']
k1 = self.vars['k1']
b1 = self.vars['b1']
if p_mode == 0:
feat_emb_mat = tf.reshape(l, [-1, num_inputs, factor_order])
feat1_emb_mat = tf.slice(feat_emb_mat, [0, 0, 0],
[-1, 1, factor_order])
feat2_emb_mat = tf.slice(feat_emb_mat, [0, 1, 0],
[-1, 1, factor_order])
p = tf.reduce_sum(
tf.reshape(
tf.matmul(
tf.reshape(tf.transpose(feat_emb_mat, [0, 2, 1]),
[-1, num_inputs]), k1),
[-1, factor_order, layer_sizes[2]]), 1)
elif p_mode == 1:
#slower inner product
feat_emb_mat = tf.reshape(l, [-1, num_inputs, factor_order])
feat1_emb_mat = tf.slice(feat_emb_mat, [0, 0, 0],
[-1, 1, factor_order])
feat2_emb_mat = tf.slice(feat_emb_mat, [0, 1, 0],
[-1, 1, factor_order])
feat_sim_vec = tf.reshape(
tf.matmul(feat_emb_mat,
tf.transpose(feat_emb_mat, [0, 2, 1])),
[-1, num_inputs * num_inputs])
p = tf.matmul(feat_sim_vec, k1)
else:
feat_emb_mat = tf.reshape(l, [-1, num_inputs, factor_order])
feat1_emb_mat = tf.slice(feat_emb_mat, [0, 0, 0],
[-1, 1, factor_order])
feat2_emb_mat = tf.slice(feat_emb_mat, [0, 1, 0],
[-1, 1, factor_order])
p = tf.reduce_sum(
tf.reshape(
tf.matmul(
tf.reshape(
tf.transpose(
tf.reshape(l,
[-1, num_inputs, factor_order]),
[0, 2, 1]), [-1, num_inputs]), k1),
[-1, factor_order, layer_sizes[2]]), 1)
#TODO
#svd_score = tf.reduce_sum(tf.multiply(tf.reshape(feat1_emb_mat,[-1,factor_order]), tf.reshape(feat2_emb_mat,[-1,factor_order])), 1)
svd_score = tf.matmul(feat1_emb_mat,
tf.transpose(feat2_emb_mat, (0, 2, 1)))
svd_score = tf.reshape(svd_score, [-1, 1])
print "svd_score: ", svd_score
embed_layer = tf.matmul(l, w1) + b1 + p
l = tf.nn.dropout(train_util.activate(embed_layer, layer_acts[1]),
layer_keeps[1])
layers.append(l)
for i in range(2, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
l = tf.nn.dropout(
train_util.activate(
tf.matmul(l, wi) +
bi if short_cuts[i] is None else tf.matmul(l, wi) +
bi + layers[short_cuts[i]], layer_acts[i]),
layer_keeps[i])
layers.append(l)
print i, layers
self.score_bias = tf.zeros([1, 1])
if add_user_product_bias:
for i in user_item_fields:
self.score_bias = tf.add(
self.score_bias,
tf.sparse_tensor_dense_matmul(
self.X[i], self.vars['field_score_b_%d' % i]))
if add_svd_score:
if drop_embedding_layer:
self.y_prob = tf.add(self.score_bias, svd_score)
else:
self.y_prob = tf.add(tf.add(l, self.score_bias), svd_score)
else:
self.y_prob = tf.add(l, self.score_bias)
print "y_prob: ", self.y_prob
self.loss = tf.sqrt(
tf.reduce_mean(tf.square(tf.subtract(self.y, self.y_prob))))
#TODO to test down side one is bes
#Problem is in the learning rate!!!
#self.loss = (tf.nn.l2_loss(tf.subtract(self.y,self.y_prob))/500.0)
if layer_l2 is not None:
for i in range(num_inputs):
self.loss += layer_l2[0] * tf.nn.l2_loss(w0[i])
if not drop_embedding_layer:
for i in range(1, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
# bi = self.vars['b%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
if kernel_l2 is not None:
if not drop_embedding_layer:
self.loss += kernel_l2 * tf.nn.l2_loss(k1)
self.optimizer = train_util.get_optimizer(opt_algo, learning_rate,
self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def __init__(self,
layer_sizes=None,
layer_acts=None,
layer_keeps=None,
short_cuts=None,
layer_l2=None,
kernel_l2=None,
init_path=None,
opt_algo='gd',
learning_rate=1e-2,
random_seed=None,
p_mode=0):
init_vars = []
num_inputs = len(layer_sizes[0])
factor_order = layer_sizes[1]
for i in range(num_inputs):
layer_input = layer_sizes[0][i]
layer_output = factor_order
init_vars.append(('w0_%d' % i, [layer_input,
layer_output], 'tnormal', dtype))
init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype))
init_vars.append(('w1', [num_inputs * factor_order,
layer_sizes[2]], 'tnormal', dtype))
print "Running in p_mode %d" % p_mode
if p_mode == 0:
init_vars.append(('k1', [num_inputs,
layer_sizes[2]], 'tnormal', dtype))
elif p_mode == 1:
init_vars.append(('k1', [num_inputs * num_inputs,
layer_sizes[2]], 'tnormal', dtype))
else:
init_vars.append(('k1', [num_inputs,
layer_sizes[2]], 'tnormal', dtype))
init_vars.append(('b1', [layer_sizes[2]], 'zero', dtype))
for i in range(2, len(layer_sizes) - 1):
layer_input = layer_sizes[i]
layer_output = layer_sizes[i + 1]
init_vars.append(('w%d' % i, [layer_input,
layer_output], 'tnormal', dtype))
init_vars.append(('b%d' % i, [layer_output], 'zero', dtype))
self.graph = tf.Graph()
with self.graph.as_default():
layers = []
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.vars = train_util.init_var_map(init_vars, init_path)
w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)]
b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)]
l = tf.nn.dropout(
train_util.activate(
tf.concat(
[
tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) +
b0[i] #transform to embedding
for i in range(num_inputs)
],
1),
layer_acts[0]),
layer_keeps[0])
layers.append(l)
w1 = self.vars['w1']
k1 = self.vars['k1']
b1 = self.vars['b1']
if p_mode == 0:
p = tf.reduce_sum(
tf.reshape(
tf.matmul(
tf.reshape(
tf.transpose(
tf.reshape(l,
[-1, num_inputs, factor_order]),
[0, 2, 1]), [-1, num_inputs]), k1),
[-1, factor_order, layer_sizes[2]]), 1)
elif p_mode == 1:
#slower inner product
feat_emb_mat = tf.reshape(l, [-1, num_inputs, factor_order])
feat_sim_vec = tf.reshape(
tf.matmul(feat_emb_mat,
tf.transpose(feat_emb_mat, [0, 2, 1])),
[-1, num_inputs * num_inputs])
p = tf.matmul(feat_sim_vec, k1)
else:
p = tf.reduce_sum(
tf.reshape(
tf.matmul(
tf.reshape(
tf.transpose(
tf.reshape(l,
[-1, num_inputs, factor_order]),
[0, 2, 1]), [-1, num_inputs]), k1),
[-1, factor_order, layer_sizes[2]]), 1)
l = tf.nn.dropout(
train_util.activate(tf.matmul(l, w1) + b1 + p, layer_acts[1]),
layer_keeps[1])
layers.append(l)
for i in range(2, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
l = tf.nn.dropout(
train_util.activate(
tf.matmul(l, wi) +
bi if short_cuts[i] is None else tf.matmul(l, wi) +
bi + layers[short_cuts[i]], layer_acts[i]),
layer_keeps[i])
layers.append(l)
print i, layers
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l,
labels=self.y))
if layer_l2 is not None:
for i in range(num_inputs):
self.loss += layer_l2[0] * tf.nn.l2_loss(w0[i])
for i in range(1, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
# bi = self.vars['b%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
if kernel_l2 is not None:
self.loss += kernel_l2 * tf.nn.l2_loss(k1)
self.optimizer = train_util.get_optimizer(opt_algo, learning_rate,
self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def __init__(self,
layer_sizes=None,
layer_acts=None,
layer_keeps=None,
init_path=None,
opt_algo='gd',
learning_rate=1e-2,
random_seed=None):
init_vars = []
num_inputs = len(layer_sizes[0])
embedding_order = layer_sizes[1]
for i in range(num_inputs):
layer_input = layer_sizes[0][i]
layer_output = embedding_order
init_vars.append(('w0_%d' % i, [layer_input,
layer_output], 'tnormal', dtype))
init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype))
init_vars.append(('f1', [embedding_order, layer_sizes[2], 1,
2], 'tnormal', dtype))
init_vars.append(('f2', [embedding_order, layer_sizes[3], 2,
2], 'tnormal', dtype))
init_vars.append(('w1', [2 * 3 * embedding_order,
1], 'tnormal', dtype))
init_vars.append(('b1', [1], 'zero', dtype))
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.vars = train_util.init_var_map(init_vars, init_path)
w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)]
b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)]
l = tf.nn.dropout(
train_util.activate(
tf.concat([
tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) + b0[i]
for i in range(num_inputs)
], 1), layer_acts[0]), layer_keeps[0])
l = tf.transpose(
tf.reshape(l, [-1, num_inputs, embedding_order, 1]),
[0, 2, 1, 3])
f1 = self.vars['f1']
l = tf.nn.conv2d(l, f1, [1, 1, 1, 1], 'SAME')
l = tf.transpose(
train_util.max_pool_4d(tf.transpose(l, [0, 1, 3, 2]),
num_inputs / 2), [0, 1, 3, 2])
f2 = self.vars['f2']
l = tf.nn.conv2d(l, f2, [1, 1, 1, 1], 'SAME')
l = tf.transpose(
train_util.max_pool_4d(tf.transpose(l, [0, 1, 3, 2]), 3),
[0, 1, 3, 2])
l = tf.nn.dropout(
train_util.activate(
tf.reshape(l, [-1, embedding_order * 3 * 2]),
layer_acts[1]), layer_keeps[1])
w1 = self.vars['w1']
b1 = self.vars['b1']
l = tf.nn.dropout(
train_util.activate(tf.matmul(l, w1) + b1, layer_acts[2]),
layer_keeps[2])
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l,
labels=self.y))
self.optimizer = train_util.get_optimizer(opt_algo, learning_rate,
self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def __init__(self,
layer_sizes=None,
layer_acts=None,
layer_keeps=None,
layer_l2=None,
init_path=None,
opt_algo='gd',
learning_rate=1e-2,
random_seed=None):
init_vars = []
num_inputs = len(layer_sizes[0])
factor_order = layer_sizes[1]
for i in range(num_inputs):
layer_input = layer_sizes[0][i]
layer_output = factor_order
init_vars.append(('w0_%d' % i, [layer_input,
layer_output], 'tnormal', dtype))
init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype))
init_vars.append(('w1', [num_inputs * factor_order,
layer_sizes[2]], 'tnormal', dtype))
init_vars.append(('b1', [layer_sizes[2]], 'zero', dtype))
for i in range(2, len(layer_sizes) - 1):
layer_input = layer_sizes[i]
layer_output = layer_sizes[i + 1]
init_vars.append(('w%d' % i, [layer_input,
layer_output], 'tnormal', dtype))
init_vars.append(('b%d' % i, [layer_output], 'zero', dtype))
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.vars = train_util.init_var_map(init_vars, init_path)
w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)]
b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)]
l = tf.nn.dropout(
train_util.activate(
tf.concat([
tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) + b0[i]
for i in range(num_inputs)
], 1), layer_acts[0]), layer_keeps[0])
for i in range(1, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
l = tf.nn.dropout(
train_util.activate(tf.matmul(l, wi) + bi, layer_acts[i]),
layer_keeps[i])
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l,
labels=self.y))
if layer_l2 is not None:
for i in range(num_inputs):
self.loss += layer_l2[0] * tf.nn.l2_loss(w0[i])
for i in range(1, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
# bi = self.vars['b%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
self.optimizer = train_util.get_optimizer(opt_algo, learning_rate,
self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)