def make_app():
app.json_encoder = utils.APIJSONEncoder
app.config["CASE_INSENSITIVE"] = config.CASE_INSENSITIVE
app.config["SQLALCHEMY_DATABASE_URI"] = config.SQLA_URI
app.config["BASIC_AUTH_USERNAME"] = os.environ.get("AUTOAPI_ADMIN_USERNAME", "")
app.config["BASIC_AUTH_PASSWORD"] = os.environ.get("AUTOAPI_ADMIN_PASSWORD", "")
CORS(app)
basic_auth = BasicAuth(app)
@app.before_request
def refresh():
tables = utils.get_tables()
if tables != app.config["SQLALCHEMY_TABLES"]:
utils.refresh_tables()
app.config["SQLALCHEMY_TABLES"] = tables
app.config["SQLALCHEMY_TABLES"] = utils.get_tables()
@app.before_request
def protect_admin():
if request.path.startswith("/admin/"):
if not basic_auth.authenticate():
return basic_auth.challenge()
blueprint = aws.make_blueprint()
app.register_blueprint(blueprint)
utils.activate(admin=True)
route = os.path.join("/api-program", config.API_NAME)
container = DispatcherMiddleware(app.wsgi_app, {route: app})
return container
def __init__(self, layer_sizes=None, layer_acts=None, layer_keeps=None, layer_l2=None, init_path=None,
opt_algo='gd', learning_rate=1e-2, random_seed=None):
init_vars = []
num_inputs = len(layer_sizes[0])
factor_order = layer_sizes[1]
for i in range(num_inputs):
layer_input = layer_sizes[0][i]
layer_output = factor_order
init_vars.append(('w0_%d' % i, [layer_input, layer_output], 'tnormal', dtype))
init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype))
init_vars.append(('w1', [num_inputs * factor_order, layer_sizes[2]], 'tnormal', dtype))
init_vars.append(('b1', [layer_sizes[2]], 'zero', dtype))
for i in range(2, len(layer_sizes) - 1):
layer_input = layer_sizes[i]
layer_output = layer_sizes[i + 1]
init_vars.append(('w%d' % i, [layer_input, layer_output], 'tnormal', dtype))
init_vars.append(('b%d' % i, [layer_output], 'zero', dtype))
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.vars = utils.init_var_map(init_vars, init_path)
w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)]
b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)]
l = tf.nn.dropout(
utils.activate(
tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) + b0[i]
for i in range(num_inputs)], 1),
layer_acts[0]),
layer_keeps[0])
for i in range(1, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
l = tf.nn.dropout(
utils.activate(
tf.matmul(l, wi) + bi,
layer_acts[i]),
layer_keeps[i])
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y))
if layer_l2 is not None:
for i in range(num_inputs):
self.loss += layer_l2[0] * tf.nn.l2_loss(w0[i])
for i in range(1, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
# bi = self.vars['b%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def _back_propogate(self, layers, prediction, labels):
delta = prediction - labels
for i in reversed(range(self._n_layers)):
if i is 0:
break
dW = np.matmul(
delta,
utils.activate(self._layer_activations[i - 1],
layers[i - 1]).T)
dB = np.sum(delta, axis=1)
dB = np.reshape(dB, (dB.shape[0], -1))
delta = utils.calc_der(self._layer_activations[i - 1],
layers[i - 1]) * (np.matmul(
self._weights[i - 1].T, delta))
self._weights[i - 1] -= self._alpha * dW
if self._bias_weights[i - 1] is not None:
self._bias_weights[i - 1] -= self._alpha * dB
def _gradient_descent(self, X, T):
# No. of training examples
n = X.shape[0]
# No. of features
m = X.shape[1]
if self._weights is None:
if self._initialisation == 'gaussian':
self._weights = np.random.randn(m, 1)
else:
self._weights = np.random.uniform(-1, 1, (m, 1))
tmp = np.matmul(X, self._weights)
tmp[tmp > 15] = 15
tmp[tmp < -15] = -15
Y = utils.activate('sigmoid', tmp)
# cost = (1/n) * np.sum (-(T * np.log(Y) + (1 - T) * np.log(1 - Y)), axis = 0)
cost = np.sum(-(T * np.log(Y) + (1 - T) * np.log(1 - Y)), axis=0)
if self._regularisation == 'L1':
cost += self._lambda_reg * np.sum(abs(self._weights), axis=0)
if self._regularisation == 'L2':
cost += self._lambda_reg * np.sum(self._weights**2)
diff = Y - T
self.train_accuracy = self._accuracy(Y >= 0.5, T)
self.costs.append(cost)
# print(cost)
self._update_weights(X, diff)
def __init__(self, field_sizes=None, embed_size=10, filter_sizes=None, layer_acts=None, drop_out=None,
init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None):
Model.__init__(self)
init_vars = []
num_inputs = len(field_sizes)
for i in range(num_inputs):
init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype))
init_vars.append(('f1', [embed_size, filter_sizes[0], 1, 2], 'xavier', dtype))
init_vars.append(('f2', [embed_size, filter_sizes[1], 2, 2], 'xavier', dtype))
init_vars.append(('w1', [2 * 3 * embed_size, 1], 'xavier', dtype))
init_vars.append(('b1', [1], 'zero', dtype))
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.keep_prob_train = 1 - np.array(drop_out)
self.keep_prob_test = np.ones_like(drop_out)
self.layer_keeps = tf.placeholder(dtype)
self.vars = init_var_map(init_vars, init_path)
w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)]
xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1)
l = xw
l = tf.transpose(tf.reshape(l, [-1, num_inputs, embed_size, 1]), [0, 2, 1, 3])
f1 = self.vars['f1']
l = tf.nn.conv2d(l, f1, [1, 1, 1, 1], 'SAME')
l = tf.transpose(
max_pool_4d(
tf.transpose(l, [0, 1, 3, 2]),
int(num_inputs / 2)),
[0, 1, 3, 2])
f2 = self.vars['f2']
l = tf.nn.conv2d(l, f2, [1, 1, 1, 1], 'SAME')
l = tf.transpose(
max_pool_4d(
tf.transpose(l, [0, 1, 3, 2]), 3),
[0, 1, 3, 2])
l = tf.nn.dropout(
activate(
tf.reshape(l, [-1, embed_size * 3 * 2]),
layer_acts[0]),
self.layer_keeps[0])
w1 = self.vars['w1']
b1 = self.vars['b1']
l = tf.matmul(l, w1) + b1
l = tf.squeeze(l)
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y))
self.optimizer = get_optimizer(opt_algo, learning_rate, self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def eval(self, data, dropout_keep_prob=1.0):
data = self.__flatten_data_shape(data)
logits = tf.matmul(data, self.weights) + self.biases
if self.activation_func == '':
return logits
activations = utils.activate(logits, self.activation_func)
activations = tf.nn.dropout(activations, dropout_keep_prob)
return activations
def _forward_propogate(self, input_vector, labels):
layers = []
layer = np.reshape(input_vector, (input_vector.shape[0], -1))
layers.append(layer)
for i in range(self._n_layers):
if i is 0:
continue
active_layer = utils.activate(self._layer_activations[i - 1],
layer)
drop = np.random.rand(
active_layer.shape[0],
active_layer.shape[1]) < self._keep_probs[i - 1]
active_layer *= drop
active_layer /= self._keep_probs[i - 1]
layer = np.matmul(self._weights[i - 1], active_layer)
if self._bias_weights[i - 1] is not None:
layer += self._bias_weights[i - 1]
layers.append(layer)
layers[-1][layers[-1] > 15] = 15
layers[-1][layers[-1] < -15] = -15
# print(layers)
prediction = utils.activate(
self._layer_activations[self._n_layers - 1], layer)
cost = np.sum(-(labels * np.log(prediction) +
(1 - labels) * np.log(1 - prediction)),
axis=1)
self.costs.append(cost)
# print(cost)
self.train_accuracy = np.sum(((prediction >= 0.5) - labels)**2, axis=1)
self.train_accuracy = (1 -
self.train_accuracy / prediction.shape[1]) * 100
return layers, prediction
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None,
embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None):
Model.__init__(self)
init_vars = []
num_inputs = len(field_sizes)
print('num_inputs:{0}\\t\tlayer_size:{1}'.format(num_inputs, layer_sizes))
for i in range(num_inputs):
init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) # 为每个特征值初始化一个长度为10的向量
node_in = num_inputs * embed_size # 将每个特征embeding 为10维的向量， 总共16个特征，所以是160个输入 网络为[160,500,1]
for i in range(len(layer_sizes)):
init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype))
init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype))
node_in = layer_sizes[i]
print('init_vars:', init_vars)
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.keep_prob_train = 1 - np.array(drop_out)
self.keep_prob_test = np.ones_like(drop_out)
self.layer_keeps = tf.placeholder(dtype)
self.vars = utils.init_var_map(init_vars, init_path)
w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)]
xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)],
1) # 将每个特征的隐含向量连起来，组成网络的输入，160维
l = xw
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
print('第{0}个隐藏层l.shape, wi.shape, bi.shape'.format(i), l.shape, wi.shape, bi.shape)
l = tf.nn.dropout(
utils.activate(
tf.matmul(l, wi) + bi,
layer_acts[i]),
self.layer_keeps[i])
l = tf.squeeze(l) # 从tensor中删除所有大小是1的维度
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y))
if layer_l2 is not None:
self.loss += embed_l2 * tf.nn.l2_loss(xw)
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None,
embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None):
Model.__init__(self)
init_vars = []
num_inputs = len(field_sizes)
print('num_inputs:{0}\\t\tlayer_size:{1}'.format(num_inputs, layer_sizes))
for i in range(num_inputs):
init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) # 为每个特征值初始化一个长度为10的向量
node_in = num_inputs * embed_size # 将每个特征embeding 为10维的向量， 总共16个特征，所以是160个输入 网络为[160， 500， 1]
for i in range(len(layer_sizes)):
init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype))
init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype))
node_in = layer_sizes[i]
print('init_vars:', init_vars)
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.keep_prob_train = 1 - np.array(drop_out)
self.keep_prob_test = np.ones_like(drop_out)
self.layer_keeps = tf.placeholder(dtype)
self.vars = utils.init_var_map(init_vars, init_path)
w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)]
xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1) # 将每个特征的隐含向量连起来，组成网络的输入，160维
l = xw
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
print('第{0}个隐藏层l.shape, wi.shape, bi.shape'.format(i), l.shape, wi.shape, bi.shape)
l = tf.nn.dropout(
utils.activate(
tf.matmul(l, wi) + bi,
layer_acts[i]),
self.layer_keeps[i])
l = tf.squeeze(l) # 从tensor中删除所有大小是1的维度
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y))
if layer_l2 is not None:
self.loss += embed_l2 * tf.nn.l2_loss(xw)
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def forward(net_type, depth, act_func, x_vec, _vars):
if net_type == "nn":
W, b = _vars
a = [0] * (depth - 1)
z = [0] * (depth - 1)
z[0] = x_vec
for _i in range(depth - 2):
a[_i + 1] = tf.matmul(z[_i], W[_i]) + b[_i]
z[_i + 1] = activate(act_func, a[_i + 1])
y_hat = tf.matmul(z[depth - 2], W[depth - 2]) + b[depth - 2]
return y_hat
def predict(self, features):
features = np.reshape(features, (features.shape[0], -1))
features = np.concatenate((np.ones((features.shape[0], 1)), features),
axis=1)
tmp = np.matmul(features, self._weights)
tmp[tmp > 15] = 15
tmp[tmp < -15] = -15
prediction = utils.activate('sigmoid', tmp)
return prediction >= 0.5
def convolution(self, features, kernels, biases):
"""
Implements a single convolution layer
:param features: single or a list of features on which convolution
needs to be performed
:type features: numpy nd array
:param kernels: an array of filter kernels. Only square kernels
are supported as of now
:type kernels: numpy nd array
:param biases: an array of biases, one for each kernel
:type biases : numpy 1d array
:returns: numpy nd array, output features
"""
n_features = features.shape[0] # number of input feature arrays
feat_rows = features.shape[1] # rows in the input features
feat_cols = features.shape[2] # cols in the input features
n_kernel_sets = kernels.shape[0] # number of kernel sets, one set
n_kernels = kernels.shape[1] # no. of kernels per set
kernel_dim = kernels.shape[2] # dimension of a kernel
# dimensions of a single convolved feature array
conv_feat_rows = feat_rows - kernel_dim + 1
conv_feat_cols = feat_cols - kernel_dim + 1
convolved_features = [] # resultant array
# iterate through all sets of filter/kernels.
for ker_set_idx in xrange(n_kernel_sets):
# initialize a single output feature
out_feature = np.zeros((conv_feat_rows, conv_feat_cols))
for ker_idx in xrange(n_kernels):
# an output feature is the sum of all convolved individual
# input features with a particular kernel
out_feature += convolve2d(features[ker_idx],
kernels[ker_set_idx][ker_idx],
mode='valid')
# add bias to the resultant feature
out_feature += biases[ker_set_idx]
# append the resultant feature array
convolved_features.append(activate(out_feature, method='sigmoid'))
return np.array(convolved_features, dtype=np.float32)
def convolution(self, features, kernels, biases):
"""
Implements a single convolution layer
:param features: single or a list of features on which convolution
needs to be performed
:type features: numpy nd array
:param kernels: an array of filter kernels. Only square kernels
are supported as of now
:type kernels: numpy nd array
:param biases: an array of biases, one for each kernel
:type biases : numpy 1d array
:returns: numpy nd array, output features
"""
n_features = features.shape[0] # number of input feature arrays
feat_rows = features.shape[1] # rows in the input features
feat_cols = features.shape[2] # cols in the input features
n_kernel_sets = kernels.shape[0] # number of kernel sets, one set
n_kernels = kernels.shape[1] # no. of kernels per set
kernel_dim = kernels.shape[2] # dimension of a kernel
# dimensions of a single convolved feature array
conv_feat_rows = feat_rows - kernel_dim + 1
conv_feat_cols = feat_cols - kernel_dim + 1
convolved_features = [] # resultant array
# iterate through all sets of filter/kernels.
for ker_set_idx in xrange(n_kernel_sets):
# initialize a single output feature
out_feature = np.zeros((conv_feat_rows, conv_feat_cols))
for ker_idx in xrange(n_kernels):
# an output feature is the sum of all convolved individual
# input features with a particular kernel
out_feature += convolve2d(features[ker_idx],
kernels[ker_set_idx][ker_idx],
mode='valid')
# add bias to the resultant feature
out_feature += biases[ker_set_idx]
# append the resultant feature array
convolved_features.append(activate(out_feature, method='sigmoid'))
return np.array(convolved_features, dtype=np.float32)
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None,
embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None):
Model.__init__(self)
init_vars = []
num_inputs = len(field_sizes)
for i in range(num_inputs):
init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype))
node_in = num_inputs * embed_size
for i in range(len(layer_sizes)):
init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype))
init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype))
node_in = layer_sizes[i]
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.keep_prob_train = 1 - np.array(drop_out)
self.keep_prob_test = np.ones_like(drop_out)
self.layer_keeps = tf.placeholder(dtype)
self.vars = init_var_map(init_vars, init_path)
w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)]
xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1)
l = xw
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
print(l.shape, wi.shape, bi.shape)
l = tf.nn.dropout(
activate(
tf.matmul(l, wi) + bi,
layer_acts[i]),
self.layer_keeps[i])
l = tf.squeeze(l)
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y))
if layer_l2 is not None:
self.loss += embed_l2 * tf.nn.l2_loss(xw)
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
self.optimizer = get_optimizer(opt_algo, learning_rate, self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def predict(self, input_vector):
layer = np.reshape(input_vector, (input_vector.shape[0], -1))
for i in range(self._n_layers):
if i is 0:
continue
layer = np.matmul(
self._weights[i - 1],
utils.activate(self._layer_activations[i - 1], layer))
if self._bias_weights[i - 1] is not None:
layer += self._bias_weights[i - 1]
layer[layer > 15] = 15
layer[layer < -15] = -15
prediction = utils.activate(
self._layer_activations[self._n_layers - 1], layer)
return prediction >= 0.5
def make_app():
app = sandman2.get_app(config.SQLA_URI, Base=utils.AutomapModel)
app.json_encoder = utils.APIJSONEncoder
app.config['CASE_INSENSITIVE'] = config.CASE_INSENSITIVE
app.config['BASIC_AUTH_USERNAME'] = os.environ.get('AUTOAPI_ADMIN_USERNAME', '')
app.config['BASIC_AUTH_PASSWORD'] = os.environ.get('AUTOAPI_ADMIN_PASSWORD', '')
CORS(app)
basic_auth = BasicAuth(app)
@app.before_request
def refresh():
tables = utils.get_tables()
if tables != app.config['SQLALCHEMY_TABLES']:
utils.refresh_tables()
app.config['SQLALCHEMY_TABLES'] = tables
@app.before_request
def protect_admin():
if request.path.startswith('/admin/'):
if not basic_auth.authenticate():
return basic_auth.challenge()
aws_blueprint = aws.make_blueprint()
app.register_blueprint(aws_blueprint)
docs_blueprint = swagger.make_blueprint(app)
app.register_blueprint(docs_blueprint)
route = os.path.join('/api-program', config.API_NAME)
container = DispatcherMiddleware(app.wsgi_app, {route: app})
with app.app_context():
app.config['SQLALCHEMY_TABLES'] = utils.get_tables()
utils.activate()
return app, container
def execute(self, features):
"""
Executes the layer, ie, constructs the output feature array
as activations of the layer absed on the stored weights
:params features: input features
:type features : numpy 1d array
"""
# perform dot product operation on input against the weights
# to get the effective input to each neuron in the layer
weighted_input = np.dot(features, self.weights) + self.biases
# the output is the activation performed on weighted_input
self.out = activate(weighted_input, method='sigmoid')
def execute(self, features):
"""
Executes the layer, ie, constructs the output feature array
as activations of the layer absed on the stored weights
:params features: input features
:type features : numpy 1d array
"""
# perform dot product operation on input against the weights
# to get the effective input to each neuron in the layer
weighted_input = np.dot(features, self.weights) + self.biases
# the output is the activation performed on weighted_input
self.out = activate(weighted_input, method='sigmoid')
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None,
embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None):
Model.__init__(self)
init_vars = []
num_inputs = len(field_sizes)
for i in range(num_inputs):
init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype))
num_pairs = int(num_inputs * (num_inputs - 1) / 2)
node_in = num_inputs * embed_size + num_pairs
# node_in = num_inputs * (embed_size + num_inputs)
for i in range(len(layer_sizes)):
init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype))
init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype))
node_in = layer_sizes[i]
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.keep_prob_train = 1 - np.array(drop_out)
self.keep_prob_test = np.ones_like(drop_out)
self.layer_keeps = tf.placeholder(dtype)
self.vars = utils.init_var_map(init_vars, init_path)
w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)]
xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1)
xw3d = tf.reshape(xw, [-1, num_inputs, embed_size])
row = []
col = []
for i in range(num_inputs-1):
for j in range(i+1, num_inputs):
row.append(i)
col.append(j)
# batch * pair * k
p = tf.transpose(
# pair * batch * k
tf.gather(
# num * batch * k
tf.transpose(
xw3d, [1, 0, 2]),
row),
[1, 0, 2])
# batch * pair * k
q = tf.transpose(
tf.gather(
tf.transpose(
xw3d, [1, 0, 2]),
col),
[1, 0, 2])
p = tf.reshape(p, [-1, num_pairs, embed_size])
q = tf.reshape(q, [-1, num_pairs, embed_size])
ip = tf.reshape(tf.reduce_sum(p * q, [-1]), [-1, num_pairs])
# simple but redundant
# batch * n * 1 * k, batch * 1 * n * k
# ip = tf.reshape(
# tf.reduce_sum(
# tf.expand_dims(xw3d, 2) *
# tf.expand_dims(xw3d, 1),
# 3),
# [-1, num_inputs**2])
l = tf.concat([xw, ip], 1)
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
l = tf.nn.dropout(
utils.activate(
tf.matmul(l, wi) + bi,
layer_acts[i]),
self.layer_keeps[i])
l = tf.squeeze(l)
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y))
if layer_l2 is not None:
self.loss += embed_l2 * tf.nn.l2_loss(xw)
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def __init__(self, feature2field , layer_sizes, layer_acts, drop_out, learning_rate=1e-2, l2_reg = 0,
feature_size = None, field_size = None, k = 40, opt_algo = 'adam', train = True):
# feature_size : feature number # field_size : field number # k : latent vector dimension
Model.__init__(self)
layers = [int(e) for e in layer_sizes.split(',')]
dropout = [float(e) for e in drop_out.split(',')]
layer_active_func = [e for e in layer_acts.split(',')]
self.X = tf.placeholder(dtype=dtype, shape=[None, feature_size], name='input')
self.y = tf.placeholder(dtype=dtype, shape=[None, ], name='label')
self.keep_prob = tf.placeholder(dtype=dtype)
with tf.variable_scope('linear_layer'): # w .* x + b
self.b = tf.get_variable(name='bias', initializer=tf.constant(0.5), dtype=dtype) # tf.zeros_initializer()
self.w1 = tf.get_variable(name='w1', shape=[feature_size], initializer=tf.truncated_normal_initializer(mean=0, stddev=1e-2), dtype=dtype)
self.linear_terms = tf.reduce_sum(tf.multiply(self.w1, self.X), 1) + self.b
with tf.variable_scope('field_aware_interaction_layer'): # sum(<vi_fj, vj_fi> * x_i * x_j)
self.nfk = tf.get_variable('nfk', shape=[feature_size, field_size, k], dtype=dtype, initializer=tf.truncated_normal_initializer(mean=0, stddev=0.01))
self.field_aware_interaction_terms = tf.constant(0, dtype=dtype)
for i in range(feature_size):
for j in range(i + 1, feature_size):
self.field_aware_interaction_terms += tf.multiply(
tf.reduce_sum(tf.multiply(self.nfk[i, feature2field[j]], self.nfk[j, feature2field[i]])),
tf.multiply(self.X[:, i], self.X[:, j])
)
init_deep_layer_vars = []
input_dim = feature_size
for i in range(len(layers)):
output_dim = layers[i]
init_deep_layer_vars.append(('deepW_%d' % i, [input_dim, output_dim], 'xavier', dtype))
init_deep_layer_vars.append(('deepB_%d' % i, [output_dim], 'zero', dtype))
input_dim = layers[i]
init_deep_layer_vars.append(('outW', [layers[-1], 1], 'xavier', dtype))
init_deep_layer_vars.append(('outB', [1], 'zero', dtype))
self.deepVars = init_var_map(init_deep_layer_vars)
with tf.variable_scope("Deep-part"):
hidden = self.X
for i in range(len(layers)):
if train:
hidden = tf.nn.dropout( # h_i = W_i * x + b_i
activate(tf.matmul(hidden, self.deepVars['deepW_%d' % i]) + self.deepVars['deepB_%d' % i], layer_active_func[i]),
dropout[i])
else:
hidden = activate(tf.matmul(hidden, self.deepVars['deepW_%d' % i]) + self.deepVars['deepB_%d' % i], layer_active_func[i])
#self.a=hidden;self.aa=self.deepVars['outW'];self.aaa=tf.matmul(hidden, self.deepVars['outW']);self.aaaa=self.deepVars['outB']
self.deepOut = tf.matmul(hidden, self.deepVars['outW']) + self.deepVars['outB']
self.deepOut = tf.reshape(self.deepOut, shape=[-1]) # tf.reshape() 防止 python 自带的广播机制算出奇怪的值
with tf.variable_scope("DeepFM-out"):
self.out_sum = self.linear_terms + self.field_aware_interaction_terms + self.deepOut
self.pred_prob = tf.sigmoid(self.out_sum)
# ------bulid loss------
self.loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=self.out_sum, labels=self.y)) + \
l2_reg * tf.nn.l2_loss(self.w1) + \
l2_reg * tf.nn.l2_loss(self.nfk)
# ------bulid optimizer------
self.optimizer = get_optimizer(opt_algo, learning_rate, self.loss)
# 保存模型的参数
self.saver = tf.train.Saver(tf.global_variables())
# GPU设定
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
def __init__(self,
layer_sizes=None,
layer_acts=None,
layer_keeps=None,
init_path=None,
opt_algo='gd',
learning_rate=1e-2,
random_seed=None):
init_vars = []
num_inputs = len(layer_sizes[0])
embedding_order = layer_sizes[1]
for i in range(num_inputs):
layer_input = layer_sizes[0][i]
layer_output = embedding_order
init_vars.append(('w0_%d' % i, [layer_input,
layer_output], 'tnormal', dtype))
init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype))
init_vars.append(('f1', [embedding_order, layer_sizes[2], 1,
2], 'tnormal', dtype))
init_vars.append(('f2', [embedding_order, layer_sizes[3], 2,
2], 'tnormal', dtype))
init_vars.append(('w1', [2 * 3 * embedding_order,
1], 'tnormal', dtype))
init_vars.append(('b1', [1], 'zero', dtype))
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.vars = utils.init_var_map(init_vars, init_path)
w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)]
b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)]
l = tf.nn.dropout(
utils.activate(
tf.concat(1, [
tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) + b0[i]
for i in range(num_inputs)
]), layer_acts[0]), layer_keeps[0])
l = tf.transpose(
tf.reshape(l, [-1, num_inputs, embedding_order, 1]),
[0, 2, 1, 3])
f1 = self.vars['f1']
l = tf.nn.conv2d(l, f1, [1, 1, 1, 1], 'SAME')
l = tf.transpose(
utils.max_pool_4d(tf.transpose(l, [0, 1, 3, 2]),
num_inputs / 2), [0, 1, 3, 2])
f2 = self.vars['f2']
l = tf.nn.conv2d(l, f2, [1, 1, 1, 1], 'SAME')
l = tf.transpose(
utils.max_pool_4d(tf.transpose(l, [0, 1, 3, 2]), 3),
[0, 1, 3, 2])
l = tf.nn.dropout(
utils.activate(tf.reshape(l, [-1, embedding_order * 3 * 2]),
layer_acts[1]), layer_keeps[1])
w1 = self.vars['w1']
b1 = self.vars['b1']
l = tf.nn.dropout(
utils.activate(tf.matmul(l, w1) + b1, layer_acts[2]),
layer_keeps[2])
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(l, self.y))
self.optimizer = utils.get_optimizer(opt_algo, learning_rate,
self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.initialize_all_variables().run(session=self.sess)
def eval(self, data, dropout_keep_prob=1.0):
logits = utils.conv2d(
data, self.weights, stride=self.stride,
padding=self.padding) + self.biases
activations = utils.activate(logits, self.activation_func)
return activations
def calc_activations(self, data, weights, biases):
logits = utils.conv2d(
data, weights, stride=self.stride, padding=self.padding) + biases
activations = utils.activate(logits, self.activation_func)
return activations
def __init__(self,
layer_sizes=None,
layer_acts=None,
layer_keeps=None,
layer_l2=None,
kernel_l2=None,
init_path=None,
opt_algo='gd',
learning_rate=1e-2,
random_seed=None):
"""
# Arguments:
layer_size: [num_fields, factor_layer, l_p size]
layer_acts: ["tanh", "none"]
layer_keep: [1, 1]
layer_l2: [0, 0]
kernel_l2: 0
"""
init_vars = []
num_inputs = len(layer_sizes[0])
factor_order = layer_sizes[1]
for i in range(num_inputs):
layer_input = layer_sizes[0][i]
layer_output = factor_order
# w0 store the embeddings for all features.
init_vars.append(('w0_%d' % i, [layer_input,
layer_output], 'tnormal', dtype))
init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype))
init_vars.append(('w_l', [num_inputs * factor_order,
layer_sizes[2]], 'tnormal', dtype))
init_vars.append(('w_p', [num_inputs * num_inputs,
layer_sizes[2]], 'tnormal', dtype))
#init_vars.append(('w1', [num_inputs * factor_order + num_inputs * num_inputs, layer_sizes[2]], 'tnormal', dtype))
init_vars.append(('b1', [layer_sizes[2]], 'zero', dtype))
for i in range(2, len(layer_sizes) - 1):
layer_input = layer_sizes[i]
layer_output = layer_sizes[i + 1]
init_vars.append(('w%d' % i, [layer_input,
layer_output], 'tnormal', dtype))
init_vars.append(('b%d' % i, [layer_output], 'zero', dtype))
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.vars = utils.init_var_map(init_vars, init_path)
w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)]
b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)]
# Multiply SparseTensor X[i] by dense matrix w0[i]
xw = [
tf.sparse_tensor_dense_matmul(self.X[i], w0[i])
for i in range(num_inputs)
]
x = tf.concat([xw[i] + b0[i] for i in range(num_inputs)], 1)
l = tf.nn.dropout(utils.activate(x, layer_acts[0]), layer_keeps[0])
w_l = self.vars['w_l']
w_p = self.vars['w_p']
b1 = self.vars['b1']
# This is where W_p \cdot p happens.
# k1 is \theta, which is the weight for each field(feature) vector
p = tf.matmul(
tf.reshape(l, [-1, num_inputs, factor_order]),
tf.transpose(tf.reshape(l, [-1, num_inputs, factor_order]),
[0, 2, 1]))
p = tf.nn.dropout(
utils.activate(
tf.matmul(tf.reshape(p, [-1, num_inputs * num_inputs]),
w_p), 'none'), 1.0)
l = tf.nn.dropout(
utils.activate(tf.matmul(l, w_l) + b1 + p, layer_acts[1]),
layer_keeps[1])
for i in range(2, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
l = tf.nn.dropout(
utils.activate(tf.matmul(l, wi) + bi, layer_acts[i]),
layer_keeps[i])
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l,
labels=self.y))
if layer_l2 is not None:
# for i in range(num_inputs):
self.loss += layer_l2[0] * tf.nn.l2_loss(tf.concat(xw, 1))
for i in range(1, len(layer_sizes) - 1):
if i == 1:
self.loss += layer_l2[i] * tf.nn.l2_loss(w_l)
self.loss += layer_l2[i] * tf.nn.l2_loss(w_p)
else:
wi = self.vars['w%d' % i]
# bi = self.vars['b%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
if kernel_l2 is not None:
pass
self.optimizer = utils.get_optimizer(opt_algo, learning_rate,
self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def __init__(self,
data_dir=None,
summary_dir=None,
eval_dir=None,
batch_size=None,
input_dim=None,
output_dim=1,
layer_sizes=None,
layer_acts=None,
drop_out=None,
layer_l2=None,
kernel_l2=None,
l2_w=0,
init_path=None,
opt_algo='gd',
learning_rate=1e-2,
sync=False,
workers=20):
Model.__init__(self)
eprint("------- create graph ---------------")
init_vars = []
num_inputs = len(layer_sizes[0])
factor_order = layer_sizes[1]
for i in range(num_inputs):
layer_input = layer_sizes[0][i]
layer_output = factor_order
init_vars.append(('w0_%d' % i, [layer_input,
layer_output], 'tnormal', dtype))
init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype))
init_vars.append(('w1', [num_inputs * factor_order,
layer_sizes[2]], 'tnormal', dtype))
init_vars.append(('k1', [num_inputs,
layer_sizes[2]], 'tnormal', dtype))
init_vars.append(('b1', [layer_sizes[2]], 'zero', dtype))
for i in range(2, len(layer_sizes) - 1):
layer_input = layer_sizes[i]
layer_output = layer_sizes[i + 1]
init_vars.append((
'w%d' % i,
[layer_input, layer_output],
'tnormal',
))
init_vars.append(('b%d' % i, [layer_output], 'zero', dtype))
with tf.name_scope('input_%d' % FLAGS.task_index) as scope:
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.B = tf.sparse_placeholder(tf.float32, name='B')
self.y = tf.placeholder(dtype)
self.keep_prob_train = 1 - np.array(drop_out)
self.keep_prob_test = np.ones_like(drop_out)
self.layer_keeps = tf.placeholder(dtype)
self.vars = utils.init_var_map(init_vars, init_path)
w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)]
b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)]
xw = [
tf.sparse_tensor_dense_matmul(self.X[i], w0[i])
for i in range(num_inputs)
]
x = tf.concat([xw[i] + b0[i] for i in range(num_inputs)], 1)
l = tf.nn.dropout(utils.activate(x, layer_acts[0]),
self.layer_keeps[0])
w1 = self.vars['w1']
k1 = self.vars['k1']
b1 = self.vars['b1']
p = tf.reduce_sum(
tf.reshape(
tf.matmul(
tf.reshape(
tf.transpose(
tf.reshape(l, [-1, num_inputs, factor_order]),
[0, 2, 1]), [-1, num_inputs]), k1),
[-1, factor_order, layer_sizes[2]]), 1)
l = tf.nn.dropout(
utils.activate(tf.matmul(l, w1) + b1 + p, layer_acts[1]),
self.layer_keeps[1])
for i in range(2, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
l = tf.nn.dropout(
utils.activate(tf.matmul(l, wi) + bi, layer_acts[i]),
self.layer_keeps[i])
## logits
l = tf.reshape(l, [-1])
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y))
if layer_l2 is not None:
self.loss += layer_l2[0] * tf.nn.l2_loss(tf.concat(xw, 1))
for i in range(1, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
if kernel_l2 is not None:
self.loss += kernel_l2 * tf.nn.l2_loss(k1)
self.global_step = _variable_on_cpu(
'global_step', [],
initializer=tf.constant_initializer(0),
trainable=False)
if sync:
self.optimizer = utils.get_sync_optimizer(opt_algo, learning_rate,
workers)
else:
self.optimizer = utils.get_optimizer(opt_algo, learning_rate)
self.train_op = self.optimizer.minimize(self.loss,
global_step=self.global_step)
self.summary_op = tf.summary.merge_all()
def __init__(self,
field_sizes=None,
embed_size=10,
layer_size=None,
layer_acts=None,
drop_out=None,
embed_l2=None,
layer_l2=None,
init_path=None,
opt_algo='gd',
learning_rate=1e-2,
random_seed=None,
layer_norm=True):
Model.__init__(self)
init_vars = []
num_input = len(field_sizes)
for i in range(num_input):
init_vars.append(('embed_%d' % i, [field_sizes[i],
embed_size], 'xavier', dtype))
num_pairs = int(num_input * (num_input - 1) / 2) # field对的个数
node_in = num_input * embed_size + num_pairs # 此处为设计的关键
init_vars.append(('kernel', [embed_size, num_pairs,
embed_size], 'xavier', dtype))
for i in range(len(layer_size)):
init_vars.append(('w%d' % i, [node_in,
layer_size[i]], 'xavier', dtype))
init_vars.append(('b%d' % i, [layer_size[i]], 'zero', dtype))
node_in = layer_size[i]
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_input)]
self.y = tf.placeholder(dtype)
self.keep_prob_train = 1 - np.array(drop_out)
self.keep_prob_test = np.ones_like(drop_out)
self.layer_keeps = tf.placeholder(dtype)
self.vars = utils.init_var_map(init_vars, init_path)
w_0 = [self.vars['embed_%d' % i] for i in range(num_input)]
# batch * (embed_size*num_field)
xw = tf.concat([
tf.sparse_tensor_dense_matmul(self.X[i], w_0[i])
for i in range(num_input)
], 1)
xw3d = tf.reshape(xw, [-1, num_input, embed_size])
row = []
col = []
# 构造pair
for i in range(num_input - 1):
for j in range(i + 1, num_input):
row.append(i)
col.append(j)
# batch * pair * embed_size
p = tf.transpose(
# pair * batch * embed_size
tf.gather(
# num_field * batch * embed_size
tf.transpose(xw3d, [1, 0, 2]),
row),
[1, 0, 2])
q = tf.transpose(
# pair * batch * embed_size
tf.gather(
# num_field * batch * embed_size
tf.transpose(xw3d, [1, 0, 2]),
col),
[1, 0, 2])
p = tf.reshape(p, [-1, num_pairs, embed_size])
q = tf.reshape(q, [-1, num_pairs, embed_size])
# embed_size*num_pairs*embed_size
k = self.vars['kernel']
p = tf.expand_dims(p, 1) # 增加维度 batch * 1 * pair * embed_size
# batch * num_pairs
# temp = tf.multiply(p,k)
# temp = tf.reduce_sum(temp,-1)
# temp = tf.transpose(temp,[0,2,1])
# temp = tf.multiply(temp,q)
# temp = tf.reduce_sum(temp,-1)
kp = tf.reduce_sum(
# batch * num_pairs * embed_size
tf.multiply(
# 置换位置 batch * num_pairs * embed_size
tf.transpose(
# 按最后一个维度求和 batch * embed_size * num_pairs
tf.reduce_sum(
# 点乘 batch * embed_size*num_pairs*embed_size
tf.multiply(p, k),
-1),
[0, 2, 1]),
q),
-1)
l = tf.concat([xw, kp], 1)
for i in range(len(layer_size)):
w_i = self.vars['w%d' % i]
b_i = self.vars['b%d' % i]
l = tf.nn.dropout(
utils.activate(tf.matmul(l, w_i) + b_i, layer_acts[i]),
self.layer_keeps[i])
l = tf.squeeze(l)
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l,
labels=self.y))
if layer_l2 is not None:
self.loss += embed_l2 * tf.nn.l2_loss(xw)
for i in range(len(layer_size)):
w_i = self.vars['w%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(w_i)
self.optimizer = utils.get_optimizer(opt_algo, learning_rate,
self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None,
embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None):
Model.__init__(self)
init_vars = []
num_inputs = len(field_sizes)
for i in range(num_inputs):
init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype))
num_pairs = int(num_inputs * (num_inputs - 1) / 2)
node_in = num_inputs * embed_size + num_pairs
# node_in = num_inputs * (embed_size + num_inputs)
for i in range(len(layer_sizes)):
init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype))
init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype))
node_in = layer_sizes[i]
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.keep_prob_train = 1 - np.array(drop_out)
self.keep_prob_test = np.ones_like(drop_out)
self.layer_keeps = tf.placeholder(dtype)
self.vars = init_var_map(init_vars, init_path)
w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)]
xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1)
xw3d = tf.reshape(xw, [-1, num_inputs, embed_size])
row = []
col = []
for i in range(num_inputs-1):
for j in range(i+1, num_inputs):
row.append(i)
col.append(j)
# batch * pair * k
p = tf.transpose(
# pair * batch * k
tf.gather(
# num * batch * k
tf.transpose(
xw3d, [1, 0, 2]),
row),
[1, 0, 2])
# batch * pair * k
q = tf.transpose(
tf.gather(
tf.transpose(
xw3d, [1, 0, 2]),
col),
[1, 0, 2])
p = tf.reshape(p, [-1, num_pairs, embed_size])
q = tf.reshape(q, [-1, num_pairs, embed_size])
ip = tf.reshape(tf.reduce_sum(p * q, [-1]), [-1, num_pairs])
# simple but redundant
# batch * n * 1 * k, batch * 1 * n * k
# ip = tf.reshape(
# tf.reduce_sum(
# tf.expand_dims(xw3d, 2) *
# tf.expand_dims(xw3d, 1),
# 3),
# [-1, num_inputs**2])
l = tf.concat([xw, ip], 1)
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
l = tf.nn.dropout(
activate(
tf.matmul(l, wi) + bi,
layer_acts[i]),
self.layer_keeps[i])
l = tf.squeeze(l)
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y))
if layer_l2 is not None:
self.loss += embed_l2 * tf.nn.l2_loss(xw)
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
self.optimizer = get_optimizer(opt_algo, learning_rate, self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def __init__(self,
feature_size,
field_size,
embedding_size=8,
optimizer_type='gd',
learning_rate=1e-2,
verbose=False,
random_seed=None,
eval_metric=roc_auc_score,
greater_is_better=True,
epoch=10,
batch_size=1024,
l2_reg=0,
deep_layers=[32, 32],
batch_norm=True,
dropout_deep=[],
cross_layer_num=3):
Model.__init__(self, eval_metric, greater_is_better, epoch, batch_size,
verbose, batch_norm, dropout_deep)
init_vars = [('weight', [feature_size, 1], 'uniform', tf.float32),
('bias', [1], 'uniform', tf.float32),
('feature_embed', [feature_size,
embedding_size], 'normal', tf.float32)]
node_in = embedding_size * field_size
for i in range(len(deep_layers)):
init_vars.extend([('layer_%d' % i, [node_in, deep_layers[i]],
'glorot_normal', tf.float32)])
init_vars.extend([('bias_%d' % i, [1, deep_layers[i]],
'glorot_normal', tf.float32)])
node_in = deep_layers[i]
for i in range(cross_layer_num):
init_vars.extend([('cross_layer_%d' % i,
[1, embedding_size * field_size
], 'glorot_normal', tf.float32)])
init_vars.extend([('cross_bias_%d' % i,
[1, embedding_size * field_size
], 'glorot_normal', tf.float32)])
node_in = embedding_size * field_size + deep_layers[-1]
init_vars.extend([('concat_projection', [node_in, 1], 'glorot_normal',
tf.float32)])
init_vars.extend([('concat_bias', [1,
1], 'glorot_normal', tf.float32)])
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.feat_index = tf.placeholder(tf.int32,
shape=[None, None],
name="feat_index") # None * F
self.feat_value = tf.placeholder(tf.float32,
shape=[None, None],
name="feat_value") # None * F
self.label = tf.placeholder(tf.float32,
shape=[None, 1],
name="label") # None * 1
self.dropout_keep_deep = tf.placeholder(tf.float32,
shape=[None],
name="dropout_keep_deep")
self.train_phase = tf.placeholder(tf.bool, name="train_phase")
self.vars = utils.init_var_map(init_vars)
self.embeddings = tf.nn.embedding_lookup(
self.vars["feature_embed"], self.feat_index) # None * F * K
feat_value = tf.reshape(self.feat_value, shape=[-1, field_size, 1])
self.embeddings = tf.reshape(
tf.multiply(self.embeddings, feat_value),
shape=[-1, embedding_size * field_size])
# ---------- cross layer ----------
self.deep_cross_input = tf.nn.dropout(self.embeddings,
self.dropout_keep_deep[0])
self.cross_layer_out = self.deep_cross_input
for i in range(1, cross_layer_num):
self.x0xiT = self.deep_cross_input * self.cross_layer_out
self.x0xiT = tf.reduce_sum(self.x0xiT, 1,
keep_dims=True) # None * 1
self.cross_layer_out = tf.add(tf.matmul(self.x0xiT, self.vars['cross_layer_%d' % i]) \
, self.vars['cross_bias_%d' % i]) + self.cross_layer_out
# ---------- deep component --------
self.y_deep = self.deep_cross_input
for i in range(len(deep_layers)):
self.y_deep = tf.add(
tf.matmul(self.y_deep, self.vars['layer_%s' % i]),
self.vars['bias_%s' % i])
if self.batch_norm:
self.y_deep = self.batch_norm_layer(
self.y_deep,
train_phase=self.train_phase,
scope_bn="bn_%s" % i)
self.y_deep = tf.nn.dropout(
utils.activate(self.y_deep, 'relu'),
self.dropout_keep_deep[i + 1])
concat_projection = self.vars['concat_projection']
concat_bias = self.vars['concat_bias']
self.out = tf.concat([self.y_deep, self.cross_layer_out], 1)
self.out = tf.add(tf.matmul(self.out, concat_projection),
concat_bias)
self.y_prob = tf.sigmoid(self.out)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(labels=self.label, logits=self.out)) + \
tf.contrib.layers.l2_regularizer(
l2_reg)(self.vars["concat_projection"])
for i in range(len(deep_layers)):
self.loss += tf.contrib.layers.l2_regularizer(l2_reg)(
self.vars["layer_%d" % i])
self.optimizer = utils.get_optimizer(optimizer_type, learning_rate,
self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
self.sess.run(tf.global_variables_initializer())
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None,
embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None,
layer_norm=True):
Model.__init__(self)
init_vars = []
num_inputs = len(field_sizes)
for i in range(num_inputs):
init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype))
num_pairs = int(num_inputs * (num_inputs - 1) / 2)
node_in = num_inputs * embed_size + num_pairs
init_vars.append(('kernel', [embed_size, num_pairs, embed_size], 'xavier', dtype))
for i in range(len(layer_sizes)):
init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype))
init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype))
node_in = layer_sizes[i]
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.keep_prob_train = 1 - np.array(drop_out)
self.keep_prob_test = np.ones_like(drop_out)
self.layer_keeps = tf.placeholder(dtype)
self.vars = init_var_map(init_vars, init_path)
w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)]
xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1)
xw3d = tf.reshape(xw, [-1, num_inputs, embed_size])
row = []
col = []
for i in range(num_inputs - 1):
for j in range(i + 1, num_inputs):
row.append(i)
col.append(j)
# batch * pair * k
p = tf.transpose(
# pair * batch * k
tf.gather(
# num * batch * k
tf.transpose(
xw3d, [1, 0, 2]),
row),
[1, 0, 2])
# batch * pair * k
q = tf.transpose(
tf.gather(
tf.transpose(
xw3d, [1, 0, 2]),
col),
[1, 0, 2])
# b * p * k
p = tf.reshape(p, [-1, num_pairs, embed_size])
# b * p * k
q = tf.reshape(q, [-1, num_pairs, embed_size])
# k * p * k
k = self.vars['kernel']
# batch * 1 * pair * k
p = tf.expand_dims(p, 1)
# batch * pair
kp = tf.reduce_sum(
# batch * pair * k
tf.multiply(
# batch * pair * k
tf.transpose(
# batch * k * pair
tf.reduce_sum(
# batch * k * pair * k
tf.multiply(
p, k),
-1),
[0, 2, 1]),
q),
-1)
#
# if layer_norm:
# # x_mean, x_var = tf.nn.moments(xw, [1], keep_dims=True)
# # xw = (xw - x_mean) / tf.sqrt(x_var)
# # x_g = tf.Variable(tf.ones([num_inputs * embed_size]), name='x_g')
# # x_b = tf.Variable(tf.zeros([num_inputs * embed_size]), name='x_b')
# # x_g = tf.Print(x_g, [x_g[:10], x_b])
# # xw = xw * x_g + x_b
# p_mean, p_var = tf.nn.moments(op, [1], keep_dims=True)
# op = (op - p_mean) / tf.sqrt(p_var)
# p_g = tf.Variable(tf.ones([embed_size**2]), name='p_g')
# p_b = tf.Variable(tf.zeros([embed_size**2]), name='p_b')
# # p_g = tf.Print(p_g, [p_g[:10], p_b])
# op = op * p_g + p_b
l = tf.concat([xw, kp], 1)
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
l = tf.nn.dropout(
activate(
tf.matmul(l, wi) + bi,
layer_acts[i]),
self.layer_keeps[i])
l = tf.squeeze(l)
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y))
if layer_l2 is not None:
self.loss += embed_l2 * tf.nn.l2_loss(xw)#tf.concat(w0, 0))
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
self.optimizer = get_optimizer(opt_algo, learning_rate, self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def __init__(self,
train_loader,
test_loader,
embed_size=None,
layer_size=None,
layer_act=None,
layer_keeps=None,
opt_algo='gd',
learning_rate=0.01,
epoch=10,
early_stop_round=None,
l2=None,
random_seed=None):
self.graph = tf.Graph()
self.train_loader = train_loader
self.test_loader = test_loader
self.embed_size = embed_size
self.layer_size = layer_size
self.layer_act = layer_act
self.layer_keeps = layer_keeps
self.num_fields = len(config.FIELD_SIZES)
self.var_list = []
for idx in range(self.num_fields):
self.var_list.append([
'embed_{}'.format(idx),
[config.FIELD_SIZES[idx], self.embed_size[idx]], 'xavier'
])
in_size = sum(self.embed_size)
for idx in range(len(layer_size)):
self.var_list.append(
['w_{}'.format(idx), [in_size, layer_size[idx]], 'xavier'])
self.var_list.append(
['b_{}'.format(idx), [layer_size[idx]], 'zero'])
in_size = layer_size[idx]
self.var_dict = utils.get_var(self.var_list)
self.opt_algo = opt_algo
self.learning_rate = learning_rate
self.epoch = epoch
self.early_stop_round = early_stop_round
self.l2 = l2
self.random_seed = random_seed
self.time_scores = []
self.train_scores = []
self.test_scores = []
# with self.graph.as_default():
if self.random_seed is not None:
tf.set_random_seed(self.random_seed)
self.X = [
tf.sparse_placeholder(config.DTYPE) for n in range(self.num_fields)
]
self.y = tf.placeholder(config.DTYPE)
with tf.variable_scope('Dense_Real_Layer'):
w_embed = [
self.var_dict['embed_{}'.format(idx)]
for idx in range(self.num_fields)
]
xw = tf.concat([
tf.sparse_tensor_dense_matmul(self.X[idx], w_embed[idx])
for idx in range(self.num_fields)
], 1)
layer_out = xw
for idx in range(len(layer_size)):
with tf.variable_scope('Hiden_Layer_{}'.format(idx)):
wi = self.var_dict['w_{}'.format(idx)]
bi = self.var_dict['b_{}'.format(idx)]
layer_out = tf.nn.dropout(
utils.activate(
tf.matmul(layer_out, wi) + bi, self.layer_act[idx]),
self.layer_keeps[idx])
layer_out = tf.squeeze(layer_out)
self.y_preds = tf.sigmoid(layer_out)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(labels=self.y,
logits=layer_out))
if self.l2 is not None:
for idx in range(self.num_fields):
self.loss += self.l2 * tf.nn.l2_loss(
self.var_dict['embed_{}'.format(idx)])
for idx in range(len(self.layer_size)):
self.loss += self.l2 * tf.nn.l2_loss(
self.var_dict['w_{}'.format(idx)])
self.optimizer = utils.get_optimizer(self.opt_algo, self.learning_rate,
self.loss)
self.sess = tf.Session()
tf.global_variables_initializer().run(session=self.sess)
def __init__(self,
data_dir=None,
summary_dir=None,
eval_dir=None,
batch_size=None,
input_dim=None,
output_dim=1,
layer_sizes=None,
layer_acts=None,
drop_out=None,
init_path=None,
opt_algo='gd',
learning_rate=1e-2,
l2_w=0,
layer_l2=None,
sync=False,
workers=20):
Model.__init__(self)
eprint("-------- create graph ----------")
init_vars = []
# linear part
init_vars.append(('linear', [input_dim, output_dim], 'xavier', dtype))
init_vars.append(('bias', [output_dim], 'zero', dtype))
num_inputs = len(layer_sizes[0])
factor_order = layer_sizes[1]
for i in range(num_inputs):
layer_input = layer_sizes[0][i]
layer_output = factor_order
# field_sizes[i] stores the i-th field feature number
init_vars.append(('w0_%d' % i, [layer_input,
layer_output], 'xavier', dtype))
init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype))
# full connection
node_in = num_inputs * factor_order
init_vars.append(('w1', [node_in, layer_sizes[2]], 'xavier', dtype))
init_vars.append(('b1', [layer_sizes[2]], 'zero', dtype))
for i in range(2, len(layer_sizes) - 1):
layer_input = layer_sizes[i]
layer_output = layer_sizes[i + 1]
init_vars.append(('w%d' % i, [layer_input,
layer_output], 'xavier', dtype))
init_vars.append(('b%d' % i, [layer_output], 'zero', dtype))
#self.graph = tf.Graph()
#with self.graph.as_default():
#with tf.device('/cpu:0'):
with tf.name_scope('input_%d' % FLAGS.task_index) as scope:
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.B = tf.sparse_placeholder(tf.float32, name='B')
self.y = tf.placeholder(dtype)
self.keep_prob_train = 1 - np.array(drop_out)
self.keep_prob_test = np.ones_like(drop_out)
self.layer_keeps = tf.placeholder(dtype)
self.vars = utils.init_var_map(init_vars, init_path)
w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)]
b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)]
xw = [
tf.sparse_tensor_dense_matmul(self.X[i], w0[i])
for i in range(num_inputs)
]
x = tf.concat([xw[i] + b0[i] for i in range(num_inputs)], 1)
## normalize
fmX = tf.sparse_add(self.X[0], self.X[1])
for i in range(2, num_inputs):
fmX = tf.sparse_add(fmX, self.X[i])
Xnorm = tf.reshape(1.0 / tf.sparse_reduce_sum(fmX, 1),
[-1, output_dim])
l = tf.nn.dropout(utils.activate(x, layer_acts[0]),
self.layer_keeps[0])
for i in range(1, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
eprint(l.get_shape(), wi.get_shape(), bi.get_shape())
l = tf.nn.dropout(
utils.activate(tf.matmul(l, wi) + bi, layer_acts[i]),
self.layer_keeps[i])
## FM linear part
fmb = self.vars['bias']
fmw = self.vars['linear']
Xw = tf.sparse_tensor_dense_matmul(self.B, fmw)
## cross term
# XV, shape: input_dim*k
fmXV = tf.add_n(xw)
XV_square = tf.square(fmXV)
eprint(XV_square.get_shape())
# X^2 * V^2, shape: input_dim*k
fmX2 = [
tf.SparseTensor(self.X[i].indices, tf.square(self.X[i].values),
tf.to_int64(tf.shape(self.X[i])))
for i in range(num_inputs)
]
fmV2 = [tf.square(w0[i]) for i in range(num_inputs)]
fmX2V2 = [
tf.sparse_tensor_dense_matmul(fmX2[i], fmV2[i])
for i in range(num_inputs)
]
X2V2 = tf.add_n(fmX2V2)
eprint(X2V2.get_shape())
# 1/2 * row_sum(XV_square - X2V2), shape: input_dim*1
p = 0.5 * Xnorm * tf.reshape(tf.reduce_sum(XV_square - X2V2, 1),
[-1, output_dim])
## logits
logits = tf.reshape(l + Xw + fmb + p, [-1])
## predict
self.y_prob = tf.sigmoid(logits)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=self.y)) + \
l2_w * tf.nn.l2_loss(Xw)
if layer_l2 is not None:
self.loss += layer_l2[0] * tf.nn.l2_loss(tf.concat(xw, 1))
for i in range(1, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
self.global_step = _variable_on_cpu(
'global_step', [],
initializer=tf.constant_initializer(0),
trainable=False)
if sync:
self.optimizer = utils.get_sync_optimizer(opt_algo, learning_rate,
workers)
else:
self.optimizer = utils.get_optimizer(opt_algo, learning_rate)
self.train_op = self.optimizer.minimize(self.loss,
global_step=self.global_step)
self.summary_op = tf.summary.merge_all()
def make_app():
app = sandman2.get_app(config.SQLA_URI, Base=utils.AutomapModel)
app.json_encoder = utils.APIJSONEncoder
app.config['CASE_INSENSITIVE'] = config.CASE_INSENSITIVE
app.config['BASIC_AUTH_USERNAME'] = os.environ.get(
'AUTOAPI_ADMIN_USERNAME', '')
app.config['BASIC_AUTH_PASSWORD'] = os.environ.get(
'AUTOAPI_ADMIN_PASSWORD', '')
CORS(app)
basic_auth = BasicAuth(app)
@app.before_request
def refresh():
tables = utils.get_tables()
if tables != app.config['SQLALCHEMY_TABLES']:
utils.refresh_tables()
app.config['SQLALCHEMY_TABLES'] = tables
@app.before_request
def protect_admin():
if request.path.startswith('/admin/'):
if not basic_auth.authenticate():
return basic_auth.challenge()
class RefreshTables(View):
task_name = 'refresh'
def dispatch_request(self):
underway = RefreshLog.refresh_underway()
if underway:
return '''Refresh begun at {} still underway.
Now: {}; timeout set for {} seconds'''.format(
underway.begun_at, datetime.now(),
config.REFRESH_TIMEOUT_SECONDS)
try:
subprocess.Popen(['invoke', self.task_name, ])
except Exception as e:
print('Problem with table refresh:')
print(e)
return 'Table refresh requested.'
class QuickRefreshTables(RefreshTables):
task_name = 'quick_refresh'
app.add_url_rule('/refresh/', view_func=RefreshTables.as_view('refresh'))
app.add_url_rule('/quick_refresh/',
view_func=QuickRefreshTables.as_view('quick_refresh'))
aws_blueprint = aws.make_blueprint()
app.register_blueprint(aws_blueprint)
docs_blueprint = swagger.make_blueprint(app)
app.register_blueprint(docs_blueprint)
with app.app_context():
app.config['SQLALCHEMY_TABLES'] = utils.get_tables()
utils.activate()
refresh_log_db.init_app(app)
refresh_log_db.create_all(app=app)
return app
def __init__(self,
field_sizes=None,
embed_size=10,
layer_sizes=None,
layer_acts=None,
drop_out=None,
embed_l2=None,
layer_l2=None,
init_path=None,
opt_algo='gd',
learning_rate=1e-2,
random_seed=None,
layer_norm=True):
Model.__init__(self)
init_vars = []
num_inputs = len(field_sizes)
for i in range(num_inputs):
init_vars.append(('embed_%d' % i, [field_sizes[i],
embed_size], 'xavier', dtype))
node_in = num_inputs * embed_size + embed_size * embed_size
for i in range(len(layer_sizes)):
init_vars.append(('w%d' % i, [node_in,
layer_sizes[i]], 'xavier', dtype))
init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype))
node_in = layer_sizes[i]
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.keep_prob_train = 1 - np.array(drop_out)
self.keep_prob_test = np.ones_like(drop_out)
self.layer_keeps = tf.placeholder(dtype)
self.vars = utils.init_var_map(init_vars, init_path)
w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)]
xw = tf.concat([
tf.sparse_tensor_dense_matmul(self.X[i], w0[i])
for i in range(num_inputs)
], 1)
z = tf.reduce_sum(tf.reshape(xw, [-1, num_inputs, embed_size]), 1)
op = tf.reshape(
tf.matmul(tf.reshape(z, [-1, embed_size, 1]),
tf.reshape(z, [-1, 1, embed_size])),
[-1, embed_size * embed_size])
if layer_norm:
# x_mean, x_var = tf.nn.moments(xw, [1], keep_dims=True)
# xw = (xw - x_mean) / tf.sqrt(x_var)
# x_g = tf.Variable(tf.ones([num_inputs * embed_size]), name='x_g')
# x_b = tf.Variable(tf.zeros([num_inputs * embed_size]), name='x_b')
# x_g = tf.Print(x_g, [x_g[:10], x_b])
# xw = xw * x_g + x_b
p_mean, p_var = tf.nn.moments(op, [1], keep_dims=True)
op = (op - p_mean) / tf.sqrt(p_var)
p_g = tf.Variable(tf.ones([embed_size**2]), name='p_g')
p_b = tf.Variable(tf.zeros([embed_size**2]), name='p_b')
# p_g = tf.Print(p_g, [p_g[:10], p_b])
op = op * p_g + p_b
l = tf.concat([xw, op], 1)
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
l = tf.nn.dropout(
utils.activate(tf.matmul(l, wi) + bi, layer_acts[i]),
self.layer_keeps[i])
l = tf.squeeze(l)
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l,
labels=self.y))
if layer_l2 is not None:
self.loss += embed_l2 * tf.nn.l2_loss(tf.concat(w0, 0))
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
self.optimizer = utils.get_optimizer(opt_algo, learning_rate)
self.train_op = self.optimizer.minimize(self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def __init__(self,
layer_sizes=None,
layer_acts=None,
drop_out=None,
layer_l2=None,
kernel_l2=None,
init_path=None,
opt_algo='gd',
learning_rate=1e-2,
random_seed=None):
Model.__init__(self)
init_vars = []
num_inputs = len(layer_sizes[0])
factor_order = layer_sizes[1]
for i in range(num_inputs):
layer_input = layer_sizes[0][i]
layer_output = factor_order
init_vars.append(('w0_%d' % i, [layer_input,
layer_output], 'tnormal', dtype))
init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype))
init_vars.append(('w1', [num_inputs * factor_order,
layer_sizes[2]], 'tnormal', dtype))
init_vars.append(('k1', [factor_order * factor_order,
layer_sizes[2]], 'tnormal', dtype))
init_vars.append(('b1', [layer_sizes[2]], 'zero', dtype))
for i in range(2, len(layer_sizes) - 1):
layer_input = layer_sizes[i]
layer_output = layer_sizes[i + 1]
init_vars.append((
'w%d' % i,
[layer_input, layer_output],
'tnormal',
))
init_vars.append(('b%d' % i, [layer_output], 'zero', dtype))
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.keep_prob_train = 1 - np.array(drop_out)
self.keep_prob_test = np.ones_like(drop_out)
self.layer_keeps = tf.placeholder(dtype)
self.vars = utils.init_var_map(init_vars, init_path)
w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)]
b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)]
xw = [
tf.sparse_tensor_dense_matmul(self.X[i], w0[i])
for i in range(num_inputs)
]
x = tf.concat([xw[i] + b0[i] for i in range(num_inputs)], 1)
l = tf.nn.dropout(utils.activate(x, layer_acts[0]),
self.layer_keeps[0])
w1 = self.vars['w1']
k1 = self.vars['k1']
b1 = self.vars['b1']
z = tf.reduce_sum(tf.reshape(l, [-1, num_inputs, factor_order]), 1)
p = tf.reshape(
tf.matmul(tf.reshape(z, [-1, factor_order, 1]),
tf.reshape(z, [-1, 1, factor_order])),
[-1, factor_order * factor_order])
l = tf.nn.dropout(
utils.activate(
tf.matmul(l, w1) + tf.matmul(p, k1) + b1, layer_acts[1]),
self.layer_keeps[1])
for i in range(2, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
l = tf.nn.dropout(
utils.activate(tf.matmul(l, wi) + bi, layer_acts[i]),
self.layer_keeps[i])
l = tf.reshape(l, [-1])
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l,
labels=self.y))
if layer_l2 is not None:
# for i in range(num_inputs):
self.loss += layer_l2[0] * tf.nn.l2_loss(tf.concat(xw, 1))
for i in range(1, len(layer_sizes) - 1):
wi = self.vars['w%d' % i]
# bi = self.vars['b%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
if kernel_l2 is not None:
self.loss += kernel_l2 * tf.nn.l2_loss(k1)
self.optimizer = utils.get_optimizer(opt_algo, learning_rate,
self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def __init__(self, field_sizes=None, embed_size=10, filter_sizes=None, layer_acts=None, drop_out=None,
init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None):
Model.__init__(self)
init_vars = []
num_inputs = len(field_sizes)
for i in range(num_inputs):
init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype))
init_vars.append(('f1', [embed_size, filter_sizes[0], 1, 2], 'xavier', dtype))
init_vars.append(('f2', [embed_size, filter_sizes[1], 2, 2], 'xavier', dtype))
init_vars.append(('w1', [2 * 3 * embed_size, 1], 'xavier', dtype))
init_vars.append(('b1', [1], 'zero', dtype))
print('init_vars: ', init_vars)
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.keep_prob_train = 1 - np.array(drop_out)
self.keep_prob_test = np.ones_like(drop_out)
self.layer_keeps = tf.placeholder(dtype)
self.vars = utils.init_var_map(init_vars, init_path)
w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)]
xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1)
l = xw
l = tf.transpose(tf.reshape(l, [-1, num_inputs, embed_size, 1]), [0, 2, 1, 3]) # 变为 16 x 10 矩阵
f1 = self.vars['f1']
l = tf.nn.conv2d(l, f1, [1, 1, 1, 1], 'SAME')
l = tf.transpose(
utils.max_pool_4d(
tf.transpose(l, [0, 1, 3, 2]),
int(num_inputs / 2)),
[0, 1, 3, 2])
f2 = self.vars['f2']
l = tf.nn.conv2d(l, f2, [1, 1, 1, 1], 'SAME')
l = tf.transpose(
utils.max_pool_4d(
tf.transpose(l, [0, 1, 3, 2]), 3),
[0, 1, 3, 2])
l = tf.nn.dropout(
utils.activate(
tf.reshape(l, [-1, embed_size * 3 * 2]),
layer_acts[0]),
self.layer_keeps[0])
w1 = self.vars['w1']
b1 = self.vars['b1']
l = tf.matmul(l, w1) + b1
l = tf.squeeze(l)
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y))
self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def write_html(self,
username,
organization,
site_code,
site_name,
forecast_model_name,
forecast_method,
forecast_model_params,
forecast_method_params,
filename=None,
htmlpage=None,
language='en'):
activate(language)
if self.y.mode == 'p':
frequency = 'fiveday'
elif self.y.mode == 'd':
frequency = 'decade'
elif self.y.mode == 'm':
frequency = 'monthly'
page = self.load_template_file()
scatter_plot = PlotUtils.plot_ts_comparison(
self.y_adj.timeseries,
self.forecast.timeseries,
frequency,
language=language,
)
scaled_error_title = _('Scaled Error [RMSE/STDEV]')
scaled_error_plot = PlotUtils.plot_rel_error(self.rel_error,
frequency,
title=scaled_error_title)
scaled_error_table = self.rel_error_table(frequency)
p_plot_title = _('P% Plot')
p_plot_plot = PlotUtils.plot_p(self.p, frequency, title=p_plot_title)
p_plot_table = self.p_plot_table(frequency)
quality_assessment_table = self.summary_table(frequency)
report_data = {
'SITE_INFO':
_('Station: {code} - {name}').format(code=to_str(site_code),
name=to_str(site_name)),
'USERNAME':
username,
'ORGANIZATION':
organization,
'TITLE':
_('Forecast Model Training Report'),
'REPORT_DATE':
format_date(format='long', locale=language),
'PLOTS_HEADER':
_('{frequency} Forecast Model Quality Assessment').format(
frequency=frequency.capitalize()),
'SCATTER_PLOT_LABEL':
_('Scatter Plot: Observed versus Predicted values'),
'SCALED_ERROR_LABEL':
scaled_error_title,
'P_PLOT_LABEL':
p_plot_title,
'QUALITY_ASSESSMENT_LABEL':
_('Quality Assessment'),
'SCATTER_PLOT_IMAGE':
scatter_plot,
'SCALED_ERROR_PLOT_IMAGE':
scaled_error_plot,
'SCALED_ERROR_TABLE':
scaled_error_table,
'P_PLOT_IMAGE':
p_plot_plot,
'P_PLOT_TABLE':
p_plot_table,
'QUALITY_ASSESSMENT_TABLE':
quality_assessment_table,
'FORECAST_MODEL_INFO':
_('Forecast model info:'),
'FORECAST_MODEL_NAME':
_('Name: ') + forecast_model_name,
'FORECAST_METHOD':
_('Method: ') + forecast_method,
'FORECAST_MODEL_PARAMS':
_('Model parameters: ') + str(forecast_model_params),
'FORECAST_METHOD_PARAMS':
_('Method parameters: ') + str(forecast_method_params),
}
report_data.update(self.get_spacers(frequency, language))
self.encode_utf8(report_data)
if filename:
htmlpage = open(filename, 'w')
htmlpage.write(page.safe_substitute(**report_data))
htmlpage.close()
return filename
elif htmlpage:
htmlpage.write(page.safe_substitute(**report_data))
return htmlpage
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None,
embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None,
layer_norm=True):
Model.__init__(self)
init_vars = []
num_inputs = len(field_sizes)
for i in range(num_inputs):
init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype))
node_in = num_inputs * embed_size + embed_size * embed_size
for i in range(len(layer_sizes)):
init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype))
init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype))
node_in = layer_sizes[i]
self.graph = tf.Graph()
with self.graph.as_default():
if random_seed is not None:
tf.set_random_seed(random_seed)
self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)]
self.y = tf.placeholder(dtype)
self.keep_prob_train = 1 - np.array(drop_out)
self.keep_prob_test = np.ones_like(drop_out)
self.layer_keeps = tf.placeholder(dtype)
self.vars = utils.init_var_map(init_vars, init_path)
w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)]
xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1)
z = tf.reduce_sum(tf.reshape(xw, [-1, num_inputs, embed_size]), 1)
op = tf.reshape(
tf.matmul(tf.reshape(z, [-1, embed_size, 1]),
tf.reshape(z, [-1, 1, embed_size])),
[-1, embed_size * embed_size])
if layer_norm:
# x_mean, x_var = tf.nn.moments(xw, [1], keep_dims=True)
# xw = (xw - x_mean) / tf.sqrt(x_var)
# x_g = tf.Variable(tf.ones([num_inputs * embed_size]), name='x_g')
# x_b = tf.Variable(tf.zeros([num_inputs * embed_size]), name='x_b')
# x_g = tf.Print(x_g, [x_g[:10], x_b])
# xw = xw * x_g + x_b
p_mean, p_var = tf.nn.moments(op, [1], keep_dims=True)
op = (op - p_mean) / tf.sqrt(p_var)
p_g = tf.Variable(tf.ones([embed_size**2]), name='p_g')
p_b = tf.Variable(tf.zeros([embed_size**2]), name='p_b')
# p_g = tf.Print(p_g, [p_g[:10], p_b])
op = op * p_g + p_b
l = tf.concat([xw, op], 1)
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
bi = self.vars['b%d' % i]
l = tf.nn.dropout(
utils.activate(
tf.matmul(l, wi) + bi,
layer_acts[i]),
self.layer_keeps[i])
l = tf.squeeze(l)
self.y_prob = tf.sigmoid(l)
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y))
if layer_l2 is not None:
self.loss += embed_l2 * tf.nn.l2_loss(tf.concat(w0, 0))
for i in range(len(layer_sizes)):
wi = self.vars['w%d' % i]
self.loss += layer_l2[i] * tf.nn.l2_loss(wi)
self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
tf.global_variables_initializer().run(session=self.sess)
def write_html(self,
username,
organization,
site_code,
site_name,
forecast_model_name,
forecast_method,
forecast_model_params,
forecast_method_params,
filename=None,
htmlpage=None,
language='en'):
activate(language)
page = self.load_template_file()
timeseries_plot = self.__encode_figure(self.plot_timeseries())
quality_assessment_table = self.__table_summary()
model_table = self.__model_htmltable()
report_data = {
'SITE_INFO':
_('Station: {code} - {name}').format(code=to_str(site_code),
name=to_str(site_name)),
'USERNAME':
username,
'ORGANIZATION':
organization,
'TITLE':
_('Forecast Model Training Report'),
'REPORT_DATE':
format_date(format='long', locale=language),
'PLOTS_HEADER':
_('{frequency} Forecast Model Quality Assessment').format(
frequency=_('seasonal').capitalize()),
'MODEL_TABLE_LABEL':
_('Model table'),
'MODEL_TABLE':
model_table,
'QUALITY_ASSESSMENT_LABEL':
_('Quality Assessment'),
'QUALITY_ASSESSMENT_TABLE':
quality_assessment_table,
'TIMESERIES_LABEL':
_('Timeseries plot'),
'TIMESERIES_PLOT':
timeseries_plot,
'FORECAST_MODEL_INFO':
_('Forecast model info:'),
'FORECAST_MODEL_NAME':
_('Name: ') + forecast_model_name,
'FORECAST_METHOD':
_('Method: ') + forecast_method,
'FORECAST_MODEL_PARAMS':
_('Model parameters: ') + str(forecast_model_params),
'FORECAST_METHOD_PARAMS':
_('Method parameters: ') + str(forecast_method_params),
}
self.encode_utf8(report_data)
if filename:
htmlpage = open(filename, 'w')
htmlpage.write(page.safe_substitute(**report_data))
htmlpage.close()
return filename
elif htmlpage:
htmlpage.write(page.safe_substitute(**report_data))
return htmlpage
