def make_app(): app.json_encoder = utils.APIJSONEncoder app.config["CASE_INSENSITIVE"] = config.CASE_INSENSITIVE app.config["SQLALCHEMY_DATABASE_URI"] = config.SQLA_URI app.config["BASIC_AUTH_USERNAME"] = os.environ.get("AUTOAPI_ADMIN_USERNAME", "") app.config["BASIC_AUTH_PASSWORD"] = os.environ.get("AUTOAPI_ADMIN_PASSWORD", "") CORS(app) basic_auth = BasicAuth(app) @app.before_request def refresh(): tables = utils.get_tables() if tables != app.config["SQLALCHEMY_TABLES"]: utils.refresh_tables() app.config["SQLALCHEMY_TABLES"] = tables app.config["SQLALCHEMY_TABLES"] = utils.get_tables() @app.before_request def protect_admin(): if request.path.startswith("/admin/"): if not basic_auth.authenticate(): return basic_auth.challenge() blueprint = aws.make_blueprint() app.register_blueprint(blueprint) utils.activate(admin=True) route = os.path.join("/api-program", config.API_NAME) container = DispatcherMiddleware(app.wsgi_app, {route: app}) return container
def __init__(self, layer_sizes=None, layer_acts=None, layer_keeps=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None): init_vars = [] num_inputs = len(layer_sizes[0]) factor_order = layer_sizes[1] for i in range(num_inputs): layer_input = layer_sizes[0][i] layer_output = factor_order init_vars.append(('w0_%d' % i, [layer_input, layer_output], 'tnormal', dtype)) init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype)) init_vars.append(('w1', [num_inputs * factor_order, layer_sizes[2]], 'tnormal', dtype)) init_vars.append(('b1', [layer_sizes[2]], 'zero', dtype)) for i in range(2, len(layer_sizes) - 1): layer_input = layer_sizes[i] layer_output = layer_sizes[i + 1] init_vars.append(('w%d' % i, [layer_input, layer_output], 'tnormal', dtype)) init_vars.append(('b%d' % i, [layer_output], 'zero', dtype)) self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)] b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)] l = tf.nn.dropout( utils.activate( tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) + b0[i] for i in range(num_inputs)], 1), layer_acts[0]), layer_keeps[0]) for i in range(1, len(layer_sizes) - 1): wi = self.vars['w%d' % i] bi = self.vars['b%d' % i] l = tf.nn.dropout( utils.activate( tf.matmul(l, wi) + bi, layer_acts[i]), layer_keeps[i]) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) if layer_l2 is not None: for i in range(num_inputs): self.loss += layer_l2[0] * tf.nn.l2_loss(w0[i]) for i in range(1, len(layer_sizes) - 1): wi = self.vars['w%d' % i] # bi = self.vars['b%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(wi) self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def _back_propogate(self, layers, prediction, labels): delta = prediction - labels for i in reversed(range(self._n_layers)): if i is 0: break dW = np.matmul( delta, utils.activate(self._layer_activations[i - 1], layers[i - 1]).T) dB = np.sum(delta, axis=1) dB = np.reshape(dB, (dB.shape[0], -1)) delta = utils.calc_der(self._layer_activations[i - 1], layers[i - 1]) * (np.matmul( self._weights[i - 1].T, delta)) self._weights[i - 1] -= self._alpha * dW if self._bias_weights[i - 1] is not None: self._bias_weights[i - 1] -= self._alpha * dB
def _gradient_descent(self, X, T): # No. of training examples n = X.shape[0] # No. of features m = X.shape[1] if self._weights is None: if self._initialisation == 'gaussian': self._weights = np.random.randn(m, 1) else: self._weights = np.random.uniform(-1, 1, (m, 1)) tmp = np.matmul(X, self._weights) tmp[tmp > 15] = 15 tmp[tmp < -15] = -15 Y = utils.activate('sigmoid', tmp) # cost = (1/n) * np.sum (-(T * np.log(Y) + (1 - T) * np.log(1 - Y)), axis = 0) cost = np.sum(-(T * np.log(Y) + (1 - T) * np.log(1 - Y)), axis=0) if self._regularisation == 'L1': cost += self._lambda_reg * np.sum(abs(self._weights), axis=0) if self._regularisation == 'L2': cost += self._lambda_reg * np.sum(self._weights**2) diff = Y - T self.train_accuracy = self._accuracy(Y >= 0.5, T) self.costs.append(cost) # print(cost) self._update_weights(X, diff)
def __init__(self, field_sizes=None, embed_size=10, filter_sizes=None, layer_acts=None, drop_out=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None): Model.__init__(self) init_vars = [] num_inputs = len(field_sizes) for i in range(num_inputs): init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) init_vars.append(('f1', [embed_size, filter_sizes[0], 1, 2], 'xavier', dtype)) init_vars.append(('f2', [embed_size, filter_sizes[1], 2, 2], 'xavier', dtype)) init_vars.append(('w1', [2 * 3 * embed_size, 1], 'xavier', dtype)) init_vars.append(('b1', [1], 'zero', dtype)) self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = init_var_map(init_vars, init_path) w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)] xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1) l = xw l = tf.transpose(tf.reshape(l, [-1, num_inputs, embed_size, 1]), [0, 2, 1, 3]) f1 = self.vars['f1'] l = tf.nn.conv2d(l, f1, [1, 1, 1, 1], 'SAME') l = tf.transpose( max_pool_4d( tf.transpose(l, [0, 1, 3, 2]), int(num_inputs / 2)), [0, 1, 3, 2]) f2 = self.vars['f2'] l = tf.nn.conv2d(l, f2, [1, 1, 1, 1], 'SAME') l = tf.transpose( max_pool_4d( tf.transpose(l, [0, 1, 3, 2]), 3), [0, 1, 3, 2]) l = tf.nn.dropout( activate( tf.reshape(l, [-1, embed_size * 3 * 2]), layer_acts[0]), self.layer_keeps[0]) w1 = self.vars['w1'] b1 = self.vars['b1'] l = tf.matmul(l, w1) + b1 l = tf.squeeze(l) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) self.optimizer = get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def eval(self, data, dropout_keep_prob=1.0): data = self.__flatten_data_shape(data) logits = tf.matmul(data, self.weights) + self.biases if self.activation_func == '': return logits activations = utils.activate(logits, self.activation_func) activations = tf.nn.dropout(activations, dropout_keep_prob) return activations
def _forward_propogate(self, input_vector, labels): layers = [] layer = np.reshape(input_vector, (input_vector.shape[0], -1)) layers.append(layer) for i in range(self._n_layers): if i is 0: continue active_layer = utils.activate(self._layer_activations[i - 1], layer) drop = np.random.rand( active_layer.shape[0], active_layer.shape[1]) < self._keep_probs[i - 1] active_layer *= drop active_layer /= self._keep_probs[i - 1] layer = np.matmul(self._weights[i - 1], active_layer) if self._bias_weights[i - 1] is not None: layer += self._bias_weights[i - 1] layers.append(layer) layers[-1][layers[-1] > 15] = 15 layers[-1][layers[-1] < -15] = -15 # print(layers) prediction = utils.activate( self._layer_activations[self._n_layers - 1], layer) cost = np.sum(-(labels * np.log(prediction) + (1 - labels) * np.log(1 - prediction)), axis=1) self.costs.append(cost) # print(cost) self.train_accuracy = np.sum(((prediction >= 0.5) - labels)**2, axis=1) self.train_accuracy = (1 - self.train_accuracy / prediction.shape[1]) * 100 return layers, prediction
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None, embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None): Model.__init__(self) init_vars = [] num_inputs = len(field_sizes) print('num_inputs:{0}\\t\tlayer_size:{1}'.format(num_inputs, layer_sizes)) for i in range(num_inputs): init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) # 为每个特征值初始化一个长度为10的向量 node_in = num_inputs * embed_size # 将每个特征embeding 为10维的向量, 总共16个特征,所以是160个输入 网络为[160,500,1] for i in range(len(layer_sizes)): init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype)) init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype)) node_in = layer_sizes[i] print('init_vars:', init_vars) self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)] xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1) # 将每个特征的隐含向量连起来,组成网络的输入,160维 l = xw for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] bi = self.vars['b%d' % i] print('第{0}个隐藏层l.shape, wi.shape, bi.shape'.format(i), l.shape, wi.shape, bi.shape) l = tf.nn.dropout( utils.activate( tf.matmul(l, wi) + bi, layer_acts[i]), self.layer_keeps[i]) l = tf.squeeze(l) # 从tensor中删除所有大小是1的维度 self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) if layer_l2 is not None: self.loss += embed_l2 * tf.nn.l2_loss(xw) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(wi) self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None, embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None): Model.__init__(self) init_vars = [] num_inputs = len(field_sizes) print('num_inputs:{0}\\t\tlayer_size:{1}'.format(num_inputs, layer_sizes)) for i in range(num_inputs): init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) # 为每个特征值初始化一个长度为10的向量 node_in = num_inputs * embed_size # 将每个特征embeding 为10维的向量, 总共16个特征,所以是160个输入 网络为[160, 500, 1] for i in range(len(layer_sizes)): init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype)) init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype)) node_in = layer_sizes[i] print('init_vars:', init_vars) self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)] xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1) # 将每个特征的隐含向量连起来,组成网络的输入,160维 l = xw for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] bi = self.vars['b%d' % i] print('第{0}个隐藏层l.shape, wi.shape, bi.shape'.format(i), l.shape, wi.shape, bi.shape) l = tf.nn.dropout( utils.activate( tf.matmul(l, wi) + bi, layer_acts[i]), self.layer_keeps[i]) l = tf.squeeze(l) # 从tensor中删除所有大小是1的维度 self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) if layer_l2 is not None: self.loss += embed_l2 * tf.nn.l2_loss(xw) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(wi) self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def forward(net_type, depth, act_func, x_vec, _vars): if net_type == "nn": W, b = _vars a = [0] * (depth - 1) z = [0] * (depth - 1) z[0] = x_vec for _i in range(depth - 2): a[_i + 1] = tf.matmul(z[_i], W[_i]) + b[_i] z[_i + 1] = activate(act_func, a[_i + 1]) y_hat = tf.matmul(z[depth - 2], W[depth - 2]) + b[depth - 2] return y_hat
def predict(self, features): features = np.reshape(features, (features.shape[0], -1)) features = np.concatenate((np.ones((features.shape[0], 1)), features), axis=1) tmp = np.matmul(features, self._weights) tmp[tmp > 15] = 15 tmp[tmp < -15] = -15 prediction = utils.activate('sigmoid', tmp) return prediction >= 0.5
def convolution(self, features, kernels, biases): """ Implements a single convolution layer :param features: single or a list of features on which convolution needs to be performed :type features: numpy nd array :param kernels: an array of filter kernels. Only square kernels are supported as of now :type kernels: numpy nd array :param biases: an array of biases, one for each kernel :type biases : numpy 1d array :returns: numpy nd array, output features """ n_features = features.shape[0] # number of input feature arrays feat_rows = features.shape[1] # rows in the input features feat_cols = features.shape[2] # cols in the input features n_kernel_sets = kernels.shape[0] # number of kernel sets, one set n_kernels = kernels.shape[1] # no. of kernels per set kernel_dim = kernels.shape[2] # dimension of a kernel # dimensions of a single convolved feature array conv_feat_rows = feat_rows - kernel_dim + 1 conv_feat_cols = feat_cols - kernel_dim + 1 convolved_features = [] # resultant array # iterate through all sets of filter/kernels. for ker_set_idx in xrange(n_kernel_sets): # initialize a single output feature out_feature = np.zeros((conv_feat_rows, conv_feat_cols)) for ker_idx in xrange(n_kernels): # an output feature is the sum of all convolved individual # input features with a particular kernel out_feature += convolve2d(features[ker_idx], kernels[ker_set_idx][ker_idx], mode='valid') # add bias to the resultant feature out_feature += biases[ker_set_idx] # append the resultant feature array convolved_features.append(activate(out_feature, method='sigmoid')) return np.array(convolved_features, dtype=np.float32)
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None, embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None): Model.__init__(self) init_vars = [] num_inputs = len(field_sizes) for i in range(num_inputs): init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) node_in = num_inputs * embed_size for i in range(len(layer_sizes)): init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype)) init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype)) node_in = layer_sizes[i] self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = init_var_map(init_vars, init_path) w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)] xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1) l = xw for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] bi = self.vars['b%d' % i] print(l.shape, wi.shape, bi.shape) l = tf.nn.dropout( activate( tf.matmul(l, wi) + bi, layer_acts[i]), self.layer_keeps[i]) l = tf.squeeze(l) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) if layer_l2 is not None: self.loss += embed_l2 * tf.nn.l2_loss(xw) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(wi) self.optimizer = get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def predict(self, input_vector): layer = np.reshape(input_vector, (input_vector.shape[0], -1)) for i in range(self._n_layers): if i is 0: continue layer = np.matmul( self._weights[i - 1], utils.activate(self._layer_activations[i - 1], layer)) if self._bias_weights[i - 1] is not None: layer += self._bias_weights[i - 1] layer[layer > 15] = 15 layer[layer < -15] = -15 prediction = utils.activate( self._layer_activations[self._n_layers - 1], layer) return prediction >= 0.5
def make_app(): app = sandman2.get_app(config.SQLA_URI, Base=utils.AutomapModel) app.json_encoder = utils.APIJSONEncoder app.config['CASE_INSENSITIVE'] = config.CASE_INSENSITIVE app.config['BASIC_AUTH_USERNAME'] = os.environ.get('AUTOAPI_ADMIN_USERNAME', '') app.config['BASIC_AUTH_PASSWORD'] = os.environ.get('AUTOAPI_ADMIN_PASSWORD', '') CORS(app) basic_auth = BasicAuth(app) @app.before_request def refresh(): tables = utils.get_tables() if tables != app.config['SQLALCHEMY_TABLES']: utils.refresh_tables() app.config['SQLALCHEMY_TABLES'] = tables @app.before_request def protect_admin(): if request.path.startswith('/admin/'): if not basic_auth.authenticate(): return basic_auth.challenge() aws_blueprint = aws.make_blueprint() app.register_blueprint(aws_blueprint) docs_blueprint = swagger.make_blueprint(app) app.register_blueprint(docs_blueprint) route = os.path.join('/api-program', config.API_NAME) container = DispatcherMiddleware(app.wsgi_app, {route: app}) with app.app_context(): app.config['SQLALCHEMY_TABLES'] = utils.get_tables() utils.activate() return app, container
def execute(self, features): """ Executes the layer, ie, constructs the output feature array as activations of the layer absed on the stored weights :params features: input features :type features : numpy 1d array """ # perform dot product operation on input against the weights # to get the effective input to each neuron in the layer weighted_input = np.dot(features, self.weights) + self.biases # the output is the activation performed on weighted_input self.out = activate(weighted_input, method='sigmoid')
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None, embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None): Model.__init__(self) init_vars = [] num_inputs = len(field_sizes) for i in range(num_inputs): init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) num_pairs = int(num_inputs * (num_inputs - 1) / 2) node_in = num_inputs * embed_size + num_pairs # node_in = num_inputs * (embed_size + num_inputs) for i in range(len(layer_sizes)): init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype)) init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype)) node_in = layer_sizes[i] self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)] xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1) xw3d = tf.reshape(xw, [-1, num_inputs, embed_size]) row = [] col = [] for i in range(num_inputs-1): for j in range(i+1, num_inputs): row.append(i) col.append(j) # batch * pair * k p = tf.transpose( # pair * batch * k tf.gather( # num * batch * k tf.transpose( xw3d, [1, 0, 2]), row), [1, 0, 2]) # batch * pair * k q = tf.transpose( tf.gather( tf.transpose( xw3d, [1, 0, 2]), col), [1, 0, 2]) p = tf.reshape(p, [-1, num_pairs, embed_size]) q = tf.reshape(q, [-1, num_pairs, embed_size]) ip = tf.reshape(tf.reduce_sum(p * q, [-1]), [-1, num_pairs]) # simple but redundant # batch * n * 1 * k, batch * 1 * n * k # ip = tf.reshape( # tf.reduce_sum( # tf.expand_dims(xw3d, 2) * # tf.expand_dims(xw3d, 1), # 3), # [-1, num_inputs**2]) l = tf.concat([xw, ip], 1) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] bi = self.vars['b%d' % i] l = tf.nn.dropout( utils.activate( tf.matmul(l, wi) + bi, layer_acts[i]), self.layer_keeps[i]) l = tf.squeeze(l) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) if layer_l2 is not None: self.loss += embed_l2 * tf.nn.l2_loss(xw) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(wi) self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def __init__(self, feature2field , layer_sizes, layer_acts, drop_out, learning_rate=1e-2, l2_reg = 0, feature_size = None, field_size = None, k = 40, opt_algo = 'adam', train = True): # feature_size : feature number # field_size : field number # k : latent vector dimension Model.__init__(self) layers = [int(e) for e in layer_sizes.split(',')] dropout = [float(e) for e in drop_out.split(',')] layer_active_func = [e for e in layer_acts.split(',')] self.X = tf.placeholder(dtype=dtype, shape=[None, feature_size], name='input') self.y = tf.placeholder(dtype=dtype, shape=[None, ], name='label') self.keep_prob = tf.placeholder(dtype=dtype) with tf.variable_scope('linear_layer'): # w .* x + b self.b = tf.get_variable(name='bias', initializer=tf.constant(0.5), dtype=dtype) # tf.zeros_initializer() self.w1 = tf.get_variable(name='w1', shape=[feature_size], initializer=tf.truncated_normal_initializer(mean=0, stddev=1e-2), dtype=dtype) self.linear_terms = tf.reduce_sum(tf.multiply(self.w1, self.X), 1) + self.b with tf.variable_scope('field_aware_interaction_layer'): # sum(<vi_fj, vj_fi> * x_i * x_j) self.nfk = tf.get_variable('nfk', shape=[feature_size, field_size, k], dtype=dtype, initializer=tf.truncated_normal_initializer(mean=0, stddev=0.01)) self.field_aware_interaction_terms = tf.constant(0, dtype=dtype) for i in range(feature_size): for j in range(i + 1, feature_size): self.field_aware_interaction_terms += tf.multiply( tf.reduce_sum(tf.multiply(self.nfk[i, feature2field[j]], self.nfk[j, feature2field[i]])), tf.multiply(self.X[:, i], self.X[:, j]) ) init_deep_layer_vars = [] input_dim = feature_size for i in range(len(layers)): output_dim = layers[i] init_deep_layer_vars.append(('deepW_%d' % i, [input_dim, output_dim], 'xavier', dtype)) init_deep_layer_vars.append(('deepB_%d' % i, [output_dim], 'zero', dtype)) input_dim = layers[i] init_deep_layer_vars.append(('outW', [layers[-1], 1], 'xavier', dtype)) init_deep_layer_vars.append(('outB', [1], 'zero', dtype)) self.deepVars = init_var_map(init_deep_layer_vars) with tf.variable_scope("Deep-part"): hidden = self.X for i in range(len(layers)): if train: hidden = tf.nn.dropout( # h_i = W_i * x + b_i activate(tf.matmul(hidden, self.deepVars['deepW_%d' % i]) + self.deepVars['deepB_%d' % i], layer_active_func[i]), dropout[i]) else: hidden = activate(tf.matmul(hidden, self.deepVars['deepW_%d' % i]) + self.deepVars['deepB_%d' % i], layer_active_func[i]) #self.a=hidden;self.aa=self.deepVars['outW'];self.aaa=tf.matmul(hidden, self.deepVars['outW']);self.aaaa=self.deepVars['outB'] self.deepOut = tf.matmul(hidden, self.deepVars['outW']) + self.deepVars['outB'] self.deepOut = tf.reshape(self.deepOut, shape=[-1]) # tf.reshape() 防止 python 自带的广播机制算出奇怪的值 with tf.variable_scope("DeepFM-out"): self.out_sum = self.linear_terms + self.field_aware_interaction_terms + self.deepOut self.pred_prob = tf.sigmoid(self.out_sum) # ------bulid loss------ self.loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=self.out_sum, labels=self.y)) + \ l2_reg * tf.nn.l2_loss(self.w1) + \ l2_reg * tf.nn.l2_loss(self.nfk) # ------bulid optimizer------ self.optimizer = get_optimizer(opt_algo, learning_rate, self.loss) # 保存模型的参数 self.saver = tf.train.Saver(tf.global_variables()) # GPU设定 config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config)
def __init__(self, layer_sizes=None, layer_acts=None, layer_keeps=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None): init_vars = [] num_inputs = len(layer_sizes[0]) embedding_order = layer_sizes[1] for i in range(num_inputs): layer_input = layer_sizes[0][i] layer_output = embedding_order init_vars.append(('w0_%d' % i, [layer_input, layer_output], 'tnormal', dtype)) init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype)) init_vars.append(('f1', [embedding_order, layer_sizes[2], 1, 2], 'tnormal', dtype)) init_vars.append(('f2', [embedding_order, layer_sizes[3], 2, 2], 'tnormal', dtype)) init_vars.append(('w1', [2 * 3 * embedding_order, 1], 'tnormal', dtype)) init_vars.append(('b1', [1], 'zero', dtype)) self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)] b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)] l = tf.nn.dropout( utils.activate( tf.concat(1, [ tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) + b0[i] for i in range(num_inputs) ]), layer_acts[0]), layer_keeps[0]) l = tf.transpose( tf.reshape(l, [-1, num_inputs, embedding_order, 1]), [0, 2, 1, 3]) f1 = self.vars['f1'] l = tf.nn.conv2d(l, f1, [1, 1, 1, 1], 'SAME') l = tf.transpose( utils.max_pool_4d(tf.transpose(l, [0, 1, 3, 2]), num_inputs / 2), [0, 1, 3, 2]) f2 = self.vars['f2'] l = tf.nn.conv2d(l, f2, [1, 1, 1, 1], 'SAME') l = tf.transpose( utils.max_pool_4d(tf.transpose(l, [0, 1, 3, 2]), 3), [0, 1, 3, 2]) l = tf.nn.dropout( utils.activate(tf.reshape(l, [-1, embedding_order * 3 * 2]), layer_acts[1]), layer_keeps[1]) w1 = self.vars['w1'] b1 = self.vars['b1'] l = tf.nn.dropout( utils.activate(tf.matmul(l, w1) + b1, layer_acts[2]), layer_keeps[2]) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(l, self.y)) self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.initialize_all_variables().run(session=self.sess)
def eval(self, data, dropout_keep_prob=1.0): logits = utils.conv2d( data, self.weights, stride=self.stride, padding=self.padding) + self.biases activations = utils.activate(logits, self.activation_func) return activations
def calc_activations(self, data, weights, biases): logits = utils.conv2d( data, weights, stride=self.stride, padding=self.padding) + biases activations = utils.activate(logits, self.activation_func) return activations
def __init__(self, layer_sizes=None, layer_acts=None, layer_keeps=None, layer_l2=None, kernel_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None): """ # Arguments: layer_size: [num_fields, factor_layer, l_p size] layer_acts: ["tanh", "none"] layer_keep: [1, 1] layer_l2: [0, 0] kernel_l2: 0 """ init_vars = [] num_inputs = len(layer_sizes[0]) factor_order = layer_sizes[1] for i in range(num_inputs): layer_input = layer_sizes[0][i] layer_output = factor_order # w0 store the embeddings for all features. init_vars.append(('w0_%d' % i, [layer_input, layer_output], 'tnormal', dtype)) init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype)) init_vars.append(('w_l', [num_inputs * factor_order, layer_sizes[2]], 'tnormal', dtype)) init_vars.append(('w_p', [num_inputs * num_inputs, layer_sizes[2]], 'tnormal', dtype)) #init_vars.append(('w1', [num_inputs * factor_order + num_inputs * num_inputs, layer_sizes[2]], 'tnormal', dtype)) init_vars.append(('b1', [layer_sizes[2]], 'zero', dtype)) for i in range(2, len(layer_sizes) - 1): layer_input = layer_sizes[i] layer_output = layer_sizes[i + 1] init_vars.append(('w%d' % i, [layer_input, layer_output], 'tnormal', dtype)) init_vars.append(('b%d' % i, [layer_output], 'zero', dtype)) self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)] b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)] # Multiply SparseTensor X[i] by dense matrix w0[i] xw = [ tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs) ] x = tf.concat([xw[i] + b0[i] for i in range(num_inputs)], 1) l = tf.nn.dropout(utils.activate(x, layer_acts[0]), layer_keeps[0]) w_l = self.vars['w_l'] w_p = self.vars['w_p'] b1 = self.vars['b1'] # This is where W_p \cdot p happens. # k1 is \theta, which is the weight for each field(feature) vector p = tf.matmul( tf.reshape(l, [-1, num_inputs, factor_order]), tf.transpose(tf.reshape(l, [-1, num_inputs, factor_order]), [0, 2, 1])) p = tf.nn.dropout( utils.activate( tf.matmul(tf.reshape(p, [-1, num_inputs * num_inputs]), w_p), 'none'), 1.0) l = tf.nn.dropout( utils.activate(tf.matmul(l, w_l) + b1 + p, layer_acts[1]), layer_keeps[1]) for i in range(2, len(layer_sizes) - 1): wi = self.vars['w%d' % i] bi = self.vars['b%d' % i] l = tf.nn.dropout( utils.activate(tf.matmul(l, wi) + bi, layer_acts[i]), layer_keeps[i]) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) if layer_l2 is not None: # for i in range(num_inputs): self.loss += layer_l2[0] * tf.nn.l2_loss(tf.concat(xw, 1)) for i in range(1, len(layer_sizes) - 1): if i == 1: self.loss += layer_l2[i] * tf.nn.l2_loss(w_l) self.loss += layer_l2[i] * tf.nn.l2_loss(w_p) else: wi = self.vars['w%d' % i] # bi = self.vars['b%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(wi) if kernel_l2 is not None: pass self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def __init__(self, data_dir=None, summary_dir=None, eval_dir=None, batch_size=None, input_dim=None, output_dim=1, layer_sizes=None, layer_acts=None, drop_out=None, layer_l2=None, kernel_l2=None, l2_w=0, init_path=None, opt_algo='gd', learning_rate=1e-2, sync=False, workers=20): Model.__init__(self) eprint("------- create graph ---------------") init_vars = [] num_inputs = len(layer_sizes[0]) factor_order = layer_sizes[1] for i in range(num_inputs): layer_input = layer_sizes[0][i] layer_output = factor_order init_vars.append(('w0_%d' % i, [layer_input, layer_output], 'tnormal', dtype)) init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype)) init_vars.append(('w1', [num_inputs * factor_order, layer_sizes[2]], 'tnormal', dtype)) init_vars.append(('k1', [num_inputs, layer_sizes[2]], 'tnormal', dtype)) init_vars.append(('b1', [layer_sizes[2]], 'zero', dtype)) for i in range(2, len(layer_sizes) - 1): layer_input = layer_sizes[i] layer_output = layer_sizes[i + 1] init_vars.append(( 'w%d' % i, [layer_input, layer_output], 'tnormal', )) init_vars.append(('b%d' % i, [layer_output], 'zero', dtype)) with tf.name_scope('input_%d' % FLAGS.task_index) as scope: self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.B = tf.sparse_placeholder(tf.float32, name='B') self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)] b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)] xw = [ tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs) ] x = tf.concat([xw[i] + b0[i] for i in range(num_inputs)], 1) l = tf.nn.dropout(utils.activate(x, layer_acts[0]), self.layer_keeps[0]) w1 = self.vars['w1'] k1 = self.vars['k1'] b1 = self.vars['b1'] p = tf.reduce_sum( tf.reshape( tf.matmul( tf.reshape( tf.transpose( tf.reshape(l, [-1, num_inputs, factor_order]), [0, 2, 1]), [-1, num_inputs]), k1), [-1, factor_order, layer_sizes[2]]), 1) l = tf.nn.dropout( utils.activate(tf.matmul(l, w1) + b1 + p, layer_acts[1]), self.layer_keeps[1]) for i in range(2, len(layer_sizes) - 1): wi = self.vars['w%d' % i] bi = self.vars['b%d' % i] l = tf.nn.dropout( utils.activate(tf.matmul(l, wi) + bi, layer_acts[i]), self.layer_keeps[i]) ## logits l = tf.reshape(l, [-1]) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) if layer_l2 is not None: self.loss += layer_l2[0] * tf.nn.l2_loss(tf.concat(xw, 1)) for i in range(1, len(layer_sizes) - 1): wi = self.vars['w%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(wi) if kernel_l2 is not None: self.loss += kernel_l2 * tf.nn.l2_loss(k1) self.global_step = _variable_on_cpu( 'global_step', [], initializer=tf.constant_initializer(0), trainable=False) if sync: self.optimizer = utils.get_sync_optimizer(opt_algo, learning_rate, workers) else: self.optimizer = utils.get_optimizer(opt_algo, learning_rate) self.train_op = self.optimizer.minimize(self.loss, global_step=self.global_step) self.summary_op = tf.summary.merge_all()
def __init__(self, field_sizes=None, embed_size=10, layer_size=None, layer_acts=None, drop_out=None, embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None, layer_norm=True): Model.__init__(self) init_vars = [] num_input = len(field_sizes) for i in range(num_input): init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) num_pairs = int(num_input * (num_input - 1) / 2) # field对的个数 node_in = num_input * embed_size + num_pairs # 此处为设计的关键 init_vars.append(('kernel', [embed_size, num_pairs, embed_size], 'xavier', dtype)) for i in range(len(layer_size)): init_vars.append(('w%d' % i, [node_in, layer_size[i]], 'xavier', dtype)) init_vars.append(('b%d' % i, [layer_size[i]], 'zero', dtype)) node_in = layer_size[i] self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_input)] self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w_0 = [self.vars['embed_%d' % i] for i in range(num_input)] # batch * (embed_size*num_field) xw = tf.concat([ tf.sparse_tensor_dense_matmul(self.X[i], w_0[i]) for i in range(num_input) ], 1) xw3d = tf.reshape(xw, [-1, num_input, embed_size]) row = [] col = [] # 构造pair for i in range(num_input - 1): for j in range(i + 1, num_input): row.append(i) col.append(j) # batch * pair * embed_size p = tf.transpose( # pair * batch * embed_size tf.gather( # num_field * batch * embed_size tf.transpose(xw3d, [1, 0, 2]), row), [1, 0, 2]) q = tf.transpose( # pair * batch * embed_size tf.gather( # num_field * batch * embed_size tf.transpose(xw3d, [1, 0, 2]), col), [1, 0, 2]) p = tf.reshape(p, [-1, num_pairs, embed_size]) q = tf.reshape(q, [-1, num_pairs, embed_size]) # embed_size*num_pairs*embed_size k = self.vars['kernel'] p = tf.expand_dims(p, 1) # 增加维度 batch * 1 * pair * embed_size # batch * num_pairs # temp = tf.multiply(p,k) # temp = tf.reduce_sum(temp,-1) # temp = tf.transpose(temp,[0,2,1]) # temp = tf.multiply(temp,q) # temp = tf.reduce_sum(temp,-1) kp = tf.reduce_sum( # batch * num_pairs * embed_size tf.multiply( # 置换位置 batch * num_pairs * embed_size tf.transpose( # 按最后一个维度求和 batch * embed_size * num_pairs tf.reduce_sum( # 点乘 batch * embed_size*num_pairs*embed_size tf.multiply(p, k), -1), [0, 2, 1]), q), -1) l = tf.concat([xw, kp], 1) for i in range(len(layer_size)): w_i = self.vars['w%d' % i] b_i = self.vars['b%d' % i] l = tf.nn.dropout( utils.activate(tf.matmul(l, w_i) + b_i, layer_acts[i]), self.layer_keeps[i]) l = tf.squeeze(l) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) if layer_l2 is not None: self.loss += embed_l2 * tf.nn.l2_loss(xw) for i in range(len(layer_size)): w_i = self.vars['w%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(w_i) self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None, embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None): Model.__init__(self) init_vars = [] num_inputs = len(field_sizes) for i in range(num_inputs): init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) num_pairs = int(num_inputs * (num_inputs - 1) / 2) node_in = num_inputs * embed_size + num_pairs # node_in = num_inputs * (embed_size + num_inputs) for i in range(len(layer_sizes)): init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype)) init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype)) node_in = layer_sizes[i] self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = init_var_map(init_vars, init_path) w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)] xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1) xw3d = tf.reshape(xw, [-1, num_inputs, embed_size]) row = [] col = [] for i in range(num_inputs-1): for j in range(i+1, num_inputs): row.append(i) col.append(j) # batch * pair * k p = tf.transpose( # pair * batch * k tf.gather( # num * batch * k tf.transpose( xw3d, [1, 0, 2]), row), [1, 0, 2]) # batch * pair * k q = tf.transpose( tf.gather( tf.transpose( xw3d, [1, 0, 2]), col), [1, 0, 2]) p = tf.reshape(p, [-1, num_pairs, embed_size]) q = tf.reshape(q, [-1, num_pairs, embed_size]) ip = tf.reshape(tf.reduce_sum(p * q, [-1]), [-1, num_pairs]) # simple but redundant # batch * n * 1 * k, batch * 1 * n * k # ip = tf.reshape( # tf.reduce_sum( # tf.expand_dims(xw3d, 2) * # tf.expand_dims(xw3d, 1), # 3), # [-1, num_inputs**2]) l = tf.concat([xw, ip], 1) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] bi = self.vars['b%d' % i] l = tf.nn.dropout( activate( tf.matmul(l, wi) + bi, layer_acts[i]), self.layer_keeps[i]) l = tf.squeeze(l) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) if layer_l2 is not None: self.loss += embed_l2 * tf.nn.l2_loss(xw) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(wi) self.optimizer = get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def __init__(self, feature_size, field_size, embedding_size=8, optimizer_type='gd', learning_rate=1e-2, verbose=False, random_seed=None, eval_metric=roc_auc_score, greater_is_better=True, epoch=10, batch_size=1024, l2_reg=0, deep_layers=[32, 32], batch_norm=True, dropout_deep=[], cross_layer_num=3): Model.__init__(self, eval_metric, greater_is_better, epoch, batch_size, verbose, batch_norm, dropout_deep) init_vars = [('weight', [feature_size, 1], 'uniform', tf.float32), ('bias', [1], 'uniform', tf.float32), ('feature_embed', [feature_size, embedding_size], 'normal', tf.float32)] node_in = embedding_size * field_size for i in range(len(deep_layers)): init_vars.extend([('layer_%d' % i, [node_in, deep_layers[i]], 'glorot_normal', tf.float32)]) init_vars.extend([('bias_%d' % i, [1, deep_layers[i]], 'glorot_normal', tf.float32)]) node_in = deep_layers[i] for i in range(cross_layer_num): init_vars.extend([('cross_layer_%d' % i, [1, embedding_size * field_size ], 'glorot_normal', tf.float32)]) init_vars.extend([('cross_bias_%d' % i, [1, embedding_size * field_size ], 'glorot_normal', tf.float32)]) node_in = embedding_size * field_size + deep_layers[-1] init_vars.extend([('concat_projection', [node_in, 1], 'glorot_normal', tf.float32)]) init_vars.extend([('concat_bias', [1, 1], 'glorot_normal', tf.float32)]) self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.feat_index = tf.placeholder(tf.int32, shape=[None, None], name="feat_index") # None * F self.feat_value = tf.placeholder(tf.float32, shape=[None, None], name="feat_value") # None * F self.label = tf.placeholder(tf.float32, shape=[None, 1], name="label") # None * 1 self.dropout_keep_deep = tf.placeholder(tf.float32, shape=[None], name="dropout_keep_deep") self.train_phase = tf.placeholder(tf.bool, name="train_phase") self.vars = utils.init_var_map(init_vars) self.embeddings = tf.nn.embedding_lookup( self.vars["feature_embed"], self.feat_index) # None * F * K feat_value = tf.reshape(self.feat_value, shape=[-1, field_size, 1]) self.embeddings = tf.reshape( tf.multiply(self.embeddings, feat_value), shape=[-1, embedding_size * field_size]) # ---------- cross layer ---------- self.deep_cross_input = tf.nn.dropout(self.embeddings, self.dropout_keep_deep[0]) self.cross_layer_out = self.deep_cross_input for i in range(1, cross_layer_num): self.x0xiT = self.deep_cross_input * self.cross_layer_out self.x0xiT = tf.reduce_sum(self.x0xiT, 1, keep_dims=True) # None * 1 self.cross_layer_out = tf.add(tf.matmul(self.x0xiT, self.vars['cross_layer_%d' % i]) \ , self.vars['cross_bias_%d' % i]) + self.cross_layer_out # ---------- deep component -------- self.y_deep = self.deep_cross_input for i in range(len(deep_layers)): self.y_deep = tf.add( tf.matmul(self.y_deep, self.vars['layer_%s' % i]), self.vars['bias_%s' % i]) if self.batch_norm: self.y_deep = self.batch_norm_layer( self.y_deep, train_phase=self.train_phase, scope_bn="bn_%s" % i) self.y_deep = tf.nn.dropout( utils.activate(self.y_deep, 'relu'), self.dropout_keep_deep[i + 1]) concat_projection = self.vars['concat_projection'] concat_bias = self.vars['concat_bias'] self.out = tf.concat([self.y_deep, self.cross_layer_out], 1) self.out = tf.add(tf.matmul(self.out, concat_projection), concat_bias) self.y_prob = tf.sigmoid(self.out) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(labels=self.label, logits=self.out)) + \ tf.contrib.layers.l2_regularizer( l2_reg)(self.vars["concat_projection"]) for i in range(len(deep_layers)): self.loss += tf.contrib.layers.l2_regularizer(l2_reg)( self.vars["layer_%d" % i]) self.optimizer = utils.get_optimizer(optimizer_type, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) self.sess.run(tf.global_variables_initializer())
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None, embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None, layer_norm=True): Model.__init__(self) init_vars = [] num_inputs = len(field_sizes) for i in range(num_inputs): init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) num_pairs = int(num_inputs * (num_inputs - 1) / 2) node_in = num_inputs * embed_size + num_pairs init_vars.append(('kernel', [embed_size, num_pairs, embed_size], 'xavier', dtype)) for i in range(len(layer_sizes)): init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype)) init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype)) node_in = layer_sizes[i] self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = init_var_map(init_vars, init_path) w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)] xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1) xw3d = tf.reshape(xw, [-1, num_inputs, embed_size]) row = [] col = [] for i in range(num_inputs - 1): for j in range(i + 1, num_inputs): row.append(i) col.append(j) # batch * pair * k p = tf.transpose( # pair * batch * k tf.gather( # num * batch * k tf.transpose( xw3d, [1, 0, 2]), row), [1, 0, 2]) # batch * pair * k q = tf.transpose( tf.gather( tf.transpose( xw3d, [1, 0, 2]), col), [1, 0, 2]) # b * p * k p = tf.reshape(p, [-1, num_pairs, embed_size]) # b * p * k q = tf.reshape(q, [-1, num_pairs, embed_size]) # k * p * k k = self.vars['kernel'] # batch * 1 * pair * k p = tf.expand_dims(p, 1) # batch * pair kp = tf.reduce_sum( # batch * pair * k tf.multiply( # batch * pair * k tf.transpose( # batch * k * pair tf.reduce_sum( # batch * k * pair * k tf.multiply( p, k), -1), [0, 2, 1]), q), -1) # # if layer_norm: # # x_mean, x_var = tf.nn.moments(xw, [1], keep_dims=True) # # xw = (xw - x_mean) / tf.sqrt(x_var) # # x_g = tf.Variable(tf.ones([num_inputs * embed_size]), name='x_g') # # x_b = tf.Variable(tf.zeros([num_inputs * embed_size]), name='x_b') # # x_g = tf.Print(x_g, [x_g[:10], x_b]) # # xw = xw * x_g + x_b # p_mean, p_var = tf.nn.moments(op, [1], keep_dims=True) # op = (op - p_mean) / tf.sqrt(p_var) # p_g = tf.Variable(tf.ones([embed_size**2]), name='p_g') # p_b = tf.Variable(tf.zeros([embed_size**2]), name='p_b') # # p_g = tf.Print(p_g, [p_g[:10], p_b]) # op = op * p_g + p_b l = tf.concat([xw, kp], 1) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] bi = self.vars['b%d' % i] l = tf.nn.dropout( activate( tf.matmul(l, wi) + bi, layer_acts[i]), self.layer_keeps[i]) l = tf.squeeze(l) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) if layer_l2 is not None: self.loss += embed_l2 * tf.nn.l2_loss(xw)#tf.concat(w0, 0)) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(wi) self.optimizer = get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def __init__(self, train_loader, test_loader, embed_size=None, layer_size=None, layer_act=None, layer_keeps=None, opt_algo='gd', learning_rate=0.01, epoch=10, early_stop_round=None, l2=None, random_seed=None): self.graph = tf.Graph() self.train_loader = train_loader self.test_loader = test_loader self.embed_size = embed_size self.layer_size = layer_size self.layer_act = layer_act self.layer_keeps = layer_keeps self.num_fields = len(config.FIELD_SIZES) self.var_list = [] for idx in range(self.num_fields): self.var_list.append([ 'embed_{}'.format(idx), [config.FIELD_SIZES[idx], self.embed_size[idx]], 'xavier' ]) in_size = sum(self.embed_size) for idx in range(len(layer_size)): self.var_list.append( ['w_{}'.format(idx), [in_size, layer_size[idx]], 'xavier']) self.var_list.append( ['b_{}'.format(idx), [layer_size[idx]], 'zero']) in_size = layer_size[idx] self.var_dict = utils.get_var(self.var_list) self.opt_algo = opt_algo self.learning_rate = learning_rate self.epoch = epoch self.early_stop_round = early_stop_round self.l2 = l2 self.random_seed = random_seed self.time_scores = [] self.train_scores = [] self.test_scores = [] # with self.graph.as_default(): if self.random_seed is not None: tf.set_random_seed(self.random_seed) self.X = [ tf.sparse_placeholder(config.DTYPE) for n in range(self.num_fields) ] self.y = tf.placeholder(config.DTYPE) with tf.variable_scope('Dense_Real_Layer'): w_embed = [ self.var_dict['embed_{}'.format(idx)] for idx in range(self.num_fields) ] xw = tf.concat([ tf.sparse_tensor_dense_matmul(self.X[idx], w_embed[idx]) for idx in range(self.num_fields) ], 1) layer_out = xw for idx in range(len(layer_size)): with tf.variable_scope('Hiden_Layer_{}'.format(idx)): wi = self.var_dict['w_{}'.format(idx)] bi = self.var_dict['b_{}'.format(idx)] layer_out = tf.nn.dropout( utils.activate( tf.matmul(layer_out, wi) + bi, self.layer_act[idx]), self.layer_keeps[idx]) layer_out = tf.squeeze(layer_out) self.y_preds = tf.sigmoid(layer_out) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(labels=self.y, logits=layer_out)) if self.l2 is not None: for idx in range(self.num_fields): self.loss += self.l2 * tf.nn.l2_loss( self.var_dict['embed_{}'.format(idx)]) for idx in range(len(self.layer_size)): self.loss += self.l2 * tf.nn.l2_loss( self.var_dict['w_{}'.format(idx)]) self.optimizer = utils.get_optimizer(self.opt_algo, self.learning_rate, self.loss) self.sess = tf.Session() tf.global_variables_initializer().run(session=self.sess)
def __init__(self, data_dir=None, summary_dir=None, eval_dir=None, batch_size=None, input_dim=None, output_dim=1, layer_sizes=None, layer_acts=None, drop_out=None, init_path=None, opt_algo='gd', learning_rate=1e-2, l2_w=0, layer_l2=None, sync=False, workers=20): Model.__init__(self) eprint("-------- create graph ----------") init_vars = [] # linear part init_vars.append(('linear', [input_dim, output_dim], 'xavier', dtype)) init_vars.append(('bias', [output_dim], 'zero', dtype)) num_inputs = len(layer_sizes[0]) factor_order = layer_sizes[1] for i in range(num_inputs): layer_input = layer_sizes[0][i] layer_output = factor_order # field_sizes[i] stores the i-th field feature number init_vars.append(('w0_%d' % i, [layer_input, layer_output], 'xavier', dtype)) init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype)) # full connection node_in = num_inputs * factor_order init_vars.append(('w1', [node_in, layer_sizes[2]], 'xavier', dtype)) init_vars.append(('b1', [layer_sizes[2]], 'zero', dtype)) for i in range(2, len(layer_sizes) - 1): layer_input = layer_sizes[i] layer_output = layer_sizes[i + 1] init_vars.append(('w%d' % i, [layer_input, layer_output], 'xavier', dtype)) init_vars.append(('b%d' % i, [layer_output], 'zero', dtype)) #self.graph = tf.Graph() #with self.graph.as_default(): #with tf.device('/cpu:0'): with tf.name_scope('input_%d' % FLAGS.task_index) as scope: self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.B = tf.sparse_placeholder(tf.float32, name='B') self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)] b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)] xw = [ tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs) ] x = tf.concat([xw[i] + b0[i] for i in range(num_inputs)], 1) ## normalize fmX = tf.sparse_add(self.X[0], self.X[1]) for i in range(2, num_inputs): fmX = tf.sparse_add(fmX, self.X[i]) Xnorm = tf.reshape(1.0 / tf.sparse_reduce_sum(fmX, 1), [-1, output_dim]) l = tf.nn.dropout(utils.activate(x, layer_acts[0]), self.layer_keeps[0]) for i in range(1, len(layer_sizes) - 1): wi = self.vars['w%d' % i] bi = self.vars['b%d' % i] eprint(l.get_shape(), wi.get_shape(), bi.get_shape()) l = tf.nn.dropout( utils.activate(tf.matmul(l, wi) + bi, layer_acts[i]), self.layer_keeps[i]) ## FM linear part fmb = self.vars['bias'] fmw = self.vars['linear'] Xw = tf.sparse_tensor_dense_matmul(self.B, fmw) ## cross term # XV, shape: input_dim*k fmXV = tf.add_n(xw) XV_square = tf.square(fmXV) eprint(XV_square.get_shape()) # X^2 * V^2, shape: input_dim*k fmX2 = [ tf.SparseTensor(self.X[i].indices, tf.square(self.X[i].values), tf.to_int64(tf.shape(self.X[i]))) for i in range(num_inputs) ] fmV2 = [tf.square(w0[i]) for i in range(num_inputs)] fmX2V2 = [ tf.sparse_tensor_dense_matmul(fmX2[i], fmV2[i]) for i in range(num_inputs) ] X2V2 = tf.add_n(fmX2V2) eprint(X2V2.get_shape()) # 1/2 * row_sum(XV_square - X2V2), shape: input_dim*1 p = 0.5 * Xnorm * tf.reshape(tf.reduce_sum(XV_square - X2V2, 1), [-1, output_dim]) ## logits logits = tf.reshape(l + Xw + fmb + p, [-1]) ## predict self.y_prob = tf.sigmoid(logits) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=self.y)) + \ l2_w * tf.nn.l2_loss(Xw) if layer_l2 is not None: self.loss += layer_l2[0] * tf.nn.l2_loss(tf.concat(xw, 1)) for i in range(1, len(layer_sizes) - 1): wi = self.vars['w%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(wi) self.global_step = _variable_on_cpu( 'global_step', [], initializer=tf.constant_initializer(0), trainable=False) if sync: self.optimizer = utils.get_sync_optimizer(opt_algo, learning_rate, workers) else: self.optimizer = utils.get_optimizer(opt_algo, learning_rate) self.train_op = self.optimizer.minimize(self.loss, global_step=self.global_step) self.summary_op = tf.summary.merge_all()
def make_app(): app = sandman2.get_app(config.SQLA_URI, Base=utils.AutomapModel) app.json_encoder = utils.APIJSONEncoder app.config['CASE_INSENSITIVE'] = config.CASE_INSENSITIVE app.config['BASIC_AUTH_USERNAME'] = os.environ.get( 'AUTOAPI_ADMIN_USERNAME', '') app.config['BASIC_AUTH_PASSWORD'] = os.environ.get( 'AUTOAPI_ADMIN_PASSWORD', '') CORS(app) basic_auth = BasicAuth(app) @app.before_request def refresh(): tables = utils.get_tables() if tables != app.config['SQLALCHEMY_TABLES']: utils.refresh_tables() app.config['SQLALCHEMY_TABLES'] = tables @app.before_request def protect_admin(): if request.path.startswith('/admin/'): if not basic_auth.authenticate(): return basic_auth.challenge() class RefreshTables(View): task_name = 'refresh' def dispatch_request(self): underway = RefreshLog.refresh_underway() if underway: return '''Refresh begun at {} still underway. Now: {}; timeout set for {} seconds'''.format( underway.begun_at, datetime.now(), config.REFRESH_TIMEOUT_SECONDS) try: subprocess.Popen(['invoke', self.task_name, ]) except Exception as e: print('Problem with table refresh:') print(e) return 'Table refresh requested.' class QuickRefreshTables(RefreshTables): task_name = 'quick_refresh' app.add_url_rule('/refresh/', view_func=RefreshTables.as_view('refresh')) app.add_url_rule('/quick_refresh/', view_func=QuickRefreshTables.as_view('quick_refresh')) aws_blueprint = aws.make_blueprint() app.register_blueprint(aws_blueprint) docs_blueprint = swagger.make_blueprint(app) app.register_blueprint(docs_blueprint) with app.app_context(): app.config['SQLALCHEMY_TABLES'] = utils.get_tables() utils.activate() refresh_log_db.init_app(app) refresh_log_db.create_all(app=app) return app
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None, embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None, layer_norm=True): Model.__init__(self) init_vars = [] num_inputs = len(field_sizes) for i in range(num_inputs): init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) node_in = num_inputs * embed_size + embed_size * embed_size for i in range(len(layer_sizes)): init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype)) init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype)) node_in = layer_sizes[i] self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)] xw = tf.concat([ tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs) ], 1) z = tf.reduce_sum(tf.reshape(xw, [-1, num_inputs, embed_size]), 1) op = tf.reshape( tf.matmul(tf.reshape(z, [-1, embed_size, 1]), tf.reshape(z, [-1, 1, embed_size])), [-1, embed_size * embed_size]) if layer_norm: # x_mean, x_var = tf.nn.moments(xw, [1], keep_dims=True) # xw = (xw - x_mean) / tf.sqrt(x_var) # x_g = tf.Variable(tf.ones([num_inputs * embed_size]), name='x_g') # x_b = tf.Variable(tf.zeros([num_inputs * embed_size]), name='x_b') # x_g = tf.Print(x_g, [x_g[:10], x_b]) # xw = xw * x_g + x_b p_mean, p_var = tf.nn.moments(op, [1], keep_dims=True) op = (op - p_mean) / tf.sqrt(p_var) p_g = tf.Variable(tf.ones([embed_size**2]), name='p_g') p_b = tf.Variable(tf.zeros([embed_size**2]), name='p_b') # p_g = tf.Print(p_g, [p_g[:10], p_b]) op = op * p_g + p_b l = tf.concat([xw, op], 1) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] bi = self.vars['b%d' % i] l = tf.nn.dropout( utils.activate(tf.matmul(l, wi) + bi, layer_acts[i]), self.layer_keeps[i]) l = tf.squeeze(l) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) if layer_l2 is not None: self.loss += embed_l2 * tf.nn.l2_loss(tf.concat(w0, 0)) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(wi) self.optimizer = utils.get_optimizer(opt_algo, learning_rate) self.train_op = self.optimizer.minimize(self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def __init__(self, layer_sizes=None, layer_acts=None, drop_out=None, layer_l2=None, kernel_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None): Model.__init__(self) init_vars = [] num_inputs = len(layer_sizes[0]) factor_order = layer_sizes[1] for i in range(num_inputs): layer_input = layer_sizes[0][i] layer_output = factor_order init_vars.append(('w0_%d' % i, [layer_input, layer_output], 'tnormal', dtype)) init_vars.append(('b0_%d' % i, [layer_output], 'zero', dtype)) init_vars.append(('w1', [num_inputs * factor_order, layer_sizes[2]], 'tnormal', dtype)) init_vars.append(('k1', [factor_order * factor_order, layer_sizes[2]], 'tnormal', dtype)) init_vars.append(('b1', [layer_sizes[2]], 'zero', dtype)) for i in range(2, len(layer_sizes) - 1): layer_input = layer_sizes[i] layer_output = layer_sizes[i + 1] init_vars.append(( 'w%d' % i, [layer_input, layer_output], 'tnormal', )) init_vars.append(('b%d' % i, [layer_output], 'zero', dtype)) self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w0 = [self.vars['w0_%d' % i] for i in range(num_inputs)] b0 = [self.vars['b0_%d' % i] for i in range(num_inputs)] xw = [ tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs) ] x = tf.concat([xw[i] + b0[i] for i in range(num_inputs)], 1) l = tf.nn.dropout(utils.activate(x, layer_acts[0]), self.layer_keeps[0]) w1 = self.vars['w1'] k1 = self.vars['k1'] b1 = self.vars['b1'] z = tf.reduce_sum(tf.reshape(l, [-1, num_inputs, factor_order]), 1) p = tf.reshape( tf.matmul(tf.reshape(z, [-1, factor_order, 1]), tf.reshape(z, [-1, 1, factor_order])), [-1, factor_order * factor_order]) l = tf.nn.dropout( utils.activate( tf.matmul(l, w1) + tf.matmul(p, k1) + b1, layer_acts[1]), self.layer_keeps[1]) for i in range(2, len(layer_sizes) - 1): wi = self.vars['w%d' % i] bi = self.vars['b%d' % i] l = tf.nn.dropout( utils.activate(tf.matmul(l, wi) + bi, layer_acts[i]), self.layer_keeps[i]) l = tf.reshape(l, [-1]) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) if layer_l2 is not None: # for i in range(num_inputs): self.loss += layer_l2[0] * tf.nn.l2_loss(tf.concat(xw, 1)) for i in range(1, len(layer_sizes) - 1): wi = self.vars['w%d' % i] # bi = self.vars['b%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(wi) if kernel_l2 is not None: self.loss += kernel_l2 * tf.nn.l2_loss(k1) self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def __init__(self, field_sizes=None, embed_size=10, filter_sizes=None, layer_acts=None, drop_out=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None): Model.__init__(self) init_vars = [] num_inputs = len(field_sizes) for i in range(num_inputs): init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) init_vars.append(('f1', [embed_size, filter_sizes[0], 1, 2], 'xavier', dtype)) init_vars.append(('f2', [embed_size, filter_sizes[1], 2, 2], 'xavier', dtype)) init_vars.append(('w1', [2 * 3 * embed_size, 1], 'xavier', dtype)) init_vars.append(('b1', [1], 'zero', dtype)) print('init_vars: ', init_vars) self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)] xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1) l = xw l = tf.transpose(tf.reshape(l, [-1, num_inputs, embed_size, 1]), [0, 2, 1, 3]) # 变为 16 x 10 矩阵 f1 = self.vars['f1'] l = tf.nn.conv2d(l, f1, [1, 1, 1, 1], 'SAME') l = tf.transpose( utils.max_pool_4d( tf.transpose(l, [0, 1, 3, 2]), int(num_inputs / 2)), [0, 1, 3, 2]) f2 = self.vars['f2'] l = tf.nn.conv2d(l, f2, [1, 1, 1, 1], 'SAME') l = tf.transpose( utils.max_pool_4d( tf.transpose(l, [0, 1, 3, 2]), 3), [0, 1, 3, 2]) l = tf.nn.dropout( utils.activate( tf.reshape(l, [-1, embed_size * 3 * 2]), layer_acts[0]), self.layer_keeps[0]) w1 = self.vars['w1'] b1 = self.vars['b1'] l = tf.matmul(l, w1) + b1 l = tf.squeeze(l) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def write_html(self, username, organization, site_code, site_name, forecast_model_name, forecast_method, forecast_model_params, forecast_method_params, filename=None, htmlpage=None, language='en'): activate(language) if self.y.mode == 'p': frequency = 'fiveday' elif self.y.mode == 'd': frequency = 'decade' elif self.y.mode == 'm': frequency = 'monthly' page = self.load_template_file() scatter_plot = PlotUtils.plot_ts_comparison( self.y_adj.timeseries, self.forecast.timeseries, frequency, language=language, ) scaled_error_title = _('Scaled Error [RMSE/STDEV]') scaled_error_plot = PlotUtils.plot_rel_error(self.rel_error, frequency, title=scaled_error_title) scaled_error_table = self.rel_error_table(frequency) p_plot_title = _('P% Plot') p_plot_plot = PlotUtils.plot_p(self.p, frequency, title=p_plot_title) p_plot_table = self.p_plot_table(frequency) quality_assessment_table = self.summary_table(frequency) report_data = { 'SITE_INFO': _('Station: {code} - {name}').format(code=to_str(site_code), name=to_str(site_name)), 'USERNAME': username, 'ORGANIZATION': organization, 'TITLE': _('Forecast Model Training Report'), 'REPORT_DATE': format_date(format='long', locale=language), 'PLOTS_HEADER': _('{frequency} Forecast Model Quality Assessment').format( frequency=frequency.capitalize()), 'SCATTER_PLOT_LABEL': _('Scatter Plot: Observed versus Predicted values'), 'SCALED_ERROR_LABEL': scaled_error_title, 'P_PLOT_LABEL': p_plot_title, 'QUALITY_ASSESSMENT_LABEL': _('Quality Assessment'), 'SCATTER_PLOT_IMAGE': scatter_plot, 'SCALED_ERROR_PLOT_IMAGE': scaled_error_plot, 'SCALED_ERROR_TABLE': scaled_error_table, 'P_PLOT_IMAGE': p_plot_plot, 'P_PLOT_TABLE': p_plot_table, 'QUALITY_ASSESSMENT_TABLE': quality_assessment_table, 'FORECAST_MODEL_INFO': _('Forecast model info:'), 'FORECAST_MODEL_NAME': _('Name: ') + forecast_model_name, 'FORECAST_METHOD': _('Method: ') + forecast_method, 'FORECAST_MODEL_PARAMS': _('Model parameters: ') + str(forecast_model_params), 'FORECAST_METHOD_PARAMS': _('Method parameters: ') + str(forecast_method_params), } report_data.update(self.get_spacers(frequency, language)) self.encode_utf8(report_data) if filename: htmlpage = open(filename, 'w') htmlpage.write(page.safe_substitute(**report_data)) htmlpage.close() return filename elif htmlpage: htmlpage.write(page.safe_substitute(**report_data)) return htmlpage
def __init__(self, field_sizes=None, embed_size=10, layer_sizes=None, layer_acts=None, drop_out=None, embed_l2=None, layer_l2=None, init_path=None, opt_algo='gd', learning_rate=1e-2, random_seed=None, layer_norm=True): Model.__init__(self) init_vars = [] num_inputs = len(field_sizes) for i in range(num_inputs): init_vars.append(('embed_%d' % i, [field_sizes[i], embed_size], 'xavier', dtype)) node_in = num_inputs * embed_size + embed_size * embed_size for i in range(len(layer_sizes)): init_vars.append(('w%d' % i, [node_in, layer_sizes[i]], 'xavier', dtype)) init_vars.append(('b%d' % i, [layer_sizes[i]], 'zero', dtype)) node_in = layer_sizes[i] self.graph = tf.Graph() with self.graph.as_default(): if random_seed is not None: tf.set_random_seed(random_seed) self.X = [tf.sparse_placeholder(dtype) for i in range(num_inputs)] self.y = tf.placeholder(dtype) self.keep_prob_train = 1 - np.array(drop_out) self.keep_prob_test = np.ones_like(drop_out) self.layer_keeps = tf.placeholder(dtype) self.vars = utils.init_var_map(init_vars, init_path) w0 = [self.vars['embed_%d' % i] for i in range(num_inputs)] xw = tf.concat([tf.sparse_tensor_dense_matmul(self.X[i], w0[i]) for i in range(num_inputs)], 1) z = tf.reduce_sum(tf.reshape(xw, [-1, num_inputs, embed_size]), 1) op = tf.reshape( tf.matmul(tf.reshape(z, [-1, embed_size, 1]), tf.reshape(z, [-1, 1, embed_size])), [-1, embed_size * embed_size]) if layer_norm: # x_mean, x_var = tf.nn.moments(xw, [1], keep_dims=True) # xw = (xw - x_mean) / tf.sqrt(x_var) # x_g = tf.Variable(tf.ones([num_inputs * embed_size]), name='x_g') # x_b = tf.Variable(tf.zeros([num_inputs * embed_size]), name='x_b') # x_g = tf.Print(x_g, [x_g[:10], x_b]) # xw = xw * x_g + x_b p_mean, p_var = tf.nn.moments(op, [1], keep_dims=True) op = (op - p_mean) / tf.sqrt(p_var) p_g = tf.Variable(tf.ones([embed_size**2]), name='p_g') p_b = tf.Variable(tf.zeros([embed_size**2]), name='p_b') # p_g = tf.Print(p_g, [p_g[:10], p_b]) op = op * p_g + p_b l = tf.concat([xw, op], 1) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] bi = self.vars['b%d' % i] l = tf.nn.dropout( utils.activate( tf.matmul(l, wi) + bi, layer_acts[i]), self.layer_keeps[i]) l = tf.squeeze(l) self.y_prob = tf.sigmoid(l) self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(logits=l, labels=self.y)) if layer_l2 is not None: self.loss += embed_l2 * tf.nn.l2_loss(tf.concat(w0, 0)) for i in range(len(layer_sizes)): wi = self.vars['w%d' % i] self.loss += layer_l2[i] * tf.nn.l2_loss(wi) self.optimizer = utils.get_optimizer(opt_algo, learning_rate, self.loss) config = tf.ConfigProto() config.gpu_options.allow_growth = True self.sess = tf.Session(config=config) tf.global_variables_initializer().run(session=self.sess)
def write_html(self, username, organization, site_code, site_name, forecast_model_name, forecast_method, forecast_model_params, forecast_method_params, filename=None, htmlpage=None, language='en'): activate(language) page = self.load_template_file() timeseries_plot = self.__encode_figure(self.plot_timeseries()) quality_assessment_table = self.__table_summary() model_table = self.__model_htmltable() report_data = { 'SITE_INFO': _('Station: {code} - {name}').format(code=to_str(site_code), name=to_str(site_name)), 'USERNAME': username, 'ORGANIZATION': organization, 'TITLE': _('Forecast Model Training Report'), 'REPORT_DATE': format_date(format='long', locale=language), 'PLOTS_HEADER': _('{frequency} Forecast Model Quality Assessment').format( frequency=_('seasonal').capitalize()), 'MODEL_TABLE_LABEL': _('Model table'), 'MODEL_TABLE': model_table, 'QUALITY_ASSESSMENT_LABEL': _('Quality Assessment'), 'QUALITY_ASSESSMENT_TABLE': quality_assessment_table, 'TIMESERIES_LABEL': _('Timeseries plot'), 'TIMESERIES_PLOT': timeseries_plot, 'FORECAST_MODEL_INFO': _('Forecast model info:'), 'FORECAST_MODEL_NAME': _('Name: ') + forecast_model_name, 'FORECAST_METHOD': _('Method: ') + forecast_method, 'FORECAST_MODEL_PARAMS': _('Model parameters: ') + str(forecast_model_params), 'FORECAST_METHOD_PARAMS': _('Method parameters: ') + str(forecast_method_params), } self.encode_utf8(report_data) if filename: htmlpage = open(filename, 'w') htmlpage.write(page.safe_substitute(**report_data)) htmlpage.close() return filename elif htmlpage: htmlpage.write(page.safe_substitute(**report_data)) return htmlpage