def Encoder(img_size, in_channel, conv_channel, filter_size, latent_dim, dense_size, bn): inner_conv_channel = conv_channel // 2 if img_size % 4 != 0: print("WARNING: image size mod 4 != 0, may produce bug.") # total input number of the input of the last conv layer, new image size = old / 2 / 2 flatten_img_size = inner_conv_channel * img_size / 4 * img_size / 4 # explain: first two layer's padding = 2, because we set W/S = W/S + floor((-F+2P)/S+1), S=2,F=5,so P=2 if VERBOSE: print(img_size, in_channel, conv_channel, filter_size, latent_dim, bn) model = nn.Sequential( layers.ConvLayer(in_channel, conv_channel, filter_size, stride=2, padding=2, bn=bn), layers.ConvLayer(conv_channel, inner_conv_channel, filter_size, stride=2, padding=2, bn=bn), layers.ConvLayer(inner_conv_channel, inner_conv_channel, filter_size, stride=1, padding=2, bn=bn), layers.Flatten(), layers.Dense(flatten_img_size, dense_size), layers.Dense(dense_size, latent_dim)) model = model.to(device=device, dtype=dtype) model = torch.nn.DataParallel(model, device_ids=GPU_IDs) return model
def __init__(self, input_size, output_size, hidden_layers_sizes, loss=mtr.CrossEntropyLoss(), learn_rate=0.01, problem='classification', scorer=mtr.AccuracyScore()): """ Create multi-layer perceptron Args: input_size: neurons on input layer output_size: neurons on output layer hidden_layers_sizes: list of neurons on each hidden layer problem: problem to solve ('classification" or 'regression') """ layer_sizes = [input_size] + hidden_layers_sizes + [output_size] layers = [] for i in range(len(layer_sizes) - 2): layers.append( lrs.Dense(layer_sizes[i], layer_sizes[i + 1], learn_rate)) layers.append(lrs.LeakyReLU()) layers.append(lrs.Dense(layer_sizes[-2], layer_sizes[-1], learn_rate)) if problem == 'classification': layers.append(lrs.Softmax()) super().__init__(layers, loss, scorer)
def Classifier(input_dim, dense_size, s_classes, bn): model = nn.Sequential(layers.Dense(input_dim, dense_size, bn=bn), layers.Dense(dense_size, dense_size, bn=bn), layers.Dense(dense_size, s_classes, bn=bn)) model = model.to(device=device, dtype=dtype) model = torch.nn.DataParallel(model, device_ids=GPU_IDs) return model
def __init__(self): # Layers self.conv1 = l.Convolution("conv1",3,6,5,1) self.conv2 = l.Convolution("conv2",6,16,5,1) self.relu = l.ReLU("relu") self.pool = l.Maxpooling("pooling",2,2) self.dense1 = l.Dense("dense1",16*5*5,120) self.dense2 = l.Dense("dense1",120,84) self.dense3 = l.Dense("dense1",84,10)
def _build_encoder(self, x): dim_mult = 2 if self.concat else 1 h = layers.Dense(dim_mult* self.dims[-1], dim_mult* self.dims[-1], dropout=self.placeholders['dropout'], act=tf.nn.relu)(x) mu = layers.Dense(dim_mult* self.dims[-1], dim_mult* self.dims[-1], dropout=self.placeholders['dropout'])(h) log_sigma_squared = layers.Dense(dim_mult*self.dims[-1], dim_mult* self.dims[-1], dropout=self.placeholders['dropout'])(h) sigma_squared = tf.exp(log_sigma_squared) sigma = tf.sqrt(sigma_squared) return mu, log_sigma_squared, sigma_squared, sigma
def _build(self): self.layers.append(layers.Dense(input_dim=self.input_dim, output_dim=self.dims[1], act=tf.nn.relu, dropout=self.placeholders['dropout'], sparse_inputs=False, logging=self.logging)) self.layers.append(layers.Dense(input_dim=self.dims[1], output_dim=self.output_dim, act=lambda x: x, dropout=self.placeholders['dropout'], logging=self.logging))
def build(self): #sample Sample neighbors to be the supportive fields for placeholders["batch1"] samples1, support_sizes1 = self.sample(self.inputs1, self.layer_infos) num_samples = [layer_info.num_samples for layer_info in self.layer_infos] #Return The hidden representatio at the final layer for all nodes in batch placeholders["batch1"] self.outputs1, self.aggregators = self.aggregate(samples1, [self.features], self.dims, num_samples, support_sizes1, concat=self.concat, model_size=self.model_size) dim_mult = 2 if self.concat else 1 self.outputs1 = tf.nn.l2_normalize(self.outputs1, 1) dim_mult = 2 if self.concat else 1 self.node_pred = layers.Dense(dim_mult*self.dims[-1], self.num_classes, dropout=self.placeholders['dropout'], act=lambda x : x) # TF graph management self.node_preds = self.node_pred(self.outputs1) self._loss() grads_and_vars = self.optimizer.compute_gradients(self.loss) clipped_grads_and_vars = [(tf.clip_by_value(grad, -5.0, 5.0) if grad is not None else None, var) for grad, var in grads_and_vars] self.grad, _ = clipped_grads_and_vars[0] self.opt_op = self.optimizer.apply_gradients(clipped_grads_and_vars) self.preds = self.predict()
def build(self): '''start SAMPLE''' samples1, support_sizes1 = self.sample(self.inputs1, self.layer_infos) num_samples = [ layer_info.num_samples for layer_info in self.layer_infos ] '''start AGGREGATE''' self.outputs1, self.aggregators = self.aggregate( samples1, [self.features], self.dims, num_samples, support_sizes1, concat=self.concat, model_size=self.model_size) dim_mult = 2 if self.concat else 1 self.outputs1 = tf.nn.l2_normalize(self.outputs1, 1) dim_mult = 2 if self.concat else 1 self.node_pred = layers.Dense(dim_mult * self.dims[-1], self.num_classes, dropout=self.placeholders['dropout'], act=lambda x: x) # TF graph management self.node_preds = self.node_pred(self.outputs1) self._loss() grads_and_vars = self.optimizer.compute_gradients(self.loss) clipped_grads_and_vars = [ (tf.clip_by_value(grad, -5.0, 5.0) if grad is not None else None, var) for grad, var in grads_and_vars ] self.grad, _ = clipped_grads_and_vars[0] self.opt_op = self.optimizer.apply_gradients(clipped_grads_and_vars) self.preds = self.predict()
def run_classification(): model = Model.Model() model.add_layer(layers.Input(784)) model.add_layer(layers.Dense(100, activation=af.relu)) model.add_layer(layers.Dense(10, activation=af.softmax)) model.compile(losses.crossentropy, optimizers.Adam()) with gzip.open('data/mnist.pkl.gz', 'rb') as f: train_set, validation_set, test_set = pickle.load(f, encoding='latin1') n_train = train_set[0].shape[0] n_test = test_set[0].shape[0] train_set_onehots = helpers.make_onehot_2d(train_set[1], 10) test_set_onehots = helpers.make_onehot_2d(test_set[1], 10) model.fit(train_set[0], train_set_onehots, 50, 50, metric_dataset_x=test_set[0], metric_dataset_y=test_set_onehots, metric_callback=classification_metric_accuracy)
def load(self, file_path="./saved_dlmb_model.json"): """ Goes to the file where the model has been saved and retrieves the data. Arguments: file_path : str : A file path where the .json file has been saved. """ layers = { "Dense":la.Dense(0, 0), "Batchnorm":la.Batchnorm(0), "Dropout":la.Dropout() } # Try to open the file at file_path. try: with open(file_path, "r") as json_file: model_layers = json.loads(json_file.read()) for i in range(len(model_layers)): layer_data = model_layers["layer: %s" % i] new_layer = copy.copy(layers[layer_data["name"]]) new_layer.load(layer_data) self.model.append(new_layer) # Gets called if the program can't find the file_path. except Exception as e: raise FileNotFoundError("Can't find file path %s. Try saving the model or enter a correct file path." % file_path)
def _build_encoder(self, x): dim_mult = 2 if self.concat else 1 fc1 = layers.Dense(dim_mult * self.dims[-1], dim_mult * self.dims[-1], dropout=self.placeholders['dropout'], act=tf.nn.relu) fc2 = layers.Dense(dim_mult * self.dims[-1], dim_mult * self.dims[-1], dropout=self.placeholders['dropout']) fc3 = layers.Dense(dim_mult * self.dims[-1], dim_mult * self.dims[-1], dropout=self.placeholders['dropout']) h = fc1(x) mu = fc2(h) log_sigma_squared = fc3(h) sigma_squared = tf.exp(log_sigma_squared) sigma = tf.sqrt(sigma_squared) return mu, log_sigma_squared, sigma_squared, sigma, [fc1, fc2, fc3]
def Decoder(s_dim, z_dim, img_size, img_channel, conv_channel, filter_size, dense_size, bn): # TODO # essentially the mirror version of Encoder inner_conv_channel = conv_channel // 2 back_img_size = img_size // 4 flatten_img_size = inner_conv_channel * back_img_size * back_img_size input_dim = s_dim + z_dim pad = int( np.floor(filter_size / 2)) # chose pad this way to fullfill floor((-F+2P)/1+1)==0 model = nn.Sequential( layers.Dense(input_dim, dense_size), layers.Dense(dense_size, inner_conv_channel * back_img_size * back_img_size), layers.Reshape((-1, inner_conv_channel, back_img_size, back_img_size)), layers.ConvLayer(inner_conv_channel, inner_conv_channel, filter_size, stride=1, padding=pad, bn=bn, upsampling=True), layers.ConvLayer(inner_conv_channel, conv_channel, filter_size, stride=1, padding=pad, bn=bn, upsampling=True), layers.ConvLayer(conv_channel, img_channel, filter_size, stride=1, padding=pad, bn=bn, upsampling=False), ) model = model.to(device=device, dtype=dtype) model = torch.nn.DataParallel(model, device_ids=GPU_IDs) return model
def run_regression(): df = np.array(pd.read_csv('data/Dataset/Training/Features_Variant_1.csv')) model = Model.Model() model.add_layer(layers.Input(53)) model.add_layer(layers.Dense(20, activation=af.relu)) model.add_layer(layers.Dense(1, activation=af.sigmoid)) model.compile(losses.mse, optimizers.SGD()) input_set = np.array([x[:-1] for x in df]) output_set = np.array([x[-1] for x in df]).reshape(len(input_set), 1) # Model.save_model(model, "test") # tmp = Model.load_model("test") # tmp.fit(input_set, output_set, 50, 50, metric_callback=regression_metric_mse) input_set = helpers.standard_scaler(input_set) output_set = helpers.standard_scaler(output_set) np.seterr(all="raise") model.fit(input_set, output_set, 50, 100, metric_callback=regression_metric_mse) Model.save_model(model,"SGD")
def _build_encoder(self, inputs, hiddens, fc1=None, fc2=None, fc3=None): dim_mult = 2 if self.concat else 1 if fc1 is None: fc1 = layers.Dense(2 * dim_mult * self.dims[-1], dim_mult * self.dims[-1], dropout=self.placeholders['dropout'], act=tf.nn.relu) if fc2 is None: fc2 = layers.Dense(dim_mult * self.dims[-1], dim_mult * self.dims[-1], dropout=self.placeholders['dropout']) if fc3 is None: fc3 = layers.Dense(dim_mult * self.dims[-1], dim_mult * self.dims[-1], dropout=self.placeholders['dropout']) x = tf.concat([inputs, hiddens], axis=-1) h = fc1(x) mu = fc2(h) log_sigma_squared = fc3(h) sigma_squared = tf.exp(log_sigma_squared) sigma = tf.sqrt(sigma_squared) return mu, log_sigma_squared, sigma_squared, sigma, fc1, fc2, fc3
def build(self): samples1_list, support_sizes1_list = [], [] for r_idx in range(self.num_relations): samples1, support_sizes1 = self.sample(self.inputs1, self.layer_infos[r_idx]) samples1_list.append(samples1) support_sizes1_list.append(support_sizes1) num_samples = [ layer_info.num_samples for layer_info in self.layer_infos[0] ] self.outputs1_list = [] dim_mult = 2 if self.concat else 1 # multiplication to get the correct output dimension dim_mult = dim_mult * 2 for r_idx in range(self.num_relations): outputs1, self.aggregators = self.aggregate( samples1_list[r_idx], [self.features], self.dims, num_samples, support_sizes1, concat=self.concat, model_size=self.model_size) self.relation_batch = tf.tile( [tf.nn.embedding_lookup(self.relation_vectors, r_idx)], [self.batch_size, 1]) outputs1 = tf.concat([outputs1, self.relation_batch], 1) self.attention_weights = tf.matmul(outputs1, self.attention_vec) self.attention_weights = tf.tile(self.attention_weights, [1, dim_mult * self.dims[-1]]) outputs1 = tf.multiply(self.attention_weights, outputs1) self.outputs1_list += [outputs1] # self.outputs1 = tf.reduce_mean(self.outputs1_list, 0) self.outputs1 = tf.stack(self.outputs1_list, 1) self.outputs1 = tf.reduce_sum(self.outputs1, axis=1, keepdims=False) self.outputs1 = tf.nn.l2_normalize(self.outputs1, 1) self.node_pred = layers.Dense(dim_mult * self.dims[-1], self.num_classes, dropout=self.placeholders['dropout'], act=lambda x: x) # TF graph management self.node_preds = self.node_pred(self.outputs1) self._loss() grads_and_vars = self.optimizer.compute_gradients(self.loss) clipped_grads_and_vars = [ (tf.clip_by_value(grad, -5.0, 5.0) if grad is not None else None, var) for grad, var in grads_and_vars ] self.grad, _ = clipped_grads_and_vars[0] self.opt_op = self.optimizer.apply_gradients(clipped_grads_and_vars) self.preds = self.predict()
def build(self): samples, support_size = self.sample(self.inputs, self.layer_infos) num_samples = [ layer_info.num_samples for layer_info in self.layer_infos ] self.hiddens, self.aggregators = self.aggregate( samples, [self.features], self.dims, num_samples, support_size, concat=self.concat, model_size=self.model_size) dim_mult = 2 if self.concat else 1 self.hiddens = tf.nn.l2_normalize(self.hiddens, 1) # VAE self.mu, log_sigma_squared, sigma_squared, sigma = self._build_encoder( self.hiddens) self.z = tf.random_normal([dim_mult * self.dims[-1]], mean=self.mu, stddev=sigma) self.regular_vae = -0.5 * tf.reduce_sum( 1 + log_sigma_squared - tf.square(self.mu) - sigma_squared, 1) self.node_pred = layers.Dense(dim_mult * self.dims[-1], self.num_classes, dropout=self.placeholders['dropout'], act=tf.nn.sigmoid) self.outputs = self.node_pred(self.z) self._loss() grads_and_vars = self.optimizer.compute_gradients(self.loss) clipped_grads_and_vars = [ (tf.clip_by_value(grad, -5.0, 5.0) if grad is not None else None, var) for grad, var in grads_and_vars ] self.grad, _ = clipped_grads_and_vars[0] self.opt_op = self.optimizer.apply_gradients(clipped_grads_and_vars)
def build(self): samples1, support_sizes1, attentions1, num_nz, out_mean = self.sample( self.inputs1, self.layer_infos) self.att = attentions1 self.out_mean = out_mean num_samples = [ layer_info.num_samples for layer_info in self.layer_infos ] self.outputs1, self.aggregators = self.aggregate( samples1, [self.features], self.dims, num_samples, support_sizes1, attentions1, num_nz, concat=self.concat, model_size=self.model_size) self.outputs1 = tf.nn.l2_normalize(self.outputs1, 1) dim_mult = 2 if self.concat else 1 self.node_pred = layers.Dense(dim_mult * self.dims[-1], self.num_classes, self.weight_decay, dropout=self.placeholders['dropout'], act=lambda x: x) # TF graph management self.node_preds = self.node_pred(self.outputs1) self._loss() # added self.sparse_loss_to_node(samples1, support_sizes1, num_samples) grads_and_vars = self.optimizer.compute_gradients(self.loss) clipped_grads_and_vars = [ (tf.clip_by_value(grad, -5.0, 5.0) if grad is not None else None, var) for grad, var in grads_and_vars ] self.grad, _ = clipped_grads_and_vars[0] self.opt_op = self.optimizer.apply_gradients(clipped_grads_and_vars) self.preds = self.predict()
def define_weights(self, hidden_dims): W = {} for i in range(self.num_layers): W[i] = tf.get_variable("W%s" % i, shape=(hidden_dims[i], hidden_dims[i + 1])) Ws_att = {} # attention weight matrix for i in range(len(hidden_dims) - 1): v = {} v[0] = tf.get_variable("v%s_0" % i, shape=(hidden_dims[i + 1], 1)) v[1] = tf.get_variable("v%s_1" % i, shape=(hidden_dims[i + 1], 1)) Ws_att[i] = v self.W, self.v = W, Ws_att self.C = {} self.project_layer = layers.Dense(input_dim=self.hidden_dims[-1] + FLAGS.fm_dims, output_dim=self.output_dim, placeholders=self.placeholders, act=lambda x: x, dropout=True, logging=self.logging, norm=False)
def _build(self): self.layers.append( layers.GraphConvolution(input_dim=self.input_dim if not self.residual else self.input_dim * 2, output_dim=FLAGS.hidden1, placeholders=self.placeholders, act=tf.nn.relu, dropout=True, sparse_inputs=self.sparse_inputs, logging=self.logging, norm=self.norm, precalc=self.precalc, residual=self.residual)) for _ in range(self.num_layers - 1): self.layers.append( layers.GraphConvolution( input_dim=FLAGS.hidden1 if not self.residual else FLAGS.hidden1 * 2, output_dim=FLAGS.hidden1, placeholders=self.placeholders, act=tf.nn.relu, dropout=True, sparse_inputs=False, logging=self.logging, norm=self.norm, precalc=False, residual=self.residual)) self.project_layer = layers.Dense(input_dim=FLAGS.hidden1 * 2, output_dim=self.output_dim, placeholders=self.placeholders, act=lambda x: x, dropout=True, logging=self.logging, norm=False)
def build(self): samples, support_size = self.sample(self.inputs, self.layer_infos) num_samples = [ layer_info.num_samples for layer_info in self.layer_infos ] self.hiddens, self.aggregators = self.aggregate( samples, [self.features], self.dims, num_samples, support_size, concat=self.concat, model_size=self.model_size) dim_mult = 2 if self.concat else 1 self.hiddens = tf.nn.l2_normalize(self.hiddens, 1) # Two Channel VAE self.mu_rec, log_sigma_squared, sigma_squared, sigma_rec, self.vars_vae_rec = self._build_encoder( self.hiddens) self.z_rec = tf.random_normal([dim_mult * self.dims[-1]], mean=self.mu_rec, stddev=sigma_rec) self.normal_rec = -0.5 * tf.reduce_sum( 1 + log_sigma_squared - tf.square(self.mu_rec) - sigma_squared, 1) self.node_pred_rec = layers.Dense(dim_mult * self.dims[-1], self.num_classes, dropout=self.placeholders['dropout'], act=None) self.outputs_rec = self.node_pred_rec(self.z_rec) self.mu_abn, log_sigma_squared, sigma_squared, sigma_abn, self.vars_vae_abn = self._build_encoder( self.hiddens) self.z_abn = tf.random_normal([dim_mult * self.dims[-1]], mean=self.mu_abn, stddev=sigma_abn) self.normal_abn = -0.5 * tf.reduce_sum( 1 + log_sigma_squared - tf.square(self.mu_abn) - sigma_squared, 1) u = tf.random_uniform(shape=(self.batch_size, dim_mult * self.dims[-1]), dtype=tf.float32) self.hidden_trans = layers.Dense(dim_mult * self.dims[-1], dim_mult * self.dims[-1], dropout=self.placeholders['dropout'], act=tf.nn.sigmoid) self.s = self.hidden_trans(self.hiddens) self.s_abn = tf.sigmoid((tf.log(self.s + 1e-20) + tf.log(u + 1e-20) - tf.log(1 - u + 1e-20)) / self.temperature) self.bernoulli_abn = tf.reduce_sum( tf.log(self.s + 1e-20) + tf.log(1 - self.s + 1e-20) - 2 * tf.log(0.5), 1) self.r_abn = tf.multiply(self.z_abn, self.s_abn) self.node_pred_abn = layers.Dense(dim_mult * self.dims[-1], self.num_classes, dropout=self.placeholders['dropout'], act=None) self.outputs_abn = self.node_pred_abn(self.r_abn) print(self.outputs_rec.get_shape()) self.outputs = tf.reduce_max(tf.concat([ tf.expand_dims(tf.nn.sigmoid(self.outputs_rec), -1), tf.expand_dims(tf.nn.sigmoid(self.outputs_abn), -1) ], axis=-1), axis=-1) print(self.outputs.get_shape()) self._loss() self.output_rec = tf.nn.sigmoid(self.outputs_rec) self.output_abn = tf.nn.sigmoid(self.outputs_abn) grads_and_vars = self.optimizer.compute_gradients(self.loss) clipped_grads_and_vars = [ (tf.clip_by_value(grad, -5.0, 5.0) if grad is not None else None, var) for grad, var in grads_and_vars ] self.grad, _ = clipped_grads_and_vars[0] self.opt_op = self.optimizer.apply_gradients(clipped_grads_and_vars)
import numpy as np import layers import functions x = np.asarray([[1, 1, 1], [2, 2, 2], [3, 3, 3]]) a = x.shape // 2 w = x[0, :] x = np.sqrt(1e-8) / np.sqrt(1e-8) net = layers.Network(layers.Adam(0.01)) net.add(layers.Dense(2, 5, functions.LeakyReLu(), '1st hidden')) net.add(layers.Dense(5, 20, functions.LeakyReLu(), '1st hidden')) net.add(layers.Dense(20, 5, functions.LeakyReLu(), '2nd hidden')) net.add(layers.Flatten()) net.add(layers.Dense(5, 3, functions.LeakyReLu(), '3nd hidden')) net.add(layers.Dense(3, 1, functions.Sigmoid(), 'Out')) inp = np.asarray([[0, 0], [1, 0], [0, 1], [1, 1]]) shape = inp.shape x = np.reshape(inp, (8)) y = np.reshape(x, shape) print(inp) target = np.asarray([0, 1, 1, 0]) print(target)
# number of attention heads num_heads = 2 # hidden layer size in feed forward network ff_dim = 32 # maximum length of each sentence max_len = splitting.max_len # implementing layers inputs = keras.Input(shape=(max_len, ), ) embedding_layer = TokenAndPositionEmbedding(max_len, len(preprocess.vocab)+1, embed_dim) x = embedding_layer(inputs) transformer_block = TransformerBlock(embed_dim, num_heads, ff_dim) x = transformer_block(x) x = layers.GlobalAveragePooling1D()(x) x = layers.Dropout(0.1)(x) x = layers.Dense(20, activation='relu')(x) x = layers.Dropout(0.1)(x) outputs = layers.Dense(7, activation='softmax')(x) # transformer model model = keras.Model(inputs=inputs, outputs=outputs) # summary of the model model.summary() # training and fitting the model to the data early_stopping = keras.callbacks.EarlyStopping() model.compile('adam', 'sparse_categorical_crossentropy', metrics=['accuracy']) history = model.fit( x_train, y_train, batch_size=32, epochs=10, validation_data=(x_val, y_val), callbacks=[early_stopping] )
# print("Predictions: ", Yhat) matches = (Yhat == Y.reshape(self.m)) # print(matches) return np.sum(matches) / len(matches) if __name__ == "__main__": nx = 10 m = 12 X = np.array(np.random.randn(nx, 12)) Y = np.array([[0, 1, 0, 1, 1, 0, 1, 0, 1, 0, 0, 1]]) nn = NeuralNetwork() nn.add_layer(layers.Flatten(nx)) nn.add_layer( layers.Dense(20, activation=ActivationFunctions.relu, initialization="He")) nn.add_layer( layers.Dense(30, activation=ActivationFunctions.relu, initialization="He")) nn.add_layer( layers.Dense(1, activation=ActivationFunctions.sigmoid, initialization="He")) costs = nn.train(X, Y, alpha=0.01, num_epochs=1500, print_cost=True) accuracy_on_train = nn.get_accuracy(X, Y) print(accuracy_on_train)
# Normalizing the data train_image = train_image / SCALER test_image = test_image / SCALER X_train, X_valid, y_train, y_valid = train_valid_split(train_image, train_label, test_size=test_size) N_train = X_train.shape[0] dimensions = X_train.shape[1] num_L1 = dimensions # Model model = dict() model['L1'] = layers.Dense(input_D=num_L1, output_D=num_L2) model['relu1'] = layers.Relu() model['L2'] = layers.Dense(input_D=num_L2, output_D=num_L3) model['loss'] = layers.SoftmaxCrossEntropy() for t in range(num_epoch): # Minibatch generate idx_permute = np.random.permutation(N_train) num_batches = int(N_train // batch_size) for i in range(num_batches): X_batch, y_batch = get_minibatch( X_train, y_train, idx_permute[i * batch_size:(i + 1) * batch_size]) a1 = model['L1'].forward(X_batch)
def build(self): dim_mult = 2 if self.concat else 1 self.outputs_rec, self.outputs_abn = [], [] #print(self.placeholders["labels_abn"].get_shape().as_list()) self.hidden_states = [] for timestamp in range(self.num_steps): print("timestamp:", timestamp) samples, support_size = self.sample(self.inputs, self.layer_infos, timestamp=timestamp) num_samples = [ layer_info.num_samples for layer_info in self.layer_infos ] if timestamp == 0: self.hiddens, self.aggregators = self.aggregate( samples, self.features[timestamp], self.dims, num_samples, support_size, concat=self.concat, model_size=self.model_size) else: self.hiddens, _ = self.aggregate(samples, self.features[timestamp], self.dims, num_samples, support_size, aggregators=self.aggregators, concat=self.concat, model_size=self.model_size) self.hiddens = tf.nn.l2_normalize(self.hiddens, 1) # Two Channel VAE if timestamp == 0: self.hidden_states.append( tf.ones_like(self.hiddens, dtype=tf.float32)) self.prior_noli_rec = layers.Dense( dim_mult * self.dims[-1], dim_mult * self.dims[-1], dropout=self.placeholders['dropout'], act=tf.nn.relu) self.prior_mu_rec = layers.Dense( dim_mult * self.dims[-1], dim_mult * self.dims[-1], dropout=self.placeholders['dropout'], act=None) self.prior_sigma_rec = layers.Dense( dim_mult * self.dims[-1], dim_mult * self.dims[-1], dropout=self.placeholders['dropout'], act=None) if timestamp == 0: self.mu_rec, log_sigma_squared_rec, sigma_squared_rec, sigma_rec, self.fc1_vae_rec, self.fc2_vae_rec, self.fc3_vae_rec = self._build_encoder( self.hiddens, self.hidden_states[-1]) else: self.mu_rec, log_sigma_squared_rec, sigma_squared_rec, sigma_rec, fc1, fc2, fc3 = self._build_encoder( self.hiddens, self.hidden_states[-1], fc1=self.fc1_vae_rec, fc2=self.fc2_vae_rec, fc3=self.fc3_vae_rec) self.z = tf.random_normal([dim_mult * self.dims[-1]], mean=self.mu_rec, stddev=sigma_rec) self.sigma_rec = sigma_rec # KL divergence with prior distribution and approximate postier distribution if timestamp == 0: self.normal_rec = -0.5 * tf.reduce_sum( 1 + log_sigma_squared_rec - tf.square(self.mu_rec) - sigma_squared_rec, 1) else: mu_pri, log_sigma_squared_pri, sigma_squared_pri, sigma_pri = self._build_prior( self.hidden_states[-1], fc1=self.prior_noli_rec, fc2=self.prior_mu_rec, fc3=self.prior_sigma_rec) sigma_pri = tf.math.reciprocal(sigma_pri + 1e-10) sigma_trace = tf.multiply(sigma_pri, sigma_rec) mu_sigma = tf.multiply(mu_pri - self.mu_rec, sigma_pri) mu_sigma = tf.multiply(sigma_pri, mu_pri - self.mu_rec) self.normal_rec += -0.5 * tf.reduce_sum( 1 + log_sigma_squared_rec - log_sigma_squared_pri - sigma_trace - mu_sigma, 1) if timestamp == 0: self.node_pred_rec = layers.Dense( dim_mult * self.dims[-1], self.num_classes, dropout=self.placeholders['dropout'], act=None) outputs_rec = self.node_pred_rec(self.z) u = tf.random_uniform(shape=(self.batch_size, FLAGS.num_classes), dtype=tf.float32) if timestamp == 0: self.bernoulli_trans = layers.Dense( 2 * dim_mult * self.dims[-1], FLAGS.num_classes, dropout=self.placeholders['dropout'], act=tf.nn.sigmoid) self.node_pred_abn = layers.Dense( 2 * dim_mult * self.dims[-1], FLAGS.num_classes, dropout=self.placeholders['dropout'], act=None) self.s = self.bernoulli_trans( tf.concat([self.z, self.hidden_states[-1]], axis=-1)) self.s_abn = tf.sigmoid( (tf.log(self.s + 1e-20) + tf.log(u + 1e-20) - tf.log(1 - u + 1e-20)) / self.temperature) self.z_abn = self.node_pred_abn( tf.concat([self.z, self.hidden_states[-1]], axis=-1)) self.r_abn = tf.multiply(self.z_abn, self.s_abn) outputs_abn = self.r_abn print(outputs_rec.get_shape()) self.outputs_rec.append(outputs_rec) self.outputs_abn.append(outputs_abn) #self.outputs = tf.reduce_max(tf.concat([tf.expand_dims(tf.nn.sigmoid(self.outputs_rec),-1) , tf.expand_dims(tf.nn.sigmoid(self.outputs_abn),-1) ], axis=-1), axis=-1) # output next hidden states if timestamp == 0: self.hidden_trans = layers.Dense( 3 * dim_mult * self.dims[-1] + FLAGS.num_classes, dim_mult * self.dims[-1], dropout=self.placeholders['dropout'], act=tf.nn.relu) next_hidden_state = self.hidden_trans( tf.concat([ self.hidden_states[-1], self.mu_rec, self.sigma_rec, self.s ], axis=-1)) self.hidden_states.append(tf.nn.l2_normalize(next_hidden_state, 1)) self._loss() self.output_rec = tf.nn.sigmoid(self.outputs_rec[-1]) self.output_abn = tf.nn.sigmoid(self.outputs_abn[-1]) self.outputs = tf.clip_by_value(self.output_abn, 1e-8, 1.0 - 1e-8) grads_and_vars = self.optimizer.compute_gradients(self.loss) clipped_grads_and_vars = [ (tf.clip_by_value(grad, -5.0, 5.0) if grad is not None else None, var) for grad, var in grads_and_vars ] self.grad, _ = clipped_grads_and_vars[0] self.opt_op = self.optimizer.apply_gradients(clipped_grads_and_vars)
import numpy as np import layers import functions net = layers.Network(layers.Adam(0.01)) net.add(layers.Dense(2, 5, functions.LeakyReLu(), '1st hidden')) net.add(layers.Dense(5, 20, functions.LeakyReLu(), '1st hidden')) net.add(layers.Dense(20, 5, functions.LeakyReLu(), '2nd hidden')) net.add(layers.Dense(5, 3, functions.LeakyReLu(), '3nd hidden')) net.add(layers.Dense(3, 2, functions.SoftMax(), 'Out')) inp = np.asarray([[0 , 0], [1, 0], [0, 1], [1, 1]]) shape = inp.shape x = np.reshape(inp, (8)) y = np.reshape(x, shape) print(inp) target = np.asarray([[1, 0], [0, 1], [0, 1], [1, 0]]) print(target) result = net.propagate(inp[0]) print(result) print("START") for _ in range(30): for i, inputX in enumerate(inp): result = net.propagate(inputX)
plt.plot(X_test, y_test) plt.plot(X_test, predictions) plt.show() ''' '' EPOCHS = 10001 LEARNING_RATE = 0.05 X_train, y_train = spiral_data(samples=100, classes=3) X_val, y_val = spiral_data(samples=100, classes=3) model = network.NeuralNetwork() model.add_layer( layers.Dense(2, 64, weight_regularizer_l2=0.000005, bias_regularizer_l2=0.000005)) model.add_layer(activations.ReLU()) model.add_layer(layers.Dropout(rate=0.2)) model.add_layer(layers.Dense(64, 3)) model.add_layer(activations.Softmax()) model.set(loss=losses.CategoricalCrossentropy(), optimizier=optimizers.Adam(learning_rate=LEARNING_RATE), accuracy=metrics.CategoricalAccuracy()) model.fit(X_train, y_train, epochs=EPOCHS, validation_data=(X_val, y_val))
def autoencoder(bioma_shape=717, domain_shape=36, output_shape=717, latent_space=10, bioma_layers=[128, 64], domain_layers=[32, 16], input_transform=CenterLogRatio(), output_transform=None, activation_function_encoder=layers.ReLU(), activation_function_decoder=layers.ReLU(), activation_function_latent='tanh', ): has_domain = domain_shape is not None has_bioma = bioma_shape is not None if not has_bioma and not has_domain: raise Exception('Either bioma or domain has to be expecified.') # encoder bioma if has_bioma: in_bioma = layers.Input(shape=(bioma_shape,), name='bioma_input_{}'.format(bioma_shape)) net = in_bioma if input_transform is not None: net = input_transform(net) for s in bioma_layers: net = layers.Dense(s, activation=activation_function_encoder, name="encoder_bioma_dense_{}".format(s))(net) encoded_bioma = layers.Dense(latent_space, activation=activation_function_latent, name='encoded_bioma_{}'.format(latent_space))(net) encoder_bioma = keras.Model(inputs=in_bioma, outputs=encoded_bioma, name='EncoderBioma') else: encoded_bioma = None encoder_bioma = None # encoder domain if has_domain: in_domain = layers.Input(shape=(domain_shape,), name='domain_input_{}'.format(domain_shape)) net = in_domain for s in domain_layers: net = layers.Dense(s, activation=activation_function_encoder, name="encoder_domain_dense_{}".format(s))(net) encoded_domain = layers.Dense(latent_space, activation=activation_function_latent, name='encoded_domain_{}'.format(latent_space))(net) encoder_domain = keras.Model(inputs=in_domain, outputs=encoded_domain, name='EncoderDomain') else: encoded_domain = None encoder_domain = None # decoder bioma for both autoencoders in_latent_space = layers.Input(shape=(latent_space,), name='latent_space_input') net = in_latent_space net_bioma = encoded_bioma net_domain = encoded_domain for s in reversed(bioma_layers): layer = layers.Dense(s, activation=activation_function_decoder, name="decoder_dense_{}".format(s)) net = layer(net) if has_bioma: net_bioma = layer(net_bioma) if has_domain: net_domain = layer(net_domain) layer = layers.Dense(output_shape, activation=None, name='decoded_bioma') decoded_bioma = layer(net) if has_bioma: net_bioma = layer(net_bioma) if has_domain: net_domain = layer(net_domain) if output_transform is not None: decoded_bioma = output_transform(decoded_bioma) if has_bioma: net_bioma = output_transform(net_bioma) if has_domain: net_domain = output_transform(net_domain) decoder_bioma = keras.Model(inputs=in_latent_space, outputs=decoded_bioma, name='DecoderBioma') # combined model for training if has_domain and has_bioma: diff_encoders = tf.math.abs(encoded_domain - encoded_bioma, name='diff_encoded') diff_encoders = Identity(name='latent')(diff_encoders) net_bioma = Identity(name='bioma')(net_bioma) net_domain = Identity(name='domain')(net_domain) model = keras.Model(inputs=[in_bioma, in_domain], outputs=[net_bioma, net_domain, diff_encoders], name='model') else: if has_bioma: net_bioma = Identity(name='bioma')(net_bioma) model = keras.Model(inputs=[in_bioma], outputs=[net_bioma], name='model') if has_domain: net_domain = Identity(name='domain')(net_domain) model = keras.Model(inputs=[in_domain], outputs=[net_domain], name='model') return model, encoder_bioma, encoder_domain, decoder_bioma
def __init__(self, nb_feature, nb_hidden, nb_class): self.dense1 = layers.Dense(N=nb_feature, H=nb_hidden) self.sigmoid = layers.Sigmoid() self.dense2 = layers.Dense(N=nb_hidden, H=nb_class) self.softmaxce = layers.SoftmaxCE()
model = get_model() model.train(training_data) validation_score = model.evaluate(validation_data) validation_scores.append(validation_score) validation_score = np.average(validation_scores) model = get_model() model.train(data) test_score = model.evaluate(test_data) #4.4.1 ネットワークのサイズを削減する from keras import models form keras import layers model = models.Sequential() model.add(layers.Dense(16, activation='relu', input_shape=(10000,))) model.add(layers.Dense(16, activation='relu')) model.add(layers.Dense(1, activation='sigmoid')) #小さくする model = models.Sequential() model.add(layers.Dense(4, activation='relu', input_shape=(10000,))) model.add(layers.Dense(4, activation='relu')) model.add(layers.Dense(1, activation='sigmoid')) #大きくする model =models.Sequential() model.add(layers.Dense(512, activaion='relu', input_shape=(10000,))) model.add(layers.Dense(512, activaion='relu')) model.add(layers.Dense(1, activaion='sigmoid'))