# TokenFile.close() train_sequences = tokenizer.texts_to_sequences(x_train['comment']) test_sequences = tokenizer.texts_to_sequences(x_test['comment']) train_data = pad_sequences(train_sequences, maxlen=maxlen) test_data = pad_sequences(test_sequences, maxlen=maxlen) train_labels = np.asarray(y_train) test_labels = np.asarray(y_test) embedding_dim = 50 CNN_model = Sequential([ layers.Embedding(max_words, embedding_dim), layers.Conv1D(50, 7, activation='relu'), layers.MaxPooling1D(5), layers.Conv1D(50, 7, activation='relu'), layers.GlobalAveragePooling1D(), layers.Dense(20, activation='relu'), layers.Dense(1, activation='sigmoid') ]) RNN_model = Sequential([ layers.Embedding(max_words, embedding_dim), layers.Bidirectional(layers.LSTM(64)), layers.Dense(64, activation='relu'), layers.Dense(1, activation='sigmoid') ]) # CNN_model.summary()
import tensorflow as tf from tensorflow import keras from tensorflow.keras import layers import load_text embedding_dim = 300 inputs = keras.Input(shape=(54, ), name="input_text") embedding_layer = layers.Embedding(load_text.vocab_size, embedding_dim, name="embedding")(inputs) conv_3_layer = layers.Conv1D(100, 3, activation='relu', name="filter_size_3")(embedding_layer) conv_4_layer = layers.Conv1D(100, 4, activation='relu', name="filter_size_4")(embedding_layer) conv_5_layer = layers.Conv1D(100, 5, activation='relu', name="filter_size_5")(embedding_layer) max_pool_3_layer = layers.MaxPool1D(pool_size=52, name="max_pool_3", padding="same")(conv_3_layer) max_pool_4_layer = layers.MaxPool1D(pool_size=51, name="max_pool_4", padding="same")(conv_4_layer) max_pool_5_layer = layers.MaxPool1D(pool_size=50, name="max_pool_5", padding="same")(conv_5_layer) flatten_3_layer = layers.Flatten()(max_pool_3_layer) flatten_4_layer = layers.Flatten()(max_pool_4_layer) flatten_5_layer = layers.Flatten()(max_pool_5_layer) concatenate_layer = layers.concatenate( [flatten_3_layer, flatten_4_layer, flatten_5_layer])
plt.xlabel('Timestamps (Each value represents a 5 minute interval.)', fontsize=28) plt.ylabel('CPU Utilization', fontsize=28) plt.title('CPU Utilization Over a Period of Time', fontsize=32) plt.show() cpu_utilization_values = cpu_utilization.value.to_list() # print(cpu_utilization_values) normalized_cpu_utilization_values, training_mean, training_std = normalize(cpu_utilization_values) x_train = create_sequences(normalized_cpu_utilization_values, time_steps=32) print("Training data shape: ", x_train.shape) # Conv1D based auto-encoder model. model = keras.Sequential([layers.Input(shape=(x_train.shape[1], x_train.shape[2])), layers.Conv1D(filters=18, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Dropout(rate=0.2), layers.Conv1D(filters=9, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Conv1DTranspose(filters=9, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Dropout(rate=0.2), layers.Conv1DTranspose(filters=18, kernel_size=7, padding="same", strides=2, activation="relu"), layers.Conv1DTranspose(filters=1, kernel_size=7, padding="same")]) model.compile(optimizer=keras.optimizers.Adam(learning_rate=0.001), loss="mse") model.summary() history = model.fit(x_train, x_train, epochs=50, batch_size=128, validation_split=0.1, callbacks=[keras.callbacks.EarlyStopping(monitor="val_loss", patience=5, mode="min")]) # Plotting the training and validation losses.
) _, train_accuracy = model.evaluate(X_train, y_train) print(f'Training Accuracy: {train_accuracy:.4f}') _, test_accuracy = model.evaluate(X_test, y_test) print(f'Testing Accuracy: {test_accuracy:.4f}') plot_history(history) # Lesson 9 model = models.Sequential() model.add(layers.Embedding( input_dim=len(tokenizer.word_index) + 1, output_dim=100, input_length=100)) model.add(layers.Conv1D(128, 5, activation='relu')) model.add(layers.GlobalMaxPool1D()) model.add(layers.Dense(10, activation='relu')) model.add(layers.Dense(1, activation='sigmoid')) model.compile( loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'] ) model.summary() history = model.fit( X_train, y_train,
def get_text_encoding_layer(name, dataset, max_features=100, max_len=20, embedding_dim=20, layer_type="bi_rnn", n_units=20, dropout=0.3): """Creates everything that is needed for a textual input pipeline. Args: name (string): name of the feature dataset (tf.DataSet): tensorflow dataset max_features (int, optional): Maximum vocab size. Defaults to 100. max_len (int, optional): Sequence length to pad the outputs to. Defaults to 20. embedding_dim (int, optional): Embedding dimension. Defaults to 20. layer_type (str, optional): layer type to use (e.g., rnn, bi_rnn, gru, cnn). Defaults to "bi_rnn". n_units (int, optional): number of units within that layer. Defaults to 20. dropout (float, optional): dropout percentage. Defaults to 0.3. Returns: lambda function: textual input pipeline """ # max_features = 100 # Maximum vocab size. # max_len = 20 # Sequence length to pad the outputs to. # embedding_dim = 20 vec_layer = TextVectorization(max_tokens=max_features, output_mode='int', output_sequence_length=max_len) text_ds = dataset.map(lambda x, y: x[name]) vec_layer.adapt(text_ds) embedding_layer = layers.Embedding(max_features + 1, embedding_dim) dropout_layer = layers.Dropout(0.4) if layer_type == "cnn": # Conv1D + global max pooling conv_layer = layers.Conv1D(n_units, 5, padding='valid', activation='relu', strides=3) max_pooling_layer = layers.GlobalMaxPooling1D() return lambda feature: max_pooling_layer( conv_layer(dropout_layer(embedding_layer(vec_layer(feature))))) elif layer_type == "rnn": rnn_layer = layers.SimpleRNN(units=n_units, activation='relu', dropout=dropout) return lambda feature: rnn_layer( dropout_layer(embedding_layer(vec_layer(feature)))) elif layer_type == "bi_rnn": bi_rnn_layer = layers.Bidirectional( layers.SimpleRNN(units=n_units, activation='relu', dropout=dropout)) return lambda feature: bi_rnn_layer( dropout_layer(embedding_layer(vec_layer(feature)))) elif layer_type == "gru": gru_layer = layers.Bidirectional( layers.GRU(units=n_units, activation='relu', dropout=dropout)) return lambda feature: gru_layer( dropout_layer(embedding_layer(vec_layer(feature))))
def conv_bn(x, filter, kernel, activation): return layers.BatchNormalization()(layers.Conv1D(filter, kernel, activation=activation)(x))
def cnn_rnn_model(input_dim, filters, kernel_size, conv_stride, conv_border_mode, units, output_dim=155): ''' CNN + RNN ''' input_data = layers.Input(name='input', shape = input_dim) # convolutional layer conv_1d = layers.Conv1D(filters, kernel_size, strides=conv_stride, padding=conv_border_mode, activation='relu',name='conv1d')(input_data) # max pool max_pool = layers.MaxPooling1D(pool_size=4)(conv_1d) # batch normalization bn_cnn = layers.BatchNormalization(name='bn_conv_1d')(max_pool) # convolutional layer #2 conv_1d_2 = layers.Conv1D(filters, kernel_size, strides=conv_stride, padding=conv_border_mode, activation='relu',name='conv1d_2')(bn_cnn) # max pool max_pool = layers.MaxPooling1D(pool_size=4)(conv_1d_2) # batch normalization bn_cnn_2 = layers.BatchNormalization(name='bn_conv_1d_2')(max_pool) # convolutional layer #3 conv_1d_2 = layers.Conv1D(filters, kernel_size, strides=conv_stride, padding=conv_border_mode, activation='relu',name='conv1d_3')(bn_cnn_2) # max pool max_pool = layers.MaxPooling1D(pool_size=4)(conv_1d_2) # batch normalization bn_cnn_2 = layers.BatchNormalization(name='bn_conv_1d_3')(conv_1d_2) # convolutional layer #4 conv_1d_2 = layers.Conv1D(filters, kernel_size, strides=conv_stride, padding=conv_border_mode, activation='relu',name='conv1d_4')(bn_cnn_2) # max pool max_pool = layers.MaxPooling1D(pool_size=4)(conv_1d_2) # batch normalization bn_cnn_2 = layers.BatchNormalization(name='bn_conv_1d_4')(conv_1d_2) # convolutional layer #5 conv_1d_2 = layers.Conv1D(filters, kernel_size, strides=conv_stride, padding=conv_border_mode, activation='relu',name='conv1d_5')(bn_cnn_2) # max pool max_pool = layers.MaxPooling1D(pool_size=4)(conv_1d_2) # batch normalization bn_cnn_2 = layers.BatchNormalization(name='bn_conv_1d_5')(conv_1d_2) # recurrent layer simp_rnn = layers.SimpleRNN(units, activation='relu', return_sequences=True, implementation=2, name='rnn')(bn_cnn_2) # batch normalization bn_rnn = layers.BatchNormalization()(simp_rnn) # recurrent layer # simp_rnn = layers.SimpleRNN(units//4, activation='relu', return_sequences=True, implementation=2, name='rnn_2')(bn_rnn) # # batch normalization # bn_rnn = layers.BatchNormalization()(simp_rnn) flat = layers.Flatten()(bn_rnn) dense = layers.Dense(512)(flat) dense = layers.Dense(256)(dense) dense = layers.Dense(output_dim)(dense) # TimeDistributed(Dense(output_dim)) layer # time_dense = layers.TimeDistributed(layers.Dense(output_dim))(bn_rnn) # sigmoid activation layer y_pred = layers.Activation('sigmoid', name='sigmoid')(dense) model = models.Model(inputs=input_data, outputs=y_pred) #model.output_length = lambda x: cnn_output_length(x, kernel_size, conv_border_mode, conv_stride) print(model.summary()) return model
def multi_output_model_eg(): """ This function will detail the building and use of a multi-output model using a simple social-media example :return: None """ # Dummy variables to make sure there are no errors posts = None age_targets = None income_targets = None gender_targets = None vocabulary_size = 50000 num_income_groups = 10 # Make an input to hold the posts shared by users, variable length just like in the previous example post_input = Input(shape=(None, ), dtype='int32', name='posts') # Embed and encode these inputs. This time we'll use a larger Conv1D network embedded_posts = layers.Embedding(256, vocabulary_size)(post_input) x = layers.Conv1D(128, 5, activation='relu')(embedded_posts) x = layers.MaxPooling1D(5)(x) x = layers.Conv1D(256, 5, activation='relu')(x) x = layers.Conv1D(256, 5, activation='relu')(x) x = layers.MaxPooling1D(5)(x) x = layers.Conv1D(256, 5, activation='relu')(x) x = layers.Conv1D(256, 5, activation='relu')(x) x = layers.GlobalMaxPool1D()(x) x = layers.Dense(128, activation='relu')(x) # Now make 3 separate heads, one for each output. Note how all 3 have unique activations. This will require # unique loss functions for each head. age_prediction = layers.Dense(1, name='age')(x) income_prediction = layers.Dense(num_income_groups, activation='softmax', name='income')(x) gender_prediction = layers.Dense(1, activation='sigmoid', name='gender')(x) # Instantiate the model model = models.Model( post_input, [age_prediction, income_prediction, gender_prediction]) # Compile and fit the model. Use the dictionary method of passing parameters for greater explicitness. Note that # very imbalanced loss contributions will lead to the model being more optimised for the output with the largest # individual loss than for the others. To counteract this we can scale the individual loss functions to diminish # the largest contributor (mse ~= 3-5) and magnify the smallest contributor (binary_crossentropy ~= 0.1) model.compile(optimizer='rmsprop', loss={ 'age': 'mse', 'income': 'categorical_crossentropy', 'gender': 'binary_crossentropy' }, loss_weights={ 'age': 0.25, 'income': 1.0, 'gender': 10.0 }) model.fit(posts, { 'age': age_targets, 'income': income_targets, 'gender': gender_targets }, epochs=10, batch_size=64)
from tensorflow.keras import layers from tensorflow.keras import Input from tensorflow.keras.models import Model from tensorflow.keras.utils import to_categorical import numpy as np vocabulary_size = 100 num_income_groups = 10 num_samples = 1000 posts_input = Input(shape=(None, ), dtype='int32', name='posts') embedded_posts = layers.Embedding(256, vocabulary_size)(posts_input) x = layers.Conv1D(128, 5, activation='relu')(embedded_posts) x = layers.MaxPooling1D(5)(x) x = layers.Conv1D(256, 5, activation='relu')(x) x = layers.Conv1D(256, 5, activation='relu')(x) x = layers.MaxPooling1D(5)(x) x = layers.Conv1D(256, 5, activation='relu')(x) x = layers.Conv1D(256, 5, activation='relu')(x) x = layers.GlobalMaxPooling1D()(x) x = layers.Dense(128, activation='relu')(x) age_prediction = layers.Dense(1, name='age')(x) income_prediction = layers.Dense(num_income_groups, activation='softmax', name='income')(x) gender_prediction = layers.Dense(1, activation='sigmoid', name='gender')(x) model = Model(posts_input, [age_prediction, income_prediction, gender_prediction])
def create_conv1d(inputsize, layerlist, ncategories, config): """Make a CNN with convolutional layers, max pooling, and fully connected layers. The convolutional layers will have 'glorot_normal' initialization. To add one, include the tuple ("conv", conv_dict) in layerlist, where conv_dict is a dictionary minimally including the keys "filters" (number of filters) and "width" (kernel size). The max pooling layers will have 'valid' padding. For a max pooling layer, include the tuple ("maxpool", pool_size) in layerlist. To include a fully-connected layer, include the tuple ("fc", size, activation). To include identity elements, use ("startskip") and ("endskip") to add and remove layers to a list. The default behavior is to use a stack. You can specify ("endskip", 4) for finer-grained control over the element that is re-added. For example, using only ("endskip", 0) would use a queue instead of a stack. You can add arbitrary layers by including (layertobeadded) in layerlist. """ inputs = tf.keras.Input(shape=(inputsize, 1)) outputs = inputs checkpoints = [] for layer in layerlist: if layer[0] == "conv": # Defaults if not layer[1].get("padding"): layer[1]["padding"] = "same" if not layer[1].get("dilation"): layer[1]["dilation"] = 1 if not layer[1].get("activation"): layer[1]["activation"] = "relu" # Construction outputs = layers.Conv1D( filters=layer[1]["filters"], kernel_size=layer[1]["width"], padding=layer[1]["padding"], dilation_rate=layer[1]["dilation"], activation=layer[1]["activation"], kernel_initializer="glorot_normal", )(outputs) elif layer[0] == "maxpool": outputs = layers.MaxPooling1D(pool_size=layer[1], strides=2)(outputs) elif layer[0] == "fc": outputs = layers.Dense(layer[1], activation=layer[2])(outputs) elif layer[0] == "startskip": checkpoints.append(outputs) elif layer[0] == "endskip": if len(layer) < 2: skipped = checkpoints.pop() else: skipped = checkpoints[layer[1]] del checkpoints[layer[1]] outputs = layers.Add()([outputs, skipped]) else: outputs = layer[0](outputs) outputs = layers.Flatten()(outputs) outputs = layers.Dense( ncategories, activation="softmax", kernel_regularizer=config.get("regularizer"))(outputs) return tf.keras.Model(inputs=inputs, outputs=outputs, name="conv1d")
args = parser.parse_args() outputFile = args.outputFile # # Build my model # import tensorflow as tf from tensorflow.keras import layers from saphyra.layers import RpLayer, rvec # Ringer NN input_rings = layers.Input(shape=(100, ), name='Input_rings') #input_rings = RpLayer(rvec, name='RpLayer')(input_rings) conv_rings = layers.Reshape((100, 1))(input_rings) conv_rings = layers.Conv1D(16, kernel_size=2, name='conv_1', activation='relu')(conv_rings) conv_rings = layers.Conv1D(32, kernel_size=2, name='conv_2', activation='relu')(conv_rings) conv_output = layers.Flatten()(conv_rings) # Shower shape NN input_shower_shapes = layers.Input(shape=(5, ), name='Input_shower_shapes') dense_shower_shapes = layers.Dense( 4, activation='relu', name='dense_shower_shapes_1')(input_shower_shapes) # Decision NN input_concat = layers.Concatenate(axis=1)([conv_output, dense_shower_shapes]) dense = layers.Dense(32, activation='relu', name='dense_layer')(input_concat) dense = layers.Dense(1, activation='linear', name='output_for_inference')(dense) output = layers.Activation('sigmoid', name='output_for_training')(dense) # Build the model model = tf.keras.Model([input_rings, input_shower_shapes],
def build_cnn(config, input_tensor): # x = input # (-1, 520) # building_layers = [[99, 22], [66, 22], [33, 22]] # for units in building_layers: # config['building_layers']: # x = layers.Reshape((common_hl_output.shape[1], 1))(common_hl_output) sdae_model = sdae( input_data=rss_train, hidden_layers=config['sdae_hidden_layers'], cache=True, model_fname=None, optimizer=config['optimizer'], corruption_level=config['corruption_level'], batch_size=config['batch_size'], epochs=config['epochs'], validation_split=config['validation_split'], ) x = sdae_model(input_tensor) input = layers.Reshape((x.shape[1], 1))(x) # building classification output x = input x = layers.Conv1D(256, 3, activation='relu')(x) x = layers.Conv1D(128, 3, activation='relu')(x) x = layers.Conv1D(64, 3, activation='relu')(x) x = layers.Flatten()(x) building_output = layers.Dense(3, activation='softmax', name='building')(x) # (-1, 3) # floor classification output x = input x = layers.Conv1D(256, 3, activation='relu')(x) x = layers.MaxPooling1D(2)(x) x = layers.Conv1D(128, 3, activation='relu')(x) x = layers.MaxPooling1D(2)(x) x = layers.Conv1D(64, 3, activation='relu')(x) x = layers.Flatten()(x) floor_output = layers.Dense(5, activation='softmax', name='floor')(x) # (-1, 5) # coordinates regression output # for units in config['coordinates_layers']: x = input x = layers.Conv1D(256, 3, activation='relu')(x) x = layers.MaxPooling1D(2)(x) x = layers.Conv1D(128, 3, activation='relu')(x) x = layers.MaxPooling1D(2)(x) x = layers.Conv1D(64, 3, activation='relu')(x) x = layers.Flatten()(x) coordinates_output = layers.Dense(2, activation='linear', name='coord')(x) # (-1, 2) model = keras.Model( inputs=input_tensor, outputs=[building_output, floor_output, coordinates_output]) model.compile(optimizer=config['optimizer'], loss={ 'building': 'categorical_crossentropy', 'floor': 'categorical_crossentropy', 'coord': 'mse' }, loss_weights=[ config['building_weight'], config['floor_weight'], config['coordinates_weight'] ], metrics={ 'building': 'categorical_accuracy', 'floor': 'categorical_accuracy', 'coord': 'mse' }) return model
test_gen = generator(float_data, lookback=lookback, delay=delay, min_index=300001, max_index=None, step=step) val_steps = (300000 - 200001 - lookback) // batch_size test_steps = (len(float_data) - 300001 - lookback) // batch_size # 一种基本的机器学习方法 model = Sequential() model.add( layers.Conv1D(32, 5, activation='relu', input_shape=(None, float_data.shape[-1]))) model.add(layers.MaxPooling1D(3)) model.add(layers.Conv1D(32, 5, activation='relu')) model.add(layers.GRU(32, dropout=0.1, recurrent_dropout=0.5)) model.add(layers.Dense(1)) model.compile(optimizer=RMSprop(), loss='mae') history = model.fit(train_gen, steps_per_epoch=500, epochs=20, validation_data=val_gen, validation_steps=val_steps) loss = history.history['loss'] val_loss = history.history['val_loss']
def build_sender_model(n_images, input_image_shape, embedding_size, vocabulary_size, optimizer, sender_type="agnostic", image_embedding_layer=None, verbose=False, **kwargs): image_inputs = [ layers.Input(shape=input_image_shape, name=f"S_image_in_{i}", dtype="float32") for i in range(n_images) ] if not image_embedding_layer: image_embedding_layer = layers.Dense(embedding_size, name="S_image_embedding") # agnostic part activation = layers.Activation("sigmoid", name="S_sigmoid") output_layer = layers.Dense(vocabulary_size, name="S_output") # informed part stack = layers.Lambda(lambda x: K.stack(x, axis=1), name="S_stack") permute = layers.Permute([2, 1], name="S_permute") feature_filters = layers.Conv1D(filters=vocabulary_size, kernel_size=(1, ), input_shape=[n_images, embedding_size], data_format="channels_last", name="S_feature_filters") permute_2 = layers.Permute([2, 1], name="S_permute_2") vocabulary_filter = layers.Conv1D(1, kernel_size=(1, ), data_format="channels_last", name="S_vocabulary_filter") flatten = layers.Flatten() y = [image_embedding_layer(x) for x in image_inputs] if sender_type == "agnostic": y = layers.concatenate(y, axis=-1) y = activation(y) y = output_layer(y) elif sender_type == "informed": y = stack(y) y = permute(y) y = feature_filters(y) y = activation(y) y = permute_2(y) y = vocabulary_filter(y) y = flatten(y) # y = layers.Activation("relu")(y) # y = layers.Dense(vocabulary_size)(y) y = layers.Activation("sigmoid")(y) model_predict = models.Model(image_inputs, y, name="S_predict") model_predict.compile(loss=losses.binary_crossentropy, optimizer=optimizer) if verbose: model_predict.summary() return model_predict, model_predict
def Smi2Smi(): #product l_in = layers.Input(shape=(None, )) l_mask = layers.Input(shape=(None, )) #reagents l_dec = layers.Input(shape=(None, )) l_dmask = layers.Input(shape=(None, )) #positional encodings for product and reagents, respectively l_pos = PositionLayer(EMBEDDING_SIZE)(l_mask) l_dpos = PositionLayer(EMBEDDING_SIZE)(l_dmask) l_emask = MaskLayerRight()([l_dmask, l_mask]) l_right_mask = MaskLayerTriangular()(l_dmask) l_left_mask = MaskLayerLeft()(l_mask) #encoder l_voc = layers.Embedding(input_dim=vocab_size, output_dim=EMBEDDING_SIZE, input_length=None) l_embed = layers.Add()([l_voc(l_in), l_pos]) l_embed = layers.Dropout(rate=0.1)(l_embed) for layer in range(n_block): #self attention l_o = [ SelfLayer(EMBEDDING_SIZE, KEY_SIZE)([l_embed, l_embed, l_embed, l_left_mask]) for i in range(n_self) ] l_con = layers.Concatenate()(l_o) l_dense = layers.TimeDistributed(layers.Dense(EMBEDDING_SIZE))(l_con) l_drop = layers.Dropout(rate=0.1)(l_dense) l_add = layers.Add()([l_drop, l_embed]) l_att = LayerNormalization()(l_add) #position-wise l_c1 = layers.Conv1D(N_HIDDEN, 1, activation='relu')(l_att) l_c2 = layers.Conv1D(EMBEDDING_SIZE, 1)(l_c1) l_drop = layers.Dropout(rate=0.1)(l_c2) l_ff = layers.Add()([l_att, l_drop]) l_embed = LayerNormalization()(l_ff) #bottleneck l_encoder = l_embed l_embed = layers.Add()([l_voc(l_dec), l_dpos]) l_embed = layers.Dropout(rate=0.1)(l_embed) for layer in range(n_block): #self attention l_o = [ SelfLayer(EMBEDDING_SIZE, KEY_SIZE)([l_embed, l_embed, l_embed, l_right_mask]) for i in range(n_self) ] l_con = layers.Concatenate()(l_o) l_dense = layers.TimeDistributed(layers.Dense(EMBEDDING_SIZE))(l_con) l_drop = layers.Dropout(rate=0.1)(l_dense) l_add = layers.Add()([l_drop, l_embed]) l_att = LayerNormalization()(l_add) #attention to the encoder l_o = [ SelfLayer(EMBEDDING_SIZE, KEY_SIZE)([l_att, l_encoder, l_encoder, l_emask]) for i in range(n_self) ] l_con = layers.Concatenate()(l_o) l_dense = layers.TimeDistributed(layers.Dense(EMBEDDING_SIZE))(l_con) l_drop = layers.Dropout(rate=0.1)(l_dense) l_add = layers.Add()([l_drop, l_att]) l_att = LayerNormalization()(l_add) #position-wise l_c1 = layers.Conv1D(N_HIDDEN, 1, activation='relu')(l_att) l_c2 = layers.Conv1D(EMBEDDING_SIZE, 1)(l_c1) l_drop = layers.Dropout(rate=0.1)(l_c2) l_ff = layers.Add()([l_att, l_drop]) l_embed = LayerNormalization()(l_ff) l_out = layers.TimeDistributed(layers.Dense(vocab_size, use_bias=False))(l_embed) mdl = tf.keras.Model([l_in, l_mask, l_dec, l_dmask], l_out) def masked_loss(y_true, y_pred): loss = tf.nn.softmax_cross_entropy_with_logits_v2(labels=y_true, logits=y_pred) mask = tf.cast(tf.not_equal(tf.reduce_sum(y_true, -1), 0), 'float32') loss = tf.reduce_sum(loss * mask, -1) / tf.reduce_sum(mask, -1) loss = K.mean(loss) return loss def masked_acc(y_true, y_pred): mask = tf.cast(tf.not_equal(tf.reduce_sum(y_true, -1), 0), 'float32') eq = K.cast( K.equal(K.argmax(y_true, axis=-1), K.argmax(y_pred, axis=-1)), 'float32') eq = tf.reduce_sum(eq * mask, -1) / tf.reduce_sum(mask, -1) eq = K.mean(eq) return eq mdl.compile(optimizer='adam', loss=masked_loss, metrics=['accuracy', masked_acc]) mdl_enc = tf.keras.Model([l_in, l_mask], l_encoder) mdl_enc.compile(optimizer="adam", loss="categorical_crossentropy") #mdl.summary(); return mdl, mdl_enc
# Do async prefetching / buffering of the data for best performance on GPU. train_ds = train_ds.cache().prefetch(buffer_size=10) val_ds = val_ds.cache().prefetch(buffer_size=10) test_ds = test_ds.cache().prefetch(buffer_size=10) # A integer input for vocab indices. inputs = tf.keras.Input(shape=(None, ), dtype="int64") # Next, we add a layer to map those vocab indices into a space of dimensionality # 'embedding_dim'. x = layers.Embedding(max_features, embedding_dim)(inputs) x = layers.Dropout(0.5)(x) # Conv1D + global max pooling x = layers.Conv1D(128, 7, padding="valid", activation="relu", strides=3)(x) x = layers.Conv1D(128, 7, padding="valid", activation="relu", strides=3)(x) x = layers.GlobalMaxPooling1D()(x) # We add a vanilla hidden layer: x = layers.Dense(128, activation="relu")(x) x = layers.Dropout(0.5)(x) # We project onto a single unit output layer, and squash it with a sigmoid: predictions = layers.Dense(1, activation="sigmoid", name="predictions")(x) model = tf.keras.Model(inputs, predictions) # Compile the model with binary crossentropy loss and an adam optimizer. model.compile(loss="binary_crossentropy", optimizer="adam",
def detection_NN_architecture(Tinputs): feature_name = "detection" x = layers.Conv1D( 64, activation='relu', batch_input_shape=(None, _DATA_STREAM_LENGTH, 3), data_format='channels_last', kernel_size=10, strides=4, kernel_regularizer=tf.keras.regularizers.l2(l=0.001))(Tinputs) x = layers.Conv1D(64, activation='relu', kernel_size=10, strides=2, kernel_regularizer=tf.keras.regularizers.l2(l=0.001), padding='valid')(x) # x = layers.Dropout(0.1)(x) x = layers.Conv1D(64, activation='relu', kernel_size=10, strides=2, kernel_regularizer=tf.keras.regularizers.l2(l=0.001), padding='valid')(x) # x = layers.Dropout(0.1)(x) x = layers.Conv1D(64, activation='relu', kernel_size=10, strides=2, kernel_regularizer=tf.keras.regularizers.l2(l=0.001), padding='valid')(x) # x = layers.Dropout(0.1)(x) x = layers.Conv1D(64, activation='relu', kernel_size=10, strides=2, kernel_regularizer=tf.keras.regularizers.l2(l=0.001), padding='valid')(x) # x = layers.Dropout(0.1)(x) x = layers.Conv1D(64, activation='relu', kernel_size=10, strides=2, kernel_regularizer=tf.keras.regularizers.l2(l=0.001), padding='valid')(x) # #x = layers.Dropout(0.1)(x) # x = layers.Conv1D(64, activation='relu', kernel_size=10, strides=2, # kernel_regularizer=tf.keras.regularizers.l2(l=0.001), padding='valid')(x) # #x = layers.Dropout(0.1)(x) # x = layers.Conv1D(32, activation='relu', kernel_size=5, strides=2, # kernel_regularizer=tf.keras.regularizers.l2(l=0.001), padding='valid')(x) # x = layers.Dropout(0.25)(x) # x = layers.Conv1D(32, activation='elu', kernel_size=3, strides=2, # kernel_regularizer=tf.keras.regularizers.l2(l=0.001), padding='valid')(x) x = layers.Flatten()(x) last = layers.Dense( 64, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(l=0.001))(x) output = layers.Dense(3, activation='softmax', name=(feature_name + '_out'))(last) return output
from tensorflow.keras import layers from tensorflow.keras import optimizers from tensorflow.keras import losses from tensorflow.keras import metrics import numpy as np from numpy.random import random_sample # 网络搭建 image_input = tf.keras.Input(shape=(32, 32, 3), name="img_input") timeseries_input = tf.keras.Input(shape=(20, 10), name="ts_input") x1 = layers.Conv2D(3, 3)(image_input) x1 = layers.GlobalMaxPooling2D()(x1) x2 = layers.Conv1D(3, 3)(timeseries_input) x2 = layers.GlobalMaxPooling1D()(x2) x = layers.concatenate([x1, x2]) score_output = layers.Dense(1, name="score_output")(x) class_output = layers.Dense(5, name="class_output")(x) # 模型(对象)构建 model = tf.keras.Model(inputs=[image_input, timeseries_input], outputs=[score_output, class_output]) loss_score_object = losses.MeanSquaredError() loss_class_object = losses.CategoricalCrossentropy(from_logits=True) optimizer = tf.keras.optimizers.Adam()
def build_1d_model(args): l2r = 1e-9 T, X = tfkl.Input((N_TOKS,)), tfkl.Input((MAX_OBJS, 3 + N_OBJS)) # print('T: ', T.shape) # print('X: ', X.shape) ti = tfkl.Embedding(N_VOCAB, N_EMBED, input_length=N_TOKS)(T) # print('ti :', ti.shape) th = tfkm.Sequential([ tfkl.Bidirectional(tfkl.LSTM(128, return_sequences=True)), tfkl.Bidirectional(tfkl.LSTM(128, return_sequences=True)), tfkl.Conv1D(256, (1,), activation='elu', kernel_regularizer=tfkr.l2(l2r)), tfkl.Conv1D(6, (1,), activation=None, kernel_regularizer=tfkr.l2(l2r)), tfkl.Softmax(axis=-2, name='lstm_attn'), ], name='lstm_layers')(ti) # print('th: ', th.shape) tia = tfkb.sum(tfkl.Reshape((N_TOKS, 1, -1))(th) * tfkl.Reshape((N_TOKS, N_EMBED, 1))(ti), axis=-3) # print('tia: ', tia.shape) Xi = tfkb.sum(X[:, :, 3:], axis=-1, keepdims=True) # print('Xi: ', Xi.shape) s1 = tfkl.Dense(N_OBJS, activation='softmax')(tia[:, :, 0]) s1b = tfkm.Sequential([tfkl.RepeatVector(MAX_OBJS), tfkl.Reshape((MAX_OBJS, N_OBJS))])(s1) Xs1 = tfkb.sum(X[:, :, 3:] * s1b, axis=-1, keepdims=True) # print('s1: ', s1.shape) # print('s1b: ', s1b.shape) # print('Xs1: ', Xs1.shape) s2 = tfkl.Dense(3)(tia[:, :, 1]) s2b = tfkm.Sequential([tfkl.RepeatVector(MAX_OBJS), tfkl.Reshape((MAX_OBJS, 3))])(s2) s2c = tfkb.sum(s2b * X[:, :, 2:3] - (1 - Xi) * 20, axis=-1, keepdims=True) Xs2 = tfkm.Sequential([tfkl.Reshape((-1, 1)), tfkl.Softmax(axis=-2), tfkl.Reshape((MAX_OBJS, 1))])(s2c) Xs2 = Xs2 - tfkb.max(Xs2, axis=[1, 2], keepdims=True) # print('Xs2: ', Xs2.shape) s3 = tfkl.Dense(N_OBJS, activation='softmax')(tia[:, :, 2]) s3b = tfkm.Sequential([tfkl.RepeatVector(MAX_OBJS), tfkl.Reshape((MAX_OBJS, N_OBJS))])(s3) Xs3 = tfkb.sum(X[:, :, 3:] * s3b, axis=-1, keepdims=True) s4 = tfkl.Dense(16, activation='softmax')(tia[:, :, 3]) s4b = tfkm.Sequential([tfkl.RepeatVector(MAX_OBJS), tfkl.Reshape((MAX_OBJS, 16))])(s4) Xs4 = s4b * Xi # print('Xs4: ', Xs2.shape) s5 = tfkl.Dense(16, activation='softmax')(tia[:, :, 4]) s5b = tfkm.Sequential([tfkl.RepeatVector(MAX_OBJS), tfkl.Reshape((MAX_OBJS, 16))])(s5) Xs5 = s5b * Xi s6 = tfkl.Dense(16, activation='softmax')(tia[:, :, 5]) s6b = tfkm.Sequential([tfkl.RepeatVector(MAX_OBJS), tfkl.Reshape((MAX_OBJS, 16))])(s6) Xs6 = s6b * Xi xt = tfkl.concatenate([Xi, Xs1, Xs2, Xs3, Xs4, Xs5, Xs6], axis=-1) # print('xt: ', xt.shape) attn = fcnet(xt) # print('attn: ', attn.shape) Y = tfkb.sum(attn * X[:, :, :2], axis=[1]) # print('Y: ', Y.shape) model = tfkm.Model(inputs=[T, X], outputs=[Y]) def acc(y_pred, y_true): return tfkb.mean(tfkb.min(tfkb.cast((tfkb.abs(y_true-y_pred) < args.tol), 'float32'), axis=1)) model.compile(tfk.optimizers.Adam(args.lr), 'mse', metrics=[acc]) return model
def __init__(self, units_w1=2048, units_w2=512): super(FeedForward, self).__init__() self.W1 = layers.Conv1D(units_w1, kernel_size=1, activation='relu') self.W2 = layers.Conv1D(units_w2, kernel_size=1)
def DefineModel_textAndNumerics_VGG16x(self, oneshape: typing.List[int], numclasses: int) -> Model: # # 分叉 神经网络 text_input = Input(shape=(oneshape[0], 1), dtype=tf.float32, name='text_input') text_bn = layers.BatchNormalization(axis=1)(text_input) conv1D_1 = layers.Conv1D(32, 5, padding='same', activation=tf.nn.relu, name='conv1D_1')(text_bn) conv1D_2 = layers.Conv1D(64, 5, padding='same', activation=tf.nn.relu, name='conv1D_2')(conv1D_1) # maxPooling_1 = layers.MaxPooling1D(5)(conv1D_2) dropout_1 = layers.Dropout(0.25)(conv1D_2) numerics_input = Input(shape=(oneshape[1], 1), dtype=tf.float32, name='numerics_input') numerics_bn = layers.BatchNormalization(axis=1)(numerics_input) conv1D_3 = layers.Conv1D(32, 5, padding='same', activation=tf.nn.relu, name='conv1D_3')(numerics_bn) conv1D_4 = layers.Conv1D(64, 5, padding='same', activation=tf.nn.relu, name='conv1D_4')(conv1D_3) # maxPooling_2 = layers.MaxPooling1D(5)(conv1D_4) dropout_2 = layers.Dropout(0.25, name='dropout_2')(conv1D_4) concatenate_1 = layers.concatenate([dropout_1, dropout_2], axis=1) bn_1 = layers.BatchNormalization(axis=1)(concatenate_1) conv1D_vgg_1_1 = layers.Conv1D(64, 3, padding='same', activation=tf.nn.relu)(bn_1) conv1D_vgg_1_2 = layers.Conv1D(64, 3, padding='same', activation=tf.nn.relu)(conv1D_vgg_1_1) # maxpooling1D_vgg_1_3 = layers.MaxPooling1D(3)(conv1D_vgg_1_2) conv1D_vgg_2_1 = layers.Conv1D(128, 3, padding='same', activation=tf.nn.relu)(conv1D_vgg_1_2) conv1D_vgg_2_2 = layers.Conv1D(128, 3, padding='same', activation=tf.nn.relu)(conv1D_vgg_2_1) # maxpooling1D_vgg_2_3 = layers.MaxPooling1D(3)(conv1D_vgg_2_2) conv1D_vgg_3_1 = layers.Conv1D(256, 3, padding='same', activation=tf.nn.relu)(conv1D_vgg_2_2) conv1D_vgg_3_2 = layers.Conv1D(256, 3, padding='same', activation=tf.nn.relu)(conv1D_vgg_3_1) conv1D_vgg_3_3 = layers.Conv1D(256, 3, padding='same', activation=tf.nn.relu)(conv1D_vgg_3_2) # maxpooling1D_vgg_3_4 = layers.MaxPooling1D(3)(conv1D_vgg_3_3) # conv1D_vgg_4_1 = layers.Conv1D(512, 3, padding='same', activation=tf.nn.relu)(conv1D_vgg_3_3) # conv1D_vgg_4_2 = layers.Conv1D(512, 3, padding='same', activation=tf.nn.relu)(conv1D_vgg_4_1) # conv1D_vgg_4_3 = layers.Conv1D(512, 3, padding='same', activation=tf.nn.relu)(conv1D_vgg_4_2) # # maxpooling1D_vgg_4_4 = layers.MaxPooling1D(3)(conv1D_vgg_4_3) # conv1D_vgg_5_1 = layers.Conv1D(512, 3, padding='same', activation=tf.nn.relu)(conv1D_vgg_4_3) # conv1D_vgg_5_2 = layers.Conv1D(512, 3, padding='same', activation=tf.nn.relu)(conv1D_vgg_5_1) # conv1D_vgg_5_3 = layers.Conv1D(512, 3, padding='same', activation=tf.nn.relu)(conv1D_vgg_5_2) # maxpooling1D_vgg_5_4 = layers.MaxPooling1D(3)(conv1D_vgg_5_3) flatten_vgg_6_1 = layers.Flatten()(conv1D_vgg_3_3) # dense_vgg_6_2 = layers.Dense(int(2*(512+numclasses) / 3), activation=tf.nn.softmax)(flatten_vgg_6_1) # dense_vgg_6_3 = layers.Dense(int((512 + numclasses) / 3), activation=tf.nn.softmax)(dense_vgg_6_2) dense_vgg_6_4 = layers.Dense(numclasses, activation=tf.nn.softmax)(flatten_vgg_6_1) model = Model(inputs=[text_input, numerics_input], outputs=[dense_vgg_6_4]) return model
def dnn_model(): num_out = 6 model = keras.Sequential() model.add(layers.Conv1D(128, 1, input_shape=(26, 2), activation='relu')) model.add(layers.Conv1D(128, 1, input_shape=(26, 2), activation='relu')) model.add(layers.Conv1D(128, 1, input_shape=(26, 2), activation='relu')) model.add(layers.Conv1D(128, 1, input_shape=(26, 2), activation='relu')) model.add(layers.MaxPool1D(2)) model.add(layers.Conv1D(96, 1, input_shape=(26, 2), activation='relu')) model.add(layers.Conv1D(96, 1, input_shape=(26, 2), activation='relu')) model.add(layers.Conv1D(96, 1, input_shape=(26, 2), activation='relu')) model.add(layers.MaxPool1D(2)) # model.add(layers.Flatten()) model.add(layers.Conv1D(64, 1, input_shape=(26, 2), activation='relu')) model.add(layers.Conv1D(64, 1, input_shape=(26, 2), activation='relu')) model.add(layers.MaxPool1D(2)) model.add(layers.Conv1D(32, 1, input_shape=(26, 2), activation='relu')) model.add(layers.Conv1D(32, 1, input_shape=(26, 2), activation='relu')) model.add(layers.Flatten()) model.add(layers.Dropout(0.2)) model.add(layers.Dense(60, activation='relu')) model.add(layers.Dense(30, activation='relu')) model.add(layers.Dense(16, activation='relu')) model.add(layers.Dense(num_out, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return model
def build_model(input_shape, type="conv", mode="sup"): layers = [] if type == "conv": layers = [ l.Conv1D(32, 10, activation=gelu, strides=1, padding="same", input_shape=input_shape), l.Dropout(0.5), # ta.layers.Maxout(32), l.Conv1D(64, 10, activation=gelu, padding="same", strides=1), l.BatchNormalization(), l.Dropout(0.5), l.LocallyConnected1D(128, 1, activation=gelu, padding="same", implementation=2), l.BatchNormalization(), l.Conv1D(32, 10, activation=gelu, strides=1, padding="same", input_shape=input_shape), l.Dropout(0.5), # ta.layers.Maxout(32), l.Conv1D(64, 10, activation=gelu, padding="same", strides=5), l.BatchNormalization(), l.Dropout(0.5), l.LocallyConnected1D(128, 1, activation=gelu, padding="same", implementation=2), l.Conv1D(32, 10, activation=gelu, strides=1, padding="same", input_shape=input_shape), l.Dropout(0.5), # ta.layers.Maxout(32), l.Conv1D(64, 10, activation=gelu, padding="same", strides=1), l.BatchNormalization(), l.Dropout(0.5), l.LocallyConnected1D(128, 5, activation=gelu, padding="same", implementation=2), l.BatchNormalization(), l.BatchNormalization(), l.Flatten(), l.Dropout(0.4), l.Dense(7 * 7, activation="softmax") ] if type == "fc": layers = [ l.InputLayer(input_shape, dtype=tf.float32), l.Reshape((91, )), l.Dense(512, activation="tanh", input_shape=(91, )), l.BatchNormalization(), l.Dropout(0.5), l.Dense(512, activation="tanh"), l.BatchNormalization(), l.Dropout(0.5), l.Dense(7 * 7, activation="softmax") ] model = tf.keras.Sequential(layers) if mode == "sup": model.compile(optimizer=tf.keras.optimizers.Adam(lr=0.005), loss='categorical_crossentropy', metrics=[tf.keras.metrics.CategoricalAccuracy()]) else: model.build(input_shape=input_shape) if DEBUG: model.summary() return model
def Conv1D(self): input_shape = (int(self.SR * self.DT), 1) i = get_melspectrogram_layer(input_shape=input_shape, n_mels=128, pad_end=True, n_fft=512, win_length=400, hop_length=160, sample_rate=self.SR, return_decibel=True, input_data_format='channels_last', output_data_format='channels_last') x = LayerNormalization(axis=2, name='batch_norm')(i.output) x = TimeDistributed(layers.Conv1D(8, kernel_size=(4), activation='tanh'), name='td_conv_1d_tanh')(x) x = layers.MaxPooling2D(pool_size=(2, 2), name='max_pool_2d_1')(x) x = TimeDistributed(layers.Conv1D(16, kernel_size=(4), activation='relu'), name='td_conv_1d_relu_1')(x) x = layers.MaxPooling2D(pool_size=(2, 2), name='max_pool_2d_2')(x) x = TimeDistributed(layers.Conv1D(32, kernel_size=(4), activation='relu'), name='td_conv_1d_relu_2')(x) x = layers.MaxPooling2D(pool_size=(2, 2), name='max_pool_2d_3')(x) x = TimeDistributed(layers.Conv1D(64, kernel_size=(4), activation='relu'), name='td_conv_1d_relu_3')(x) x = layers.MaxPooling2D(pool_size=(2, 2), name='max_pool_2d_4')(x) x = TimeDistributed(layers.Conv1D(128, kernel_size=(4), activation='relu'), name='td_conv_1d_relu_4')(x) x = layers.GlobalMaxPooling2D(name='global_max_pooling_2d')(x) x = layers.Dropout(rate=0.1, name='dropout')(x) x = layers.Dense(64, activation='relu', activity_regularizer=l2(0.001), name='dense')(x) if self.N_CLASSES == 2: o = layers.Dense(1, activation='sigmoid', name='sigmoid')(x) self.model = Model(inputs=i.input, outputs=o, name='1d_convolution') self.model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) else: o = layers.Dense(self.N_CLASSES, activation='softmax', name='softmax')(x) self.model = Model(inputs=i.input, outputs=o, name='1d_convolution') self.model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) return self.model
def __init__(self): super(MyRnn, self).__init__() self.hidden_dim = hidden_dim self.projection1 = layers.Dense(units=hidden_dim, activation='relu') self.projection2 = layers.Dense(units=hidden_dim, activation='relu') self.classifier = layers.Dense(1, activation='sigmoid') def call(self, inputs): outs = [] states = tf.zeros(shape=[inputs.shape[0], self.hidden_dim]) for t in range(inputs.shape[1]): x = inputs[:, t, :] h = self.projection1(x) y = h + self.projection2(states) states = y outs.append(y) # print(outs) features = tf.stack(outs, axis=1) print(features.shape) return self.classifier(features) # 构建网络 inputs = keras.Input(batch_shape=(batch_size, time_step, inputs_dim)) x = layers.Conv1D(32, 3)(inputs) print(x.shape) outputs = MyRnn()(x) model = keras.Model(inputs, outputs) rnn_model = MyRnn() _ = rnn_model(tf.zeros((1, 10, 5)))
def createModel(self, generateGraph=False): """ Creates a brand new neural network for this model. """ # Should go over minutes, not seconds input_layer = layers.Input(shape=(SAMPLES_OF_DATA_TO_LOOK_AT, self._numberOfInputChannels)) # print(input_layer.shape) # layer = tf.transpose(input_layer, [0, 2, 1]) print(input_layer.shape) layer = layers.Conv1D( filters=64, kernel_size=9, activation='relu', input_shape=(SAMPLES_OF_DATA_TO_LOOK_AT, self._numberOfInputChannels), kernel_regularizer=regularizers.l2( self.hyperparameters.regularization))(input_layer) layer = tf.transpose(layer, [0, 2, 1]) # default number of units: layer.shape[2] print(layer.shape) forward_lstm = tf.keras.layers.LSTM( layer.shape[1], return_sequences=True, kernel_regularizer=regularizers.l2( self.hyperparameters.regularization)) backward_lstm = tf.keras.layers.LSTM( layer.shape[1], activation='relu', return_sequences=True, go_backwards=True, kernel_regularizer=regularizers.l2( self.hyperparameters.regularization)) layer = tf.keras.layers.Bidirectional(forward_lstm, backward_layer=backward_lstm, input_shape=layer.shape)(layer) print(layer.shape) layer = layers.Flatten()(layer) print(layer.shape) layer = layers.Dense(60, activation='relu', kernel_regularizer=regularizers.l2( self.hyperparameters.regularization))(layer) layer = tf.keras.layers.Dropout(self.hyperparameters.dropout)(layer) layer = layers.Dense(10, activation='relu', kernel_regularizer=regularizers.l2( self.hyperparameters.regularization))(layer) layer = tf.keras.layers.Dropout(self.hyperparameters.dropout)(layer) output = layers.Dense(1, activation='sigmoid', name="output", kernel_regularizer=regularizers.l2( self.hyperparameters.regularization))(layer) self.model = tf.keras.Model(input_layer, outputs=output) if self.binary: # This compiles the model if we are using binary prediction. self.model.compile(loss="binary_crossentropy", optimizer=tf.keras.optimizers.Adam( lr=self.hyperparameters.learningRate), metrics=self.listOfMetrics) else: # This compiles the model if we are using percentage prediction. self.model.compile(loss="mean_squared_error", optimizer=tf.keras.optimizers.Adam( lr=self.hyperparameters.learningRate), metrics=self.listOfMetrics) if generateGraph: tf.keras.utils.plot_model(self.model, "crypto_model.png", show_shapes=True)
from pylab import mpl mpl.rcParams['font.sans-serif'] = ['SimHei'] max_features = 10000 # 特征单词的个数 maxlen = 500 # 在这么多单词之后截断文本 # 加载数据 (x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features) # 填充序列 x_train = sequence.pad_sequences(x_train, maxlen=maxlen) # x_test = sequence.pad_sequences(x_test, maxlen=maxlen) model = Sequential() model.add(layers.Embedding(max_features, 128, input_length=maxlen)) model.add(layers.Conv1D(32, 7, activation='relu')) model.add(layers.MaxPooling1D(5)) model.add(layers.Conv1D(32, 7, activation='relu')) model.add(layers.GlobalMaxPooling1D()) model.add(layers.Dense(1)) model.summary() model.compile(optimizer=RMSprop(lr=1e-4), loss='binary_crossentropy', metrics=['acc']) history = model.fit(x_train, y_train, epochs=10, batch_size=128, validation_split=0.2)
def buildNetwork(): unfreeze = False l_in = layers.Input(shape=(None, )) l_mask = layers.Input(shape=(None, )) l_ymask = [] for i in range(len(props)): l_ymask.append(layers.Input(shape=(1, ))) #transformer part #positional encodings for product and reagents, respectively l_pos = PositionLayer(EMBEDDING_SIZE)(l_mask) l_left_mask = MaskLayerLeft()(l_mask) #encoder l_voc = layers.Embedding(input_dim=vocab_size, output_dim=EMBEDDING_SIZE, input_length=None, trainable=unfreeze) l_embed = layers.Add()([l_voc(l_in), l_pos]) for layer in range(n_block): #self attention l_o = [ SelfLayer(EMBEDDING_SIZE, KEY_SIZE, trainable=unfreeze)( [l_embed, l_embed, l_embed, l_left_mask]) for i in range(n_self) ] l_con = layers.Concatenate()(l_o) l_dense = layers.TimeDistributed(layers.Dense(EMBEDDING_SIZE, trainable=unfreeze), trainable=unfreeze)(l_con) if unfreeze == True: l_dense = layers.Dropout(rate=0.1)(l_dense) l_add = layers.Add()([l_dense, l_embed]) l_att = LayerNormalization(trainable=unfreeze)(l_add) #position-wise l_c1 = layers.Conv1D(N_HIDDEN, 1, activation='relu', trainable=unfreeze)(l_att) l_c2 = layers.Conv1D(EMBEDDING_SIZE, 1, trainable=unfreeze)(l_c1) if unfreeze == True: l_c2 = layers.Dropout(rate=0.1)(l_c2) l_ff = layers.Add()([l_att, l_c2]) l_embed = LayerNormalization(trainable=unfreeze)(l_ff) #end of Transformer's part l_encoder = l_embed #text-cnn part #https://github.com/deepchem/deepchem/blob/b7a6d3d759145d238eb8abaf76183e9dbd7b683c/deepchem/models/tensorgraph/models/text_cnn.py l_in2 = layers.Input(shape=(None, EMBEDDING_SIZE)) kernel_sizes = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20] num_filters = [100, 200, 200, 200, 200, 100, 100, 100, 100, 100, 160, 160] l_pool = [] for i in range(len(kernel_sizes)): l_conv = layers.Conv1D(num_filters[i], kernel_size=kernel_sizes[i], padding='valid', kernel_initializer='normal', activation='relu')(l_in2) l_maxpool = layers.Lambda(lambda x: tf.reduce_max(x, axis=1))(l_conv) l_pool.append(l_maxpool) l_cnn = layers.Concatenate(axis=1)(l_pool) l_cnn_drop = layers.Dropout(rate=0.25)(l_cnn) #dense part l_dense = layers.Dense(N_HIDDEN_CNN, activation='relu')(l_cnn_drop) #https://github.com/ParikhKadam/Highway-Layer-Keras transform_gate = layers.Dense( units=N_HIDDEN_CNN, activation="sigmoid", bias_initializer=tf.keras.initializers.Constant(-1))(l_dense) carry_gate = layers.Lambda(lambda x: 1.0 - x, output_shape=(N_HIDDEN_CNN, ))(transform_gate) transformed_data = layers.Dense(units=N_HIDDEN_CNN, activation="relu")(l_dense) transformed_gated = layers.Multiply()([transform_gate, transformed_data]) identity_gated = layers.Multiply()([carry_gate, l_dense]) l_highway = layers.Add()([transformed_gated, identity_gated]) #Because of multitask we have here a few different outputs and a custom loss. def mse_loss(prop): def loss(y_true, y_pred): y2 = y_true * l_ymask[prop] + y_pred * (1 - l_ymask[prop]) return tf.keras.losses.mse(y2, y_pred) return loss def binary_loss(prop): def loss(y_true, y_pred): y_pred = tf.clip_by_value(y_pred, K.epsilon(), 1.0 - K.epsilon()) r = y_true * K.log(y_pred) + (1.0 - y_true) * K.log(1.0 - y_pred) r = -tf.reduce_mean(r * l_ymask[prop]) return r return loss l_out = [] losses = [] for prop in props: if props[prop][2] == "regression": l_out.append( layers.Dense(1, activation='linear', name="Regression-" + props[prop][1])(l_highway)) losses.append(mse_loss(prop)) else: l_out.append( layers.Dense(1, activation='sigmoid', name="Classification-" + props[prop][1])(l_highway)) losses.append(binary_loss(prop)) l_input = [l_in2] l_input.extend(l_ymask) mdl = tf.keras.Model(l_input, l_out) mdl.compile(optimizer='adam', loss=losses) #mdl.summary(); K.set_value(mdl.optimizer.lr, 1.0e-4) #so far we do not train the encoder part of the model. encoder = tf.keras.Model([l_in, l_mask], l_encoder) encoder.compile(optimizer='adam', loss='mse') encoder.set_weights(np.load("embeddings.npy", allow_pickle=True)) #encoder.summary(); return mdl, encoder
def DenseNet(number, include_top=True, weights='hasc', input_shape=None, pooling=None, classes=6, classifier_activation='softmax'): if input_shape is None: input_shape = (256 * 3, 1) if weights in ['hasc', 'HASC'] and include_top and classes != 6: raise ValueError('If using `weights` as `"hasc"` with `include_top`' ' as true, `classes` should be 6') # number of blocks if number == 121: blocks = [6, 12, 24, 16] elif number == 169: blocks = [6, 12, 32, 32] elif number == 201: blocks = [6, 12, 48, 32] else: raise ValueError('`number` should be 121, 169 or 201') inputs = layers.Input(shape=input_shape) x = layers.ZeroPadding1D(padding=(3, 3))(inputs) x = layers.Conv1D(64, 7, strides=2, use_bias=False, name='conv1/conv')(x) x = layers.BatchNormalization(epsilon=1.001e-5, name='conv1/bn')(x) x = layers.Activation('relu', name='conv1/relu')(x) x = layers.ZeroPadding1D(padding=(1, 1))(x) x = layers.MaxPooling1D(3, strides=2, name='pool1')(x) x = DenseBlock(blocks[0], name='conv2')(x) x = TransitionBlock(0.5, name='pool2')(x) x = DenseBlock(blocks[1], name='conv3')(x) x = TransitionBlock(0.5, name='pool3')(x) x = DenseBlock(blocks[2], name='conv4')(x) x = TransitionBlock(0.5, name='pool4')(x) x = DenseBlock(blocks[3], name='conv5')(x) x = layers.BatchNormalization(epsilon=1.001e-5, name='bn')(x) x = layers.Activation('relu', name='relu')(x) x = layers.GlobalAveragePooling1D(name='avg_pool')(x) y = layers.Dense(classes, activation=classifier_activation, name='predictions')(x) # Create model. model_ = Model(inputs, y) if weights is not None: if weights in ['hasc', "HASC"]: weights = 'weights/densenet{}/densenet{}_hasc_weights_{}_{}.hdf5'.format( number, number, int(input_shape[0]), int(input_shape[1])) # hasc or weights fileで初期化 if os.path.exists(weights): print("Load weights from {}".format(weights)) model_.load_weights(weights) else: print("Not exist weights: {}".format(weights)) # topを含まないとき if not include_top: if pooling is None: # topを削除する model_ = Model(inputs=model_.input, outputs=model_.layers[-3].output) elif pooling == 'avg': y = layers.GlobalAveragePooling1D()(model_.layers[-3].output) model_ = Model(inputs=model_.input, outputs=y) elif pooling == 'max': y = layers.GlobalMaxPooling1D()(model_.layers[-3].output) model_ = Model(inputs=model_.input, outputs=y) else: print("Not exist pooling option: {}".format(pooling)) model_ = Model(inputs=model_.input, outputs=model_.layers[-3].output) return model_
print("test data done") train_data = Dataset.from_tensor_slices((Xtrain, Ytrain[:train_size])) test_data = Dataset.from_tensor_slices((Xtest, Ytest[:test_size])) train_data = train_data.batch(batch_size).repeat() test_data = test_data.batch(batch_size).repeat() readin_time = time() - t0 print(train_data.output_types) print(train_data.output_shapes) print('read in data time: %0.3fs' % readin_time) # 构建卷积神经网络模型 model = tf.keras.Sequential() model.add( layers.Conv1D(input_shape=(200, 100), filters=64, kernel_size=5, activation='relu')) model.add(layers.GlobalMaxPool1D()) model.add(layers.Dense(32, activation='relu')) model.add(layers.Dropout(rate=drop_prob)) model.add(layers.Dense(2, activation='softmax')) model.compile(optimizer=tf.keras.optimizers.Adam(lr), loss='categorical_crossentropy', metrics=['accuracy']) callbacks = [tf.keras.callbacks.TensorBoard(log_dir='./logs')] # 训练 model.fit(train_data, epochs=epochs, steps_per_epoch=20, callbacks=callbacks) # 测试