def _model_fn(self, features, labels, mode, params, config): embedding = tf.Variable(tf.truncated_normal( [256, params.embedding_size]), name='char_embedding') texts = features[base_model.TEXT_FEATURE_KEY] batch_size = tf.shape(texts)[0] byte_ids = tf.reshape( tf.cast( tf.decode_raw( tf.sparse_tensor_to_dense(tf.string_split(texts, ''), default_value='\0'), tf.uint8), tf.int32), [batch_size, -1]) padded_ids = tf.slice( tf.concat([ byte_ids, tf.zeros([batch_size, params.string_len], tf.int32) ], axis=1), [0, 0], [batch_size, params.string_len]) inputs = tf.nn.embedding_lookup(params=embedding, ids=padded_ids) # Conv X = inputs for filter_size in params.filter_sizes: X = layers.Conv1D(params.num_filters, filter_size, activation='relu', padding='same')(X) if params.pooling_type == 'average': X = layers.GlobalAveragePooling1D()(X) elif params.pooling_type == 'max': X = layers.GlobalMaxPooling1D()(X) else: raise ValueError('Unrecognized pooling type parameter') # FC logits = X for num_units in params.dense_units: logits = tf.layers.dense(inputs=logits, units=num_units, activation=tf.nn.relu) logits = tf.layers.dropout(logits, rate=params.dropout_rate) logits = tf.layers.dense(inputs=logits, units=len(self._target_labels), activation=None) output_heads = [ tf.contrib.estimator.binary_classification_head(name=name) for name in self._target_labels ] multihead = tf.contrib.estimator.multi_head(output_heads) optimizer = tf.train.AdamOptimizer(learning_rate=params.learning_rate) return multihead.create_estimator_spec(features=features, labels=labels, mode=mode, logits=logits, optimizer=optimizer)
def multi_output_model(): vocabulary_size = 50000 num_income_groups = 10 posts_input = Input(shape=(None,), dtype='int32', name='posts') embedded_posts = layers.Embedding(256, vocabulary_size)(posts_input) x = layers.Conv1D(128, 5, activation='relu')(embedded_posts) x = layers.MaxPooling1D(5)(x) x = layers.Conv1D(256, 5, activation='relu')(x) x = layers.Conv1D(256, 5, activation='relu')(x) x = layers.MaxPooling1D(5)(x) x = layers.Conv1D(256, 5, activation='relu')(x) x = layers.Conv1D(256, 5, activation='relu')(x) x = layers.GlobalMaxPooling1D()(x) x = layers.Dense(128, activation='relu')(x) # 两个输出 age_prediction = layers.Dense(1, name='age')(x) income_prediction = layers.Dense(num_income_groups, activation='softmax', name='income')(x) gender_prediction = layers.Dense(1, activation='sigmoid', name='gender')(x) model = Model(posts_input, [age_prediction, income_prediction, gender_prediction]) ''' #编译时选择多个损失标准 model.compile(optimizer='rmsprop', loss=['mse', 'categorical_crossentropy', 'binary_crossentropy']) model.compile(optimizer='rmsprop', loss={'age': 'mse', 'income': 'categorical_crossentropy', 'gender': 'binary_crossentropy'}) model.compile(optimizer='rmsprop', loss=['mse', 'categorical_crossentropy', 'binary_crossentropy'], loss_weights=[0.25, 1., 10.]) model.compile(optimizer='rmsprop', loss={'age': 'mse', 'income': 'categorical_crossentropy', 'gender': 'binary_crossentropy'}, loss_weights={'age': 0.25, 'income': 1., 'gender': 10.}) model.fit(posts, [age_targets, income_targets, gender_targets], epochs=10, batch_size=64) model.fit(posts, {'age': age_targets, 'income': income_targets, 'gender': gender_targets}, epochs=10, batch_size=64) ''' return model
def _model_fn(self, features, labels, mode, params, config): inputs = features[base_model.TOKENS_FEATURE_KEY] batch_size = tf.shape(inputs)[0] # Conv X = inputs for filter_size in params.filter_sizes: X = layers.Conv1D(params.num_filters, filter_size, activation='relu', padding='same')(X) if params.pooling_type == 'average': X = layers.GlobalAveragePooling1D()(X) elif params.pooling_type == 'max': X = layers.GlobalMaxPooling1D()(X) else: raise ValueError('Unrecognized pooling type parameter') # FC logits = X for num_units in params.dense_units: logits = tf.layers.dense(inputs=logits, units=num_units, activation=tf.nn.relu) logits = tf.layers.dropout(logits, rate=params.dropout_rate) logits = tf.layers.dense(inputs=logits, units=len(self._target_labels), activation=None) output_heads = [ tf.contrib.estimator.binary_classification_head(name=name) for name in self._target_labels ] multihead = tf.contrib.estimator.multi_head(output_heads) optimizer = tf.train.AdamOptimizer(learning_rate=params.learning_rate) return multihead.create_estimator_spec(features=features, labels=labels, mode=mode, logits=logits, optimizer=optimizer)
def _train(mutated, module_name): mutated = mutated[mutated['mod_keys_found_string'] == module_name] train_set, val_set, test_set = np.split( mutated.sample(frac=1), [int(.6 * len(mutated)), int(.8 * len(mutated))]) tasks_sent_train = [row for row in train_set['task_complete']] model_tasks3 = Word2Vec(tasks_sent_train, sg=0, size=100, window=6, min_count=1, workers=4, iter=1000) train_set['task_complete_one_string'] = train_set['task_complete'].apply( lambda x: list_to_string(x)) test_set['task_complete_one_string'] = test_set['task_complete'].apply( lambda x: list_to_string(x)) val_set['task_complete_one_string'] = val_set['task_complete'].apply( lambda x: list_to_string(x)) y_train = train_set['consistent'].astype(int) print(y_train.value_counts(), y_train.shape) y_test = test_set['consistent'].astype(int) print(y_test.value_counts(), y_test.shape) y_val = val_set['consistent'].astype(int) tokenizer_train = Tokenizer(lower=False) tokenizer_train.fit_on_texts(train_set['task_complete']) print(tokenizer_train) tokenizer_train = Tokenizer(lower=False) tokenizer_train.fit_on_texts(train_set['task_complete']) print(tokenizer_train) tokenizer_test = Tokenizer(lower=False) tokenizer_test.fit_on_texts(test_set['task_complete']) print(tokenizer_test) tokenizer_val = Tokenizer(lower=False) tokenizer_val.fit_on_texts(val_set['task_complete']) tasks_train_tokens = tokenizer_train.texts_to_sequences( train_set['task_complete_one_string']) tasks_test_tokens = tokenizer_test.texts_to_sequences( test_set['task_complete_one_string']) tasks_val_tokens = tokenizer_val.texts_to_sequences( val_set['task_complete_one_string']) num_tokens = [len(tokens) for tokens in tasks_train_tokens] num_tokens = np.array(num_tokens) np.max(num_tokens) np.argmax(num_tokens) max_tokens = np.mean(num_tokens) + 2 * np.std(num_tokens) max_tokens = int(max_tokens) tasks_train_pad = pad_sequences(tasks_train_tokens, maxlen=max_tokens, padding='post') tasks_test_pad = pad_sequences(tasks_test_tokens, maxlen=max_tokens, padding='post') tasks_val_pad = pad_sequences(tasks_val_tokens, maxlen=max_tokens, padding='post') embedding_size = 100 num_words = len(list(tokenizer_train.word_index)) + 1 embedding_matrix = np.random.uniform(-1, 1, (num_words, embedding_size)) for word, i in tokenizer_train.word_index.items(): if i < num_words: embedding_vector = model_tasks3[word] if embedding_vector is not None: embedding_matrix[i] = embedding_vector sequence_length = max_tokens batch_size = 256 tensorflow.compat.v1.disable_eager_execution() # CNN architecture num_classes = 2 # Training params num_epochs = 20 # Model parameters num_filters = 64 weight_decay = 1e-4 print("training CNN ...") model = Sequential() # Model add word2vec embedding model.add( Embedding( input_dim=num_words, output_dim=embedding_size, weights=[embedding_matrix], input_length=max_tokens, trainable=True, # the layer is trained name='embedding_layer')) model.add( layers.Conv1D(filters=num_filters, kernel_size=max_tokens, activation='relu', padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.MaxPooling1D(2)) model.add(Dropout(0.25)) model.add( layers.Conv1D(filters=num_filters + num_filters, kernel_size=max_tokens, activation='relu', padding='same', kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.GlobalMaxPooling1D()) model.add(Dropout(0.25)) model.add(layers.Flatten()) model.add( layers.Dense(128, activation='relu', kernel_regularizer=regularizers.l2(weight_decay))) model.add(layers.Dense(num_classes, activation='softmax')) sgd = SGD(lr=1e-2, decay=1e-6, momentum=0.9, nesterov=True) model.compile(loss=tensorflow.keras.losses.MeanAbsoluteError(), optimizer=sgd, metrics=['accuracy']) model.summary() model.fit(tasks_train_pad, to_categorical(y_train), batch_size=batch_size, epochs=num_epochs, validation_data=(tasks_test_pad, to_categorical(y_test)), shuffle=True, verbose=2) score = model.evaluate(tasks_val_pad, to_categorical(y_val), verbose=0) print('loss:', score[0]) print('Validation accuracy:', score[1]) y_pred = model.predict_classes(tasks_val_pad) cm = confusion_matrix(y_val, y_pred) tp = cm[1][1] fp = cm[0][1] fn = cm[1][0] tn = cm[0][0] precision = round(tp / (tp + fp), 2) print('Consistent: precision=%.3f' % (precision)) recall = round(tp / (tp + fn), 2) print('Consistent: recall=%.3f' % (recall)) f1_score = (2 * precision * recall) / (precision + recall) print('Consistent: f1_score=%.3f' % (f1_score)) precision_neg = round(tn / (tn + fn), 2) print('Inconsistent: precision=%.3f' % (precision_neg)) recall_neg = round(tn / (tn + fp), 2) print('Inconsistent: recall=%.3f' % (recall_neg)) f1_score_neg = (2 * precision_neg * recall_neg) / (precision_neg + recall_neg) print('Inconsistent: f1_score=%.3f' % (f1_score_neg)) ns_probs = [0 for _ in range(len(y_val))] ns_auc = roc_auc_score(y_val, ns_probs) lr_auc = roc_auc_score(y_val, y_pred) mcc = matthews_corrcoef(y_val, y_pred) print(precision) print('No Skill: ROC AUC=%.3f' % (ns_auc)) print('Our model: ROC AUC=%.3f' % (lr_auc)) print('Our model: MCC=%.3f' % (mcc)) json_out = {"module": module_name, "MCC": mcc, "AUC": lr_auc} model.save('models/' + module_name) return json_out
"""### Passing data to multi-input, multi-output models In the previous examples, we were considering a model with a single input (a tensor of shape `(764,)`) and a single output (a prediction tensor of shape `(10,)`). But what about models that have multiple inputs or outputs? Consider the following model, which has an image input of shape `(32, 32, 3)` (that's `(height, width, channels)`) and a timeseries input of shape `(None, 10)` (that's `(timesteps, features)`). Our model will have two outputs computed from the combination of these inputs: a "score" (of shape `(1,)`) and a probability distribution over 5 classes (of shape `(10,)`). """ image_input = keras.Input(shape=(32, 32, 3), name='img_input') timeseries_input = keras.Input(shape=(None, 10), name='ts_input') x1 = layers.Conv2D(3, 3)(image_input) x1 = layers.GlobalMaxPooling2D()(x1) x2 = layers.Conv1D(3, 3)(timeseries_input) x2 = layers.GlobalMaxPooling1D()(x2) x = layers.concatenate([x1, x2]) score_output = layers.Dense(1, name='score_output')(x) class_output = layers.Dense(5, activation='softmax', name='class_output')(x) model = keras.Model(inputs=[image_input, timeseries_input], outputs=[score_output, class_output]) """Let's plot this model, so you can clearly see what we're doing here (note that the shapes shown in the plot are batch shapes, rather than per-sample shapes).""" keras.utils.plot_model(model, 'multi_input_and_output_model.png', show_shapes=True) """At compilation time, we can specify different losses to different ouptuts, by passing the loss functions as a list:"""
layers.Embedding(dict_len, 64, input_length=seq_length), layers.Dropout(0.2), layers.SeparableConv1D(32, 5, activation='sigmoid', bias_initializer='random_uniform', depthwise_initializer='random_uniform', padding='same'), layers.MaxPooling1D(5), layers.SeparableConv1D(32, 5, activation='sigmoid', bias_initializer='random_uniform', depthwise_initializer='random_uniform', padding='same'), layers.GlobalMaxPooling1D(), layers.Dense(1, activation='sigmoid') ] networkCNN = Sequential(layers=the_cnn_layers) # Configure the network in preparation for training. networkCNN.compile(optimizer='adam', metrics=['accuracy'], loss='binary_crossentropy') # Train the model using the data. # Results in a history object that contains training and validation loss and metrics values. trainingCNN = networkCNN.fit( sarcasm_feature_sequence[train_ind], sarcasm_target_numpy[train_ind], validation_data=(sarcasm_feature_sequence[val_ind],
def _add_model_layers(self): self.model.add(layers.Embedding(input_dim=len(self.vocab_processor.vocabulary_), output_dim=self.X_train.shape[1])) self.model.add(layers.Conv1D(128, 5, activation='relu')) self.model.add(layers.GlobalMaxPooling1D()) self.model.add(layers.Dense(10, activation='relu')) self.model.add(layers.Dense(1, activation='sigmoid'))
max_features = 2000 max_len = 500 (x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features) x_train = sequence.pad_sequences(x_train, maxlen=max_len) x_test = sequence.pad_sequences(x_test, maxlen=max_len) model = models.Sequential() model.add(layers.Embedding(max_features, 128, input_length=max_len, name='embed')) model.add(layers.Conv1D(32, 7, activation='relu')) model.add(layers.MaxPooling1D(5)) model.add(layers.Conv1D(32, 7, activation='relu')) model.add(layers.GlobalMaxPooling1D()) model.add(layers.Dense(1)) model.summary() model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc']) callbacks = [ TensorBoard( log_dir='logs/', histogram_freq=1 ) ] history = model.fit(x_train, y_train, epochs=20, batch_size=128, validation_split=0.2,