def _model_fn(self, features, labels, mode, params, config):
embedding = tf.Variable(tf.truncated_normal(
[256, params.embedding_size]),
name='char_embedding')
texts = features[base_model.TEXT_FEATURE_KEY]
batch_size = tf.shape(texts)[0]
byte_ids = tf.reshape(
tf.cast(
tf.decode_raw(
tf.sparse_tensor_to_dense(tf.string_split(texts, ''),
default_value='\0'), tf.uint8),
tf.int32), [batch_size, -1])
padded_ids = tf.slice(
tf.concat([
byte_ids,
tf.zeros([batch_size, params.string_len], tf.int32)
],
axis=1), [0, 0], [batch_size, params.string_len])
inputs = tf.nn.embedding_lookup(params=embedding, ids=padded_ids)
# Conv
X = inputs
for filter_size in params.filter_sizes:
X = layers.Conv1D(params.num_filters,
filter_size,
activation='relu',
padding='same')(X)
if params.pooling_type == 'average':
X = layers.GlobalAveragePooling1D()(X)
elif params.pooling_type == 'max':
X = layers.GlobalMaxPooling1D()(X)
else:
raise ValueError('Unrecognized pooling type parameter')
# FC
logits = X
for num_units in params.dense_units:
logits = tf.layers.dense(inputs=logits,
units=num_units,
activation=tf.nn.relu)
logits = tf.layers.dropout(logits, rate=params.dropout_rate)
logits = tf.layers.dense(inputs=logits,
units=len(self._target_labels),
activation=None)
output_heads = [
tf.contrib.estimator.binary_classification_head(name=name)
for name in self._target_labels
]
multihead = tf.contrib.estimator.multi_head(output_heads)
optimizer = tf.train.AdamOptimizer(learning_rate=params.learning_rate)
return multihead.create_estimator_spec(features=features,
labels=labels,
mode=mode,
logits=logits,
optimizer=optimizer)
def multi_output_model():
vocabulary_size = 50000
num_income_groups = 10
posts_input = Input(shape=(None,), dtype='int32', name='posts')
embedded_posts = layers.Embedding(256, vocabulary_size)(posts_input)
x = layers.Conv1D(128, 5, activation='relu')(embedded_posts)
x = layers.MaxPooling1D(5)(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.MaxPooling1D(5)(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.GlobalMaxPooling1D()(x)
x = layers.Dense(128, activation='relu')(x)
# 两个输出
age_prediction = layers.Dense(1, name='age')(x)
income_prediction = layers.Dense(num_income_groups,
activation='softmax',
name='income')(x)
gender_prediction = layers.Dense(1, activation='sigmoid', name='gender')(x)
model = Model(posts_input,
[age_prediction, income_prediction, gender_prediction])
'''
#编译时选择多个损失标准
model.compile(optimizer='rmsprop',
loss=['mse', 'categorical_crossentropy', 'binary_crossentropy'])
model.compile(optimizer='rmsprop',
loss={'age': 'mse',
'income': 'categorical_crossentropy',
'gender': 'binary_crossentropy'})
model.compile(optimizer='rmsprop',
loss=['mse', 'categorical_crossentropy', 'binary_crossentropy'],
loss_weights=[0.25, 1., 10.])
model.compile(optimizer='rmsprop',
loss={'age': 'mse',
'income': 'categorical_crossentropy',
'gender': 'binary_crossentropy'},
loss_weights={'age': 0.25,
'income': 1.,
'gender': 10.})
model.fit(posts, [age_targets, income_targets, gender_targets],
epochs=10, batch_size=64)
model.fit(posts, {'age': age_targets,
'income': income_targets,
'gender': gender_targets},
epochs=10, batch_size=64)
'''
return model
def _model_fn(self, features, labels, mode, params, config):
inputs = features[base_model.TOKENS_FEATURE_KEY]
batch_size = tf.shape(inputs)[0]
# Conv
X = inputs
for filter_size in params.filter_sizes:
X = layers.Conv1D(params.num_filters,
filter_size,
activation='relu',
padding='same')(X)
if params.pooling_type == 'average':
X = layers.GlobalAveragePooling1D()(X)
elif params.pooling_type == 'max':
X = layers.GlobalMaxPooling1D()(X)
else:
raise ValueError('Unrecognized pooling type parameter')
# FC
logits = X
for num_units in params.dense_units:
logits = tf.layers.dense(inputs=logits,
units=num_units,
activation=tf.nn.relu)
logits = tf.layers.dropout(logits, rate=params.dropout_rate)
logits = tf.layers.dense(inputs=logits,
units=len(self._target_labels),
activation=None)
output_heads = [
tf.contrib.estimator.binary_classification_head(name=name)
for name in self._target_labels
]
multihead = tf.contrib.estimator.multi_head(output_heads)
optimizer = tf.train.AdamOptimizer(learning_rate=params.learning_rate)
return multihead.create_estimator_spec(features=features,
labels=labels,
mode=mode,
logits=logits,
optimizer=optimizer)
def _train(mutated, module_name):
mutated = mutated[mutated['mod_keys_found_string'] == module_name]
train_set, val_set, test_set = np.split(
mutated.sample(frac=1),
[int(.6 * len(mutated)),
int(.8 * len(mutated))])
tasks_sent_train = [row for row in train_set['task_complete']]
model_tasks3 = Word2Vec(tasks_sent_train,
sg=0,
size=100,
window=6,
min_count=1,
workers=4,
iter=1000)
train_set['task_complete_one_string'] = train_set['task_complete'].apply(
lambda x: list_to_string(x))
test_set['task_complete_one_string'] = test_set['task_complete'].apply(
lambda x: list_to_string(x))
val_set['task_complete_one_string'] = val_set['task_complete'].apply(
lambda x: list_to_string(x))
y_train = train_set['consistent'].astype(int)
print(y_train.value_counts(), y_train.shape)
y_test = test_set['consistent'].astype(int)
print(y_test.value_counts(), y_test.shape)
y_val = val_set['consistent'].astype(int)
tokenizer_train = Tokenizer(lower=False)
tokenizer_train.fit_on_texts(train_set['task_complete'])
print(tokenizer_train)
tokenizer_train = Tokenizer(lower=False)
tokenizer_train.fit_on_texts(train_set['task_complete'])
print(tokenizer_train)
tokenizer_test = Tokenizer(lower=False)
tokenizer_test.fit_on_texts(test_set['task_complete'])
print(tokenizer_test)
tokenizer_val = Tokenizer(lower=False)
tokenizer_val.fit_on_texts(val_set['task_complete'])
tasks_train_tokens = tokenizer_train.texts_to_sequences(
train_set['task_complete_one_string'])
tasks_test_tokens = tokenizer_test.texts_to_sequences(
test_set['task_complete_one_string'])
tasks_val_tokens = tokenizer_val.texts_to_sequences(
val_set['task_complete_one_string'])
num_tokens = [len(tokens) for tokens in tasks_train_tokens]
num_tokens = np.array(num_tokens)
np.max(num_tokens)
np.argmax(num_tokens)
max_tokens = np.mean(num_tokens) + 2 * np.std(num_tokens)
max_tokens = int(max_tokens)
tasks_train_pad = pad_sequences(tasks_train_tokens,
maxlen=max_tokens,
padding='post')
tasks_test_pad = pad_sequences(tasks_test_tokens,
maxlen=max_tokens,
padding='post')
tasks_val_pad = pad_sequences(tasks_val_tokens,
maxlen=max_tokens,
padding='post')
embedding_size = 100
num_words = len(list(tokenizer_train.word_index)) + 1
embedding_matrix = np.random.uniform(-1, 1, (num_words, embedding_size))
for word, i in tokenizer_train.word_index.items():
if i < num_words:
embedding_vector = model_tasks3[word]
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
sequence_length = max_tokens
batch_size = 256
tensorflow.compat.v1.disable_eager_execution()
# CNN architecture
num_classes = 2
# Training params
num_epochs = 20
# Model parameters
num_filters = 64
weight_decay = 1e-4
print("training CNN ...")
model = Sequential()
# Model add word2vec embedding
model.add(
Embedding(
input_dim=num_words,
output_dim=embedding_size,
weights=[embedding_matrix],
input_length=max_tokens,
trainable=True, # the layer is trained
name='embedding_layer'))
model.add(
layers.Conv1D(filters=num_filters,
kernel_size=max_tokens,
activation='relu',
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.MaxPooling1D(2))
model.add(Dropout(0.25))
model.add(
layers.Conv1D(filters=num_filters + num_filters,
kernel_size=max_tokens,
activation='relu',
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.GlobalMaxPooling1D())
model.add(Dropout(0.25))
model.add(layers.Flatten())
model.add(
layers.Dense(128,
activation='relu',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.Dense(num_classes, activation='softmax'))
sgd = SGD(lr=1e-2, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss=tensorflow.keras.losses.MeanAbsoluteError(),
optimizer=sgd,
metrics=['accuracy'])
model.summary()
model.fit(tasks_train_pad,
to_categorical(y_train),
batch_size=batch_size,
epochs=num_epochs,
validation_data=(tasks_test_pad, to_categorical(y_test)),
shuffle=True,
verbose=2)
score = model.evaluate(tasks_val_pad, to_categorical(y_val), verbose=0)
print('loss:', score[0])
print('Validation accuracy:', score[1])
y_pred = model.predict_classes(tasks_val_pad)
cm = confusion_matrix(y_val, y_pred)
tp = cm[1][1]
fp = cm[0][1]
fn = cm[1][0]
tn = cm[0][0]
precision = round(tp / (tp + fp), 2)
print('Consistent: precision=%.3f' % (precision))
recall = round(tp / (tp + fn), 2)
print('Consistent: recall=%.3f' % (recall))
f1_score = (2 * precision * recall) / (precision + recall)
print('Consistent: f1_score=%.3f' % (f1_score))
precision_neg = round(tn / (tn + fn), 2)
print('Inconsistent: precision=%.3f' % (precision_neg))
recall_neg = round(tn / (tn + fp), 2)
print('Inconsistent: recall=%.3f' % (recall_neg))
f1_score_neg = (2 * precision_neg * recall_neg) / (precision_neg +
recall_neg)
print('Inconsistent: f1_score=%.3f' % (f1_score_neg))
ns_probs = [0 for _ in range(len(y_val))]
ns_auc = roc_auc_score(y_val, ns_probs)
lr_auc = roc_auc_score(y_val, y_pred)
mcc = matthews_corrcoef(y_val, y_pred)
print(precision)
print('No Skill: ROC AUC=%.3f' % (ns_auc))
print('Our model: ROC AUC=%.3f' % (lr_auc))
print('Our model: MCC=%.3f' % (mcc))
json_out = {"module": module_name, "MCC": mcc, "AUC": lr_auc}
model.save('models/' + module_name)
return json_out
"""### Passing data to multi-input, multi-output models
In the previous examples, we were considering a model with a single input (a tensor of shape `(764,)`) and a single output (a prediction tensor of shape `(10,)`). But what about models that have multiple inputs or outputs?
Consider the following model, which has an image input of shape `(32, 32, 3)` (that's `(height, width, channels)`) and a timeseries input of shape `(None, 10)` (that's `(timesteps, features)`). Our model will have two outputs computed from the combination of these inputs: a "score" (of shape `(1,)`) and a probability distribution over 5 classes (of shape `(10,)`).
"""
image_input = keras.Input(shape=(32, 32, 3), name='img_input')
timeseries_input = keras.Input(shape=(None, 10), name='ts_input')
x1 = layers.Conv2D(3, 3)(image_input)
x1 = layers.GlobalMaxPooling2D()(x1)
x2 = layers.Conv1D(3, 3)(timeseries_input)
x2 = layers.GlobalMaxPooling1D()(x2)
x = layers.concatenate([x1, x2])
score_output = layers.Dense(1, name='score_output')(x)
class_output = layers.Dense(5, activation='softmax', name='class_output')(x)
model = keras.Model(inputs=[image_input, timeseries_input],
outputs=[score_output, class_output])
"""Let's plot this model, so you can clearly see what we're doing here (note that the shapes shown in the plot are batch shapes, rather than per-sample shapes)."""
keras.utils.plot_model(model, 'multi_input_and_output_model.png', show_shapes=True)
"""At compilation time, we can specify different losses to different ouptuts, by passing the loss functions as a list:"""
layers.Embedding(dict_len, 64, input_length=seq_length),
layers.Dropout(0.2),
layers.SeparableConv1D(32,
5,
activation='sigmoid',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'),
layers.MaxPooling1D(5),
layers.SeparableConv1D(32,
5,
activation='sigmoid',
bias_initializer='random_uniform',
depthwise_initializer='random_uniform',
padding='same'),
layers.GlobalMaxPooling1D(),
layers.Dense(1, activation='sigmoid')
]
networkCNN = Sequential(layers=the_cnn_layers)
# Configure the network in preparation for training.
networkCNN.compile(optimizer='adam',
metrics=['accuracy'],
loss='binary_crossentropy')
# Train the model using the data.
# Results in a history object that contains training and validation loss and metrics values.
trainingCNN = networkCNN.fit(
sarcasm_feature_sequence[train_ind],
sarcasm_target_numpy[train_ind],
validation_data=(sarcasm_feature_sequence[val_ind],
def _add_model_layers(self):
self.model.add(layers.Embedding(input_dim=len(self.vocab_processor.vocabulary_), output_dim=self.X_train.shape[1]))
self.model.add(layers.Conv1D(128, 5, activation='relu'))
self.model.add(layers.GlobalMaxPooling1D())
self.model.add(layers.Dense(10, activation='relu'))
self.model.add(layers.Dense(1, activation='sigmoid'))
max_features = 2000
max_len = 500
(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=max_features)
x_train = sequence.pad_sequences(x_train, maxlen=max_len)
x_test = sequence.pad_sequences(x_test, maxlen=max_len)
model = models.Sequential()
model.add(layers.Embedding(max_features, 128,
input_length=max_len,
name='embed'))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.MaxPooling1D(5))
model.add(layers.Conv1D(32, 7, activation='relu'))
model.add(layers.GlobalMaxPooling1D())
model.add(layers.Dense(1))
model.summary()
model.compile(optimizer='rmsprop', loss='binary_crossentropy', metrics=['acc'])
callbacks = [
TensorBoard(
log_dir='logs/',
histogram_freq=1
)
]
history = model.fit(x_train, y_train,
epochs=20,
batch_size=128,
validation_split=0.2,