def build_cnn_model():
#Instantiate a Keras tensor
sequences= layers.Input(shape = (max_length, ))
#Turns positive integers (indexes) into dense vectors of fixed size
embedded = layers.Embedding(12000, 64) (sequences)
#Convolution kernel is convoled with the layer to produce a tensor of outputs
#(output_space, kernel_size, linear_unit_activation_function)
x = layers.Conv1D(64, 3, activation='relu') (embedded)
#Normalize and scale inputs or activations
x = layers.BatchNormalization() (x)
#Downsamples the input representation by taking the maximum value over the window
x = layers.MaxPool1D(3) (x)
x = layers.Conv1D(64, 5, activation='relu') (x)
x = layers.BatchNormalization() (x)
x = layers.MaxPool1D(5) (x)
x = layers.Conv1D(64, 5, activation='relu') (x)
#Downsamples the input representation by taking the maximum value over the time dimension
x = layers.GlobalMaxPool1D() (x)
x = layers.Flatten() (x)
#First parameter represents the dimension of the output space
x = layers.Dense(100, activation='relu') (x)
#Sigmoid function: values <-5 => value close to 0; values >5 => values close to 1
predictions = layers.Dense(1, activation='sigmoid') (x)
model = models.Model(inputs = sequences, outputs = predictions)
model.compile(
optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['binary_accuracy']
)
return model
def multi_output_model():
vocabulary_size = 50000
num_income_groups = 10
posts_input = Input(shape=(None,), dtype='int32', name='posts')
embedded_posts = layers.Embedding(256, vocabulary_size)(posts_input)
x = layers.Conv1D(128, 5, activation='relu')(embedded_posts)
x = layers.MaxPooling1D(5)(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.MaxPooling1D(5)(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.Conv1D(256, 5, activation='relu')(x)
x = layers.GlobalMaxPooling1D()(x)
x = layers.Dense(128, activation='relu')(x)
# 两个输出
age_prediction = layers.Dense(1, name='age')(x)
income_prediction = layers.Dense(num_income_groups,
activation='softmax',
name='income')(x)
gender_prediction = layers.Dense(1, activation='sigmoid', name='gender')(x)
model = Model(posts_input,
[age_prediction, income_prediction, gender_prediction])
'''
#编译时选择多个损失标准
model.compile(optimizer='rmsprop',
loss=['mse', 'categorical_crossentropy', 'binary_crossentropy'])
model.compile(optimizer='rmsprop',
loss={'age': 'mse',
'income': 'categorical_crossentropy',
'gender': 'binary_crossentropy'})
model.compile(optimizer='rmsprop',
loss=['mse', 'categorical_crossentropy', 'binary_crossentropy'],
loss_weights=[0.25, 1., 10.])
model.compile(optimizer='rmsprop',
loss={'age': 'mse',
'income': 'categorical_crossentropy',
'gender': 'binary_crossentropy'},
loss_weights={'age': 0.25,
'income': 1.,
'gender': 10.})
model.fit(posts, [age_targets, income_targets, gender_targets],
epochs=10, batch_size=64)
model.fit(posts, {'age': age_targets,
'income': income_targets,
'gender': gender_targets},
epochs=10, batch_size=64)
'''
return model
def main():
vocabulary_size = 10000
maxlen = 24
model = Sequential()
model.add(layers.Embedding(vocabulary_size, 64, name="text"))
model.add(
layers.Conv1D(64, 4, padding='valid', activation='relu', strides=1))
model.add(layers.MaxPooling1D(pool_size=3))
model.add(layers.LSTM(64))
model.add(layers.Dense(32, activation='relu'))
model.add(layers.Dense(1, activation='sigmoid'))
# # if use keras not tf.keras
# model = tf.keras.models.Model(model)
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['acc'])
estimator_model = tf.keras.estimator.model_to_estimator(keras_model=model)
data, labels = mr_load_data(max_word_num=vocabulary_size)
data = pad_sequences(data, padding="pre", maxlen=maxlen)
labels = np.asarray(labels).reshape(-1, 1)
print(labels.shape)
x_train, y_train = data, labels
input_dict = {"text_input": x_train}
input_fn = train_input_fn(input_dict, y_train, batch_size=32)
print(input_fn)
#
# estimator_model.train(input_fn=input_fn, steps=10000)
estimator_model.train(input_fn=input_function(input_dict, y_train),
steps=10000)
def _get_keras_model(self) -> models.Model:
I = layers.Input(shape=(KerasCNNModel.MAX_SEQUENCE_LENGTH, 300),
dtype='float32',
name='comment_text')
# Convolutional Layers
X = I
for filter_size in self.hparams().filter_sizes:
X = layers.Conv1D(self.hparams().num_filters,
filter_size,
activation='relu',
padding='same')(X)
X = layers.GlobalAveragePooling1D()(X)
# Dense
for num_units in self.hparams().dense_units:
X = layers.Dense(num_units, activation='relu')(X)
X = layers.Dropout(self.hparams().dropout_rate)(X)
# Outputs
outputs = []
for label in self._labels:
outputs.append(
layers.Dense(1, activation='sigmoid', name=label)(X))
model = models.Model(inputs=I, outputs=outputs)
model.compile(
optimizer=optimizers.Adam(lr=self.hparams().learning_rate),
loss='binary_crossentropy',
metrics=['binary_accuracy', super().roc_auc])
tf.logging.info(model.summary())
return model
def _model_fn(self, features, labels, mode, params, config):
embedding = tf.Variable(tf.truncated_normal(
[256, params.embedding_size]),
name='char_embedding')
texts = features[base_model.TEXT_FEATURE_KEY]
batch_size = tf.shape(texts)[0]
byte_ids = tf.reshape(
tf.cast(
tf.decode_raw(
tf.sparse_tensor_to_dense(tf.string_split(texts, ''),
default_value='\0'), tf.uint8),
tf.int32), [batch_size, -1])
padded_ids = tf.slice(
tf.concat([
byte_ids,
tf.zeros([batch_size, params.string_len], tf.int32)
],
axis=1), [0, 0], [batch_size, params.string_len])
inputs = tf.nn.embedding_lookup(params=embedding, ids=padded_ids)
# Conv
X = inputs
for filter_size in params.filter_sizes:
X = layers.Conv1D(params.num_filters,
filter_size,
activation='relu',
padding='same')(X)
if params.pooling_type == 'average':
X = layers.GlobalAveragePooling1D()(X)
elif params.pooling_type == 'max':
X = layers.GlobalMaxPooling1D()(X)
else:
raise ValueError('Unrecognized pooling type parameter')
# FC
logits = X
for num_units in params.dense_units:
logits = tf.layers.dense(inputs=logits,
units=num_units,
activation=tf.nn.relu)
logits = tf.layers.dropout(logits, rate=params.dropout_rate)
logits = tf.layers.dense(inputs=logits,
units=len(self._target_labels),
activation=None)
output_heads = [
tf.contrib.estimator.binary_classification_head(name=name)
for name in self._target_labels
]
multihead = tf.contrib.estimator.multi_head(output_heads)
optimizer = tf.train.AdamOptimizer(learning_rate=params.learning_rate)
return multihead.create_estimator_spec(features=features,
labels=labels,
mode=mode,
logits=logits,
optimizer=optimizer)
def __init__(self, ch, kernel_size=3, strides=1, padding='same'):
super().__init__()
self.model = keras.Sequential([
layers.Conv1D(ch, kernel_size, strides=strides, padding='same'),
# layers.Dropout(0.1),
layers.BatchNormalization(),
layers.Activation('relu')
])
def create_lstm():
model = tf.keras.Sequential()
model.add(layers.Embedding(MAX_WORDS, 64, input_length=MAX_LEN))
model.add(layers.Conv1D(32, 3, padding='same', activation='relu'))
model.add(layers.MaxPooling1D(pool_size=4))
model.add(layers.LSTM(64))
model.add(layers.Dense(250, activation='relu'))
model.add(layers.Dense(1, activation="sigmoid"))
return model
def build_model():
sequences = layers.Input(shape=(MAX_LENGTH, ))
embedded = layers.Embedding(MAX_FEATURES, 64)(sequences)
x = layers.Conv1D(64, 3, activation='relu')(embedded)
x = layers.BatchNormalization()(x)
x = layers.MaxPool1D(3)(x)
x = layers.Conv1D(64, 5, activation='relu')(x)
x = layers.BatchNormalization()(x)
x = layers.MaxPool1D(5)(x)
x = layers.Conv1D(64, 5, activation='relu')(x)
x = layers.GlobalMaxPool1D()(x)
x = layers.Flatten()(x)
x = layers.Dense(100, activation='relu')(x)
predictions = layers.Dense(1, activation='sigmoid')(x)
model = models.Model(inputs=sequences, outputs=predictions)
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['binary_accuracy'])
return model
def __call__(self, model):
with tf.name_scope("Convs") as scope:
sc = KL.MaxPooling1D(8)(KL.Conv1D(self.dim * 3,
8,
activation='relu',
padding="same")(model))
model = KL.MaxPooling1D(2)(KL.Conv1D(self.dim,
4,
activation='relu',
padding="same")(model))
model = KL.MaxPooling1D(2)(KL.Conv1D(self.dim * 2,
4,
activation='relu',
padding="same")(model))
model = KL.MaxPooling1D(2)(KL.Conv1D(self.dim * 3,
4,
activation='relu',
padding="same")(model))
model = KL.add([model, sc])
model = KL.Dropout(0.25)(model)
return model
def __init__(self, filter_num, channel, stride=1, reduction=16):
super().__init__()
self.conv1 = layers.Conv1D(filter_num,
3,
strides=stride,
padding='same')
self.bn1 = layers.BatchNormalization()
self.relu = layers.Activation('relu')
self.conv2 = layers.Conv1D(filter_num, 3, strides=1, padding='same')
self.bn2 = layers.BatchNormalization()
self.add = layers.Add()
self.se = SELayer(channel, reduction)
self.se2 = SELayer(channel, reduction)
if stride != 1:
self.downsample = Sequential()
self.downsample.add(layers.Conv1D(filter_num, 1, strides=stride))
self.downsample.add(layers.BatchNormalization())
else:
self.downsample = lambda x: x
def call(self, x):
#print(x.shape) # [32, 1014]
lookup = self._lookup(x)
#print(lookup.shape) # [32, 1014, 256]
conv = layers.Conv1D(filters=FLAGS.embedding_dim,
activation=tf.nn.relu,
kernel_size=7)(lookup)
#print(conv.shape) # [32, 1008, 256]
conv = self._maxpooling(conv)
#print(conv.shape) # [32, 336, 256]
conv = layers.Conv1D(filters=FLAGS.embedding_dim,
activation=tf.nn.relu,
kernel_size=7)(conv)
conv = self._maxpooling(conv)
#print(conv.shape) # [32, 110, 256]
conv = layers.Conv1D(filters=FLAGS.embedding_dim,
activation=tf.nn.relu,
kernel_size=3)(conv)
conv = layers.Conv1D(filters=FLAGS.embedding_dim,
activation=tf.nn.relu,
kernel_size=3)(conv)
conv = layers.Conv1D(filters=FLAGS.embedding_dim,
activation=tf.nn.relu,
kernel_size=3)(conv)
conv = layers.Conv1D(filters=FLAGS.embedding_dim,
activation=tf.nn.relu,
kernel_size=3)(conv)
conv = self._maxpooling(conv)
#print(conv.shape) # [32, 34, 256]
return layers.Flatten()(conv)
def convolution():
inn = layers.Input(shape=(sequence_length,1))
cnns = []
for i,size in enumerate(filter_sizes):
conv = layers.Conv1D(filters=8, kernel_size=(size),
strides=size, padding='valid', activation='relu',kernel_initializer=initConv([size,8]))(inn)
#if i%2:
pool_size =int(conv.shape[1]/100)
pool = layers.MaxPool1D(pool_size=(pool_size), padding='valid')(conv)
#pool = MaxMin(pool_size)(conv)
cnns.append(pool)
outt = layers.concatenate(cnns)
model = keras.Model(inputs=inn, outputs=outt,name='cnns')
model.summary()
return model
def _model_fn(self, features, labels, mode, params, config):
inputs = features[base_model.TOKENS_FEATURE_KEY]
batch_size = tf.shape(inputs)[0]
# Conv
X = inputs
for filter_size in params.filter_sizes:
X = layers.Conv1D(params.num_filters,
filter_size,
activation='relu',
padding='same')(X)
if params.pooling_type == 'average':
X = layers.GlobalAveragePooling1D()(X)
elif params.pooling_type == 'max':
X = layers.GlobalMaxPooling1D()(X)
else:
raise ValueError('Unrecognized pooling type parameter')
# FC
logits = X
for num_units in params.dense_units:
logits = tf.layers.dense(inputs=logits,
units=num_units,
activation=tf.nn.relu)
logits = tf.layers.dropout(logits, rate=params.dropout_rate)
logits = tf.layers.dense(inputs=logits,
units=len(self._target_labels),
activation=None)
output_heads = [
tf.contrib.estimator.binary_classification_head(name=name)
for name in self._target_labels
]
multihead = tf.contrib.estimator.multi_head(output_heads)
optimizer = tf.train.AdamOptimizer(learning_rate=params.learning_rate)
return multihead.create_estimator_spec(features=features,
labels=labels,
mode=mode,
logits=logits,
optimizer=optimizer)
def _add_model_layers(self):
self.model.add(layers.Embedding(input_dim=len(self.vocab_processor.vocabulary_), output_dim=self.X_train.shape[1]))
self.model.add(layers.Conv1D(128, 5, activation='relu'))
self.model.add(layers.GlobalMaxPooling1D())
self.model.add(layers.Dense(10, activation='relu'))
self.model.add(layers.Dense(1, activation='sigmoid'))
#define rnn model rnn = Sequential(name='rnn') rnn.add(layers.BatchNormalization(input_shape=(None, num_channels*30*FreqSample/step))) rnn.add(layers.LSTM(1000,dropout=0.1, recurrent_dropout=0.1,return_sequences=True)) rnn.add(layers.LSTM(1000,dropout=0.1, recurrent_dropout=0.1,return_sequences=True)) rnn.add(layers.LSTM(1000,dropout=0.1, recurrent_dropout=0.1,return_sequences=True)) rnn.add(layers.LSTM(1000,dropout=0.1, recurrent_dropout=0.1,return_sequences=True)) rnn.add(layers.LSTM(1000,dropout=0.1, recurrent_dropout=0.1)) rnn.add(layers.Dense(5, activation="softmax")) rnn.summary() #define rcnn model rcnn = Sequential(name='rcnn') rcnn.add(layers.BatchNormalization(input_shape=(None, num_channels*30*FreqSample/step))) rcnn.add(layers.Conv1D(32, 3, activation='relu')) rcnn.add(layers.MaxPooling1D(2)) rcnn.add(layers.Conv1D(64, 3, activation='relu')) rcnn.add(layers.LSTM(1000,dropout=0.1, recurrent_dropout=0.1,return_sequences=True)) rcnn.add(layers.LSTM(1000,dropout=0.1, recurrent_dropout=0.1,return_sequences=True)) rcnn.add(layers.LSTM(1000,dropout=0.1, recurrent_dropout=0.1,return_sequences=True)) rcnn.add(layers.LSTM(1000,dropout=0.1, recurrent_dropout=0.1)) rcnn.add(layers.Dense(5, activation="softmax")) rcnn.summary() conv.load_weights(cnn_model) rnn.load_weights(rnn_model) rcnn.load_weights(rcnn_model) sl = SuperLearner([conv,rnn,rcnn],['cnn','rnn','rcnn'], loss='nloglik') # fit the super learner to learn the best coefficient
model.fit(train_dataset, epochs=3)
"""### Passing data to multi-input, multi-output models
In the previous examples, we were considering a model with a single input (a tensor of shape `(764,)`) and a single output (a prediction tensor of shape `(10,)`). But what about models that have multiple inputs or outputs?
Consider the following model, which has an image input of shape `(32, 32, 3)` (that's `(height, width, channels)`) and a timeseries input of shape `(None, 10)` (that's `(timesteps, features)`). Our model will have two outputs computed from the combination of these inputs: a "score" (of shape `(1,)`) and a probability distribution over 5 classes (of shape `(10,)`).
"""
image_input = keras.Input(shape=(32, 32, 3), name='img_input')
timeseries_input = keras.Input(shape=(None, 10), name='ts_input')
x1 = layers.Conv2D(3, 3)(image_input)
x1 = layers.GlobalMaxPooling2D()(x1)
x2 = layers.Conv1D(3, 3)(timeseries_input)
x2 = layers.GlobalMaxPooling1D()(x2)
x = layers.concatenate([x1, x2])
score_output = layers.Dense(1, name='score_output')(x)
class_output = layers.Dense(5, activation='softmax', name='class_output')(x)
model = keras.Model(inputs=[image_input, timeseries_input],
outputs=[score_output, class_output])
"""Let's plot this model, so you can clearly see what we're doing here (note that the shapes shown in the plot are batch shapes, rather than per-sample shapes)."""
keras.utils.plot_model(model, 'multi_input_and_output_model.png', show_shapes=True)
"""At compilation time, we can specify different losses to different ouptuts, by passing the loss functions as a list:"""
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.3)
# Model Building
sequential_cnn = Sequential()
vocab_size = len(tokenizer.word_index) + 1
output_dim = 50
print(vocab_size)
print(X_train.shape[1])
input_length = X_train.shape[1]
sequential_cnn.add(
layers.Embedding(input_dim=vocab_size,
output_dim=output_dim,
input_length=input_length))
sequential_cnn.add(layers.Conv1D(100, 5, activation='relu'))
sequential_cnn.add(layers.GlobalMaxPool1D())
sequential_cnn.add(layers.Dense(10, activation='relu'))
sequential_cnn.add(layers.Dense(6, activation='softmax'))
print(sequential_cnn.summary())
sequential_cnn.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
history_cnn = sequential_cnn.fit(X_train,
Y_train,
epochs=10,
batch_size=32,
validation_data=(X_test, Y_test))
plt_performance(history_cnn)
# accuracy - 1
# visualize error/acc with epochs
train_loss = history.history['loss']
val_loss = history.history['val_loss']
train_acc = history.history['acc']
val_acc = history.history['val_acc']
print(train_loss)
print(val_loss)
print(train_acc)
print(val_acc)
model = Sequential(name='rcnn')
model.add(
layers.BatchNormalization(input_shape=(None, num_channels * 30 *
FreqSample / step)))
model.add(layers.Conv1D(32, 3, activation='relu'))
model.add(layers.MaxPooling1D(2))
model.add(layers.Conv1D(64, 3, activation='relu'))
model.add(
layers.LSTM(1000,
dropout=0.1,
recurrent_dropout=0.1,
return_sequences=True))
model.add(
layers.LSTM(1000,
dropout=0.1,
recurrent_dropout=0.1,
return_sequences=True))
model.add(
layers.LSTM(1000,
dropout=0.1,
def supervised_learning(load):
# Loading dataset
data = []
g = gzip.open("Software.json.gz", 'r')
print("Loading dataset ...")
for l in g:
data.append(json.loads(l))
N = 100000
print("The dataset used has ", len(data), "entries! Of this dataset", N,
"entries are used to train the model.")
reviews = []
ratings = []
print("Text preprocessing ...")
for d in data[:N]:
if d.get('reviewText') is None:
continue
# remove all unwanted chars
review = re.compile(r'[^a-z0-9\s]').sub(
r'',
re.compile(r'[\W]').sub(r' ',
d.get('reviewText').lower()))
reviews.append(review)
rating = float(d.get('overall'))
ratings.append(rating)
# vectorized the input texts
max_features = 200000
tokenizer = Tokenizer(num_words=max_features)
tokenizer.fit_on_texts(reviews)
reviews = tokenizer.texts_to_sequences(reviews)
# calculating the maximal review length & pad all inputs to the same length for the neural network
max_length = max(len(train_r) for train_r in reviews)
reviews = tf.keras.preprocessing.sequence.pad_sequences(reviews,
maxlen=max_length)
# split the data into training set, test set and validation set
print("Splitting dataset ...")
train_reviews, test_reviews, train_ratings, test_ratings = train_test_split(
np.array(reviews), np.array(ratings), test_size=0.1)
train_reviews, validation_reviews, train_ratings, validation_ratings = train_test_split(
train_reviews, train_ratings, test_size=0.2)
# Create the neural network. Input size was calculated above
input = layers.Input(shape=(max_length, ))
x = layers.Embedding(max_features, 64)(input)
# three times, use a convolutional layer, normalization and max pooling layer
x = layers.Conv1D(64, 5, activation='relu')(x)
x = layers.BatchNormalization()(x)
x = layers.MaxPool1D(3)(x)
x = layers.Conv1D(64, 5, activation='relu')(x)
x = layers.BatchNormalization()(x)
x = layers.MaxPool1D(5)(x)
x = layers.Conv1D(64, 5, activation='relu')(x)
x = layers.GlobalMaxPool1D()(x)
x = layers.Flatten()(x)
# two normally connected layers to condense the output to a single number
x = layers.Dense(64, activation='relu')(x)
output = layers.Dense(1, activation=mapping_to_target_range)(x)
model = models.Model(inputs=input, outputs=output)
# Adam (a stochastic gradient descent variant) as optimization function
opt = tf.keras.optimizers.Adam(learning_rate=0.001)
# compiling the model. As error the MSE is specified since the output and target are floats
model.compile(optimizer=opt, loss='mean_squared_error')
# loading model weights if wanted
if load:
print("\nLoading previous model weights:\n")
model.load_weights('weights/supervisedLearning')
# training the model
print("Training Model:\n")
model.fit(train_reviews,
train_ratings,
batch_size=64,
epochs=3,
validation_data=(validation_reviews, validation_ratings))
# calculating the predictions on the test set
test_pred = model.predict(test_reviews)
# printing the classification report
print(classification_report(test_ratings, np.round(test_pred)))
# testing the model with 3 examples (positive, negative, neutral review)
print("\nTesting model: ")
x = "I really like this book. It is one of the best I have read."
x = re.compile(r'[^a-z\s]').sub(r'',
re.compile(r'[\W]').sub(r' ', x.lower()))
x = tokenizer.texts_to_sequences(np.array([x]))
x = tf.keras.preprocessing.sequence.pad_sequences(x, maxlen=max_length)
result = model.predict(x)
print(
"'I really like this book. It is one of the best I have read.' got the rating: ",
round(result[0][0]))
x = "I really hate this book. It is one of the worst I have read."
x = re.compile(r'[^a-z\s]').sub(r'',
re.compile(r'[\W]').sub(r' ', x.lower()))
x = tokenizer.texts_to_sequences(np.array([x]))
x = tf.keras.preprocessing.sequence.pad_sequences(x, maxlen=max_length)
result = model.predict(x)
print(
"'I really hate this book. It is one of the worst I have read.' got the rating: ",
round(result[0][0]))
x = "This book is ok. It is very average."
x = re.compile(r'[^a-z\s]').sub(r'',
re.compile(r'[\W]').sub(r' ', x.lower()))
x = tokenizer.texts_to_sequences(np.array([x]))
x = tf.keras.preprocessing.sequence.pad_sequences(x, maxlen=max_length)
result = model.predict(x)
print("'This book is ok. It is very average.' got the rating: ",
round(result[0][0]))
# saving the model weights
print("\n\nSaving model weights ...")
model.save_weights('weights/supervisedLearning')
def __init__(self,
stage,
block,
filters,
kernel_size,
dilation,
padding,
pre_act=False,
**kwargs):
super(ResidualBlock,
self).__init__(name='residual_block_' + str(stage), **kwargs)
assert padding in ['causal', 'same']
filters1, filters2, filters3 = filters
self.name_base = name = 'residual_block_' + str(
stage) + '_' + block + str(pre_act) + '_branch'
conv_name_base = 'res' + str(stage) + block + str(pre_act) + '_branch'
bn_name_base = 'bn' + str(stage) + block + str(pre_act) + '_branch'
self.pre_act = pre_act
# block1
self.conv1 = layers.Conv1D(
filters=filters1,
kernel_size=kernel_size,
dilation_rate=dilation,
padding=padding,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2a')
self.batch1 = layers.BatchNormalization(axis=-1,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2a')
self.ac1 = layers.Activation('relu')
self.drop_1 = layers.SpatialDropout1D(rate=0.5)
# block2
self.conv2 = layers.Conv1D(
filters=filters2,
kernel_size=kernel_size,
dilation_rate=dilation,
padding=padding,
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2b')
self.batch2 = layers.BatchNormalization(axis=-1,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2b')
self.ac2 = layers.Activation('relu')
self.drop_2 = layers.SpatialDropout1D(rate=0.5)
# 为了防止维度不一致使用 1*1 卷积在channel处进行匹配
self.downsample = layers.Conv1D(
filters=filters3,
kernel_size=1,
padding='same',
kernel_initializer='he_normal',
kernel_regularizer=regularizers.l2(L2_WEIGHT_DECAY),
name=conv_name_base + '2c')
self.batch3 = layers.BatchNormalization(axis=-1,
momentum=BATCH_NORM_DECAY,
epsilon=BATCH_NORM_EPSILON,
name=bn_name_base + '2c')
self.ac3 = layers.Activation('relu')
def _train(mutated, module_name):
mutated = mutated[mutated['mod_keys_found_string'] == module_name]
train_set, val_set, test_set = np.split(
mutated.sample(frac=1),
[int(.6 * len(mutated)),
int(.8 * len(mutated))])
tasks_sent_train = [row for row in train_set['task_complete']]
model_tasks3 = Word2Vec(tasks_sent_train,
sg=0,
size=100,
window=6,
min_count=1,
workers=4,
iter=1000)
train_set['task_complete_one_string'] = train_set['task_complete'].apply(
lambda x: list_to_string(x))
test_set['task_complete_one_string'] = test_set['task_complete'].apply(
lambda x: list_to_string(x))
val_set['task_complete_one_string'] = val_set['task_complete'].apply(
lambda x: list_to_string(x))
y_train = train_set['consistent'].astype(int)
print(y_train.value_counts(), y_train.shape)
y_test = test_set['consistent'].astype(int)
print(y_test.value_counts(), y_test.shape)
y_val = val_set['consistent'].astype(int)
tokenizer_train = Tokenizer(lower=False)
tokenizer_train.fit_on_texts(train_set['task_complete'])
print(tokenizer_train)
tokenizer_train = Tokenizer(lower=False)
tokenizer_train.fit_on_texts(train_set['task_complete'])
print(tokenizer_train)
tokenizer_test = Tokenizer(lower=False)
tokenizer_test.fit_on_texts(test_set['task_complete'])
print(tokenizer_test)
tokenizer_val = Tokenizer(lower=False)
tokenizer_val.fit_on_texts(val_set['task_complete'])
tasks_train_tokens = tokenizer_train.texts_to_sequences(
train_set['task_complete_one_string'])
tasks_test_tokens = tokenizer_test.texts_to_sequences(
test_set['task_complete_one_string'])
tasks_val_tokens = tokenizer_val.texts_to_sequences(
val_set['task_complete_one_string'])
num_tokens = [len(tokens) for tokens in tasks_train_tokens]
num_tokens = np.array(num_tokens)
np.max(num_tokens)
np.argmax(num_tokens)
max_tokens = np.mean(num_tokens) + 2 * np.std(num_tokens)
max_tokens = int(max_tokens)
tasks_train_pad = pad_sequences(tasks_train_tokens,
maxlen=max_tokens,
padding='post')
tasks_test_pad = pad_sequences(tasks_test_tokens,
maxlen=max_tokens,
padding='post')
tasks_val_pad = pad_sequences(tasks_val_tokens,
maxlen=max_tokens,
padding='post')
embedding_size = 100
num_words = len(list(tokenizer_train.word_index)) + 1
embedding_matrix = np.random.uniform(-1, 1, (num_words, embedding_size))
for word, i in tokenizer_train.word_index.items():
if i < num_words:
embedding_vector = model_tasks3[word]
if embedding_vector is not None:
embedding_matrix[i] = embedding_vector
sequence_length = max_tokens
batch_size = 256
tensorflow.compat.v1.disable_eager_execution()
# CNN architecture
num_classes = 2
# Training params
num_epochs = 20
# Model parameters
num_filters = 64
weight_decay = 1e-4
print("training CNN ...")
model = Sequential()
# Model add word2vec embedding
model.add(
Embedding(
input_dim=num_words,
output_dim=embedding_size,
weights=[embedding_matrix],
input_length=max_tokens,
trainable=True, # the layer is trained
name='embedding_layer'))
model.add(
layers.Conv1D(filters=num_filters,
kernel_size=max_tokens,
activation='relu',
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.MaxPooling1D(2))
model.add(Dropout(0.25))
model.add(
layers.Conv1D(filters=num_filters + num_filters,
kernel_size=max_tokens,
activation='relu',
padding='same',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.GlobalMaxPooling1D())
model.add(Dropout(0.25))
model.add(layers.Flatten())
model.add(
layers.Dense(128,
activation='relu',
kernel_regularizer=regularizers.l2(weight_decay)))
model.add(layers.Dense(num_classes, activation='softmax'))
sgd = SGD(lr=1e-2, decay=1e-6, momentum=0.9, nesterov=True)
model.compile(loss=tensorflow.keras.losses.MeanAbsoluteError(),
optimizer=sgd,
metrics=['accuracy'])
model.summary()
model.fit(tasks_train_pad,
to_categorical(y_train),
batch_size=batch_size,
epochs=num_epochs,
validation_data=(tasks_test_pad, to_categorical(y_test)),
shuffle=True,
verbose=2)
score = model.evaluate(tasks_val_pad, to_categorical(y_val), verbose=0)
print('loss:', score[0])
print('Validation accuracy:', score[1])
y_pred = model.predict_classes(tasks_val_pad)
cm = confusion_matrix(y_val, y_pred)
tp = cm[1][1]
fp = cm[0][1]
fn = cm[1][0]
tn = cm[0][0]
precision = round(tp / (tp + fp), 2)
print('Consistent: precision=%.3f' % (precision))
recall = round(tp / (tp + fn), 2)
print('Consistent: recall=%.3f' % (recall))
f1_score = (2 * precision * recall) / (precision + recall)
print('Consistent: f1_score=%.3f' % (f1_score))
precision_neg = round(tn / (tn + fn), 2)
print('Inconsistent: precision=%.3f' % (precision_neg))
recall_neg = round(tn / (tn + fp), 2)
print('Inconsistent: recall=%.3f' % (recall_neg))
f1_score_neg = (2 * precision_neg * recall_neg) / (precision_neg +
recall_neg)
print('Inconsistent: f1_score=%.3f' % (f1_score_neg))
ns_probs = [0 for _ in range(len(y_val))]
ns_auc = roc_auc_score(y_val, ns_probs)
lr_auc = roc_auc_score(y_val, y_pred)
mcc = matthews_corrcoef(y_val, y_pred)
print(precision)
print('No Skill: ROC AUC=%.3f' % (ns_auc))
print('Our model: ROC AUC=%.3f' % (lr_auc))
print('Our model: MCC=%.3f' % (mcc))
json_out = {"module": module_name, "MCC": mcc, "AUC": lr_auc}
model.save('models/' + module_name)
return json_out
pad = 'pre'
x_train_pad = pad_sequences(x_train_tokens, maxlen=max_tokens,
padding=pad, truncating=pad)
x_test_pad = pad_sequences(x_test_tokens, maxlen=max_tokens,
padding=pad, truncating=pad)
model = Sequential()
embedding_size = 8
model.add(Embedding(input_dim=num_words,
output_dim=embedding_size,
input_length=max_tokens,
name='layer_embedding'))
#model.add(LSTM(units=128))
model.add(layers.Conv1D(128, 5, activation='relu'))
model.add(layers.MaxPooling1D(5))
model.add(layers.Conv1D(64, 5, activation='relu'))
model.add(layers.MaxPooling1D(5))
model.add(layers.Conv1D(32, 5, activation='relu'))
model.add(layers.MaxPooling1D(5))
model.add(layers.Flatten())
model.add(Dense(3, activation='softmax'))
tensorboard = TensorBoard(log_dir="logs/{}".format(time()))
optimizer = Adam(lr=0.008)
model.compile(loss='sparse_categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.fit(x_train_pad, y_train, validation_split=0.05, epochs=10, batch_size=5000, callbacks=[tensorboard])
self.classifier = layers.Dense(1, activation='sigmoid')
def call(self, inputs):
outputs = []
state = tf.zeros(shape=(inputs.shape[0], self.units))
for t in range(inputs.shape[1]):
x = inputs[:, t, :]
h = self.projection_1(x)
y = h + self.projection_2(state)
state = y
outputs.append(y)
features = tf.stack(outputs, axis=1)
return self.classifier(features)
# Note that we specify a static batch size for the inputs with the `batch_shape`
# arg, because the inner computation of `CustomRNN` requires a static batch size
# (when we create the `state` zeros tensor).
inputs = keras.Input(batch_shape=(batch_size, timesteps, input_dim))
x = layers.Conv1D(32, 3)(inputs)
outputs = CustomRNN()(x)
model = keras.Model(inputs, outputs)
rnn_model = CustomRNN()
_ = rnn_model(tf.zeros((1, 10, 5)))
"""This concludes our guide on the Functional API!
Now you have at your fingertips a powerful set of tools for building deep learning models.
"""
maxlen = 50
X = pad_sequences(X, maxlen=maxlen)
Y = complain_data['Satisfaction'].values
X_train, X_test, Y_train, Y_test = train_test_split(
X, Y, test_size=0.33, random_state=42)
# print(X_train.shape,Y_train.shape)
# print(X_test.shape,Y_test.shape)
embedding_vector_length = 32
model = Sequential()
model.add(layers.Embedding(max_features,
embedding_vector_length, input_length=maxlen))
model.add(layers.Conv1D(filters=32, kernel_size=3,
padding='same', activation='relu'))
model.add(layers.MaxPooling1D(pool_size=2))
model.add(layers.LSTM(100, recurrent_activation='sigmoid'))
model.add(layers.Dense(1, activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam', metrics=['accuracy'])
model.summary()
filename = os.path.join(current_dir, 'data', 'complain_model.h5')
is_training = False
if is_training:
model.fit(X_train, Y_train, validation_data=(
X_test, Y_test), epochs=20, batch_size=64)
# Evaluate the model
scores = model.evaluate(X_test, Y_test, verbose=0)
target = backend.argmax(y_true, axis=-1)
accuracy = backend.equal(
backend.array_ops.boolean_mask(prediction,
backend.not_equal(target, 0)),
backend.array_ops.boolean_mask(target, backend.not_equal(target, 0)))
return backend.mean(accuracy)
concatenate = layers.Concatenate()
flatten = layers.Flatten()
input_1 = layers.Input(shape=(word_length, len(dictionary)))
convolution_1 = layers.Conv1D(32, 2, padding="same", activation="tanh")
convolution_2 = layers.Conv1D(32, 3, padding="same", activation="tanh")
convolution_3 = layers.Conv1D(32, 5, padding="same", activation="tanh")
convolution_4 = layers.Conv1D(32, 7, padding="same", activation="tanh")
convolution_1_output = convolution_1(input_1)
convolution_2_output = convolution_2(input_1)
convolution_3_output = convolution_3(input_1)
convolution_4_output = convolution_4(input_1)
convolution_layer_1_output = concatenate([
convolution_1_output, convolution_2_output, convolution_3_output,
convolution_4_output
])
convolution_5 = layers.Conv1D(32, 2, padding="same", activation="tanh")
# In[106]:
model.fit(X_train,
Y_train,
validation_data=(X_test, Y_test),
callbacks=[es],
epochs=50)
# # CNN
# In[107]:
model = Sequential(name="CNN")
model.add(
layers.Conv1D(filters=32,
kernel_size=4,
input_shape=(X_train.shape[1], 1),
activation="relu"))
model.add(layers.Dropout(0.5))
model.add(layers.Conv1D(filters=64, kernel_size=6, activation="relu"))
model.add(layers.Flatten())
model.add(layers.Dense(64))
model.add(layers.Dense(6, activation="softmax"))
# In[108]:
model.compile(optimizer=optimizer,
loss="categorical_crossentropy",
metrics=["accuracy"])
# In[109]:
std_train = X_train[:, -1, :].std(axis=0)
X_train /= std_train
X_test -= mean_train
X_test /= std_train
# reshape to be suitable for Keras
y_train = utils.to_categorical(y_train)
y_test = utils.to_categorical(y_test)
print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)
model = Sequential()
model.add(
layers.Conv1D(32,
3,
activation='relu',
input_shape=(None, N_channels * 30 * FreqSample / step)))
model.add(layers.MaxPooling1D(2))
model.add(layers.Conv1D(64, 3, activation='relu'))
model.add(
layers.LSTM(1000,
dropout=0.1,
recurrent_dropout=0.1,
return_sequences=True))
model.add(
layers.LSTM(1000,
dropout=0.1,
recurrent_dropout=0.1,
return_sequences=True))
model.add(
layers.LSTM(1000,
def general_input(self):
role1_actions = Input(shape=(self.input_steps, ), name='role1_actions')
role2_actions = Input(shape=(self.input_steps, ), name='role2_actions')
# 鉴于embedding就是onehot+全连接，这里加大embedding的size
role1_actions_embedding = layers.Embedding(
512, 32, name='role1_actions_embedding')(role1_actions)
role2_actions_embedding = layers.Embedding(
512, 32, name='role2_actions_embedding')(role2_actions)
role1_energy = Input(shape=(self.input_steps, ), name='role1_energy')
role1_energy_embedding = layers.Embedding(
5, 4, name='role1_energy_embedding')(role1_energy)
role2_energy = Input(shape=(self.input_steps, ), name='role2_energy')
role2_energy_embedding = layers.Embedding(
5, 4, name='role2_energy_embedding')(role2_energy)
role1_baoqi = Input(shape=(self.input_steps, ), name='role1_baoqi')
role1_baoqi_embedding = layers.Embedding(
2, 8, name='role1_baoqi_embedding')(role1_baoqi)
role2_baoqi = Input(shape=(self.input_steps, ), name='role2_baoqi')
role2_baoqi_embedding = layers.Embedding(
2, 8, name='role2_baoqi_embedding')(role2_baoqi)
role_position = Input(shape=(self.input_steps, 4), name='role_x_y')
# 感觉这种环境每次都不同，小批量数据bn可能不太稳定，需要测试
# 步长1 距离，步长2速度
role_position_1 = layers.Conv1D(filters=32,
kernel_size=1,
strides=1,
padding='same')(role_position)
role_position_2 = layers.Conv1D(filters=32,
kernel_size=2,
strides=1,
padding='same')(role_position)
# role_position_1 = BatchNormalization(name='bn_1')(role_position_1)
# role_position_2 = BatchNormalization(name='bn_2')(role_position_2)
role_position_1 = layers.LeakyReLU(0.05)(role_position_1)
role_position_2 = layers.LeakyReLU(0.05)(role_position_2)
actions_input = Input(shape=(self.action_steps, ), name='last_action')
actions_embedding = layers.Embedding(
self.action_num, 64, name='last_action_embedding')(actions_input)
model_input = [
role1_actions, role2_actions, role1_energy, role2_energy,
role_position, role1_baoqi, role2_baoqi, actions_input
]
encoder_input = [
role1_actions_embedding,
role2_actions_embedding,
# normal_role_position, normal_role_distance, normal_role_abs_distance,
role_position,
role_position_1,
role_position_2,
role1_energy_embedding,
role2_energy_embedding,
role1_baoqi_embedding,
role2_baoqi_embedding,
]
decoder_output = actions_embedding
return model_input, encoder_input, decoder_output
from tensorflow.python.keras import layers from tensorflow.python.keras import models mdl = models.Sequential() mdl.add(layers.Conv1D(32, 5, activation='relu', input_shape=(20, 10))) mdl.add(layers.MaxPooling1D(2)) mdl.add(layers.Conv1D(32, 5, activation='relu')) mdl.summary()
