stripped_html = tf.strings.regex_replace(lowercase, '<br />', ' ')
return tf.strings.regex_replace(stripped_html,
'[%s]' % re.escape(string.punctuation), '')
vectorize_layer = TextVectorization(standardize=custom_standardization,
max_tokens=vocab_size,
output_mode='int',
output_sequence_length=sequence_lenght)
text_ds = train_ds.map(lambda x, y: x)
vectorize_layer.adapt(text_ds)
model = Sequential([
vectorize_layer,
Embedding(vocab_size, Embedding_dim, name="embedding"),
GlobalAveragePooling1D(),
Dense(16, activation='relu'),
Dense(1)
])
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir="logs")
model.compile(optimizer='adam',
loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
metrics=['accuracy'])
model.fit(train_ds,
validation_data=val_ds,
epochs=15,
callbacks=[tensorboard_callback])
from tensorflow.keras.layers import Embedding, Dropout, Conv1D, GlobalAveragePooling1D, Dense
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint
from tensorflow.keras.models import load_model
embedding_size = 256
batch_size = 256
# 모델 설계
"""
1D합성곱 연산을 수행하되, 커널수는 256 , 커널의 크기느 3을 사용. 두개의 밀집층으로 은닉층과 출력층 설계
"""
model = Sequential()
model.add(Embedding(vocab_size, 256))
model.add(Dropout(0.3))
model.add(Conv1D(256, 3, padding='valid', activation='relu'))
model.add(GlobalAveragePooling1D())
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))
# +
es = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=3)
mc = ModelCheckpoint('CNN_MODEL.h5',
monitor='val_acc',
mode='max',
verbose=1,
save_best_only=True)
model.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc'])
history = model.fit(X_train,
y_train,
output_dim=embedding_dim,
input_shape=(input_length, ))(input_layer)
output_layer = SpatialDropout1D(spatial_dropout)(output_layer)
output_layer = Bidirectional(
LSTM(lstm_units,
return_sequences=True,
dropout=lstm_dropout,
recurrent_dropout=recurrent_dropout))(output_layer)
output_layer = Conv1D(filters,
kernel_size=kernel_size,
padding='valid',
kernel_initializer='glorot_uniform')(output_layer)
avg_pool = GlobalAveragePooling1D()(output_layer)
max_pool = GlobalMaxPooling1D()(output_layer)
output_layer = concatenate([avg_pool, max_pool])
output_layer = Dense(num_classes, activation='softmax')(output_layer)
model = Model(input_layer, output_layer)
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
train_sequences = [text.split() for text in train.text]
validation_sequences = [text.split() for text in validation.text]
list_tokenized_train = tokenizer.texts_to_sequences(train_sequences)
def create_model(self,
filters_number,
network_depth=6,
use_residual=True,
use_bottleneck=True,
max_kernel_size=20,
learning_rate=0.01,
regularization_rate=0.0):
"""
Generate a InceptionTime model. See Fawaz et al. 2019.
The compiled Keras model is returned.
Parameters
----------
input_shape : tuple
Shape of the input dataset: (num_samples, num_timesteps, num_channels)
class_number : int
Number of classes for classification task
filters_number : int
number of filters for each convolutional layer
network_depth : int
Depth of network, i.e. number of Inception modules to stack.
use_residual: bool
If =True, then residual connections are used. Default is True.
use_bottleneck: bool
If=True, bottleneck layer is used at the entry of Inception modules.
Default is true.
max_kernel_size: int,
Maximum kernel size for convolutions within Inception module.
learning_rate : float
learning rate
regularization_rate: float
regularization rate
Returns
-------
model : Keras model
The compiled Keras model
"""
dim_length = self.x_shape[1] # number of samples in a time series
dim_channels = self.x_shape[2] # number of channels
weightinit = 'lecun_uniform' # weight initialization
bottleneck_size = 32
def inception_module(input_tensor, stride=1, activation='linear'):
if use_bottleneck and int(input_tensor.shape[-1]) > 1:
input_inception = Conv1D(filters=bottleneck_size,
kernel_size=1,
padding='same',
activation=activation,
kernel_initializer=weightinit,
use_bias=False)(input_tensor)
else:
input_inception = input_tensor
kernel_sizes = [max_kernel_size // (2**i) for i in range(3)]
conv_list = []
for kernel_size in kernel_sizes:
conv_list.append(
Conv1D(filters=filters_number,
kernel_size=kernel_size,
strides=stride,
padding='same',
activation=activation,
kernel_initializer=weightinit,
use_bias=False)(input_inception))
max_pool_1 = MaxPool1D(pool_size=3, strides=stride,
padding='same')(input_tensor)
conv_last = Conv1D(filters=filters_number,
kernel_size=1,
padding='same',
activation=activation,
kernel_initializer=weightinit,
use_bias=False)(max_pool_1)
conv_list.append(conv_last)
x = Concatenate(axis=2)(conv_list)
x = BatchNormalization()(x)
x = Activation(activation='relu')(x)
return x
def shortcut_layer(input_tensor, out_tensor):
shortcut_y = Conv1D(filters=int(out_tensor.shape[-1]),
kernel_size=1,
padding='same',
kernel_initializer=weightinit,
use_bias=False)(input_tensor)
shortcut_y = BatchNormalization()(shortcut_y)
x = Add()([shortcut_y, out_tensor])
x = Activation('relu')(x)
return x
# Build the actual model:
input_layer = Input((dim_length, dim_channels))
x = BatchNormalization()(
input_layer) # Added batchnorm (not in original paper)
input_res = x
for depth in range(network_depth):
x = inception_module(x)
if use_residual and depth % 3 == 2:
x = shortcut_layer(input_res, x)
input_res = x
gap_layer = GlobalAveragePooling1D()(x)
# Final classification layer
output_layer = Dense(self.number_of_classes,
activation='relu')(gap_layer)
# Create model and compile
model = Model(inputs=input_layer, outputs=output_layer)
model.compile(loss='mean_absolute_error',
optimizer=Adam(lr=learning_rate),
metrics=self.metrics)
return model
def __init__(self, num_actions):
super().__init__('mlp_policy')
embeddings = []
embeddings_shape = []
# Generations
embedding_size = POKEMON_EXTRA_SMALL_EMBEDDINGS_DIM
generations_embedding = Embedding(POKEMON_EXTRA_SMALL_VOCAB_SIZE,
embedding_size,
input_length=1)
embeddings.append(generations_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
# Game Types
embedding_size = POKEMON_EXTRA_SMALL_EMBEDDINGS_DIM
gametypes_embedding = Embedding(POKEMON_EXTRA_SMALL_VOCAB_SIZE,
embedding_size,
input_length=1)
embeddings.append(gametypes_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
#Tiers
embedding_size = POKEMON_EXTRA_SMALL_EMBEDDINGS_DIM
tiers_embedding = Embedding(POKEMON_EXTRA_SMALL_VOCAB_SIZE,
embedding_size,
input_length=1)
embeddings.append(tiers_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
# Weather
embedding_size = POKEMON_EXTRA_SMALL_EMBEDDINGS_DIM
weather_embedding = Embedding(POKEMON_EXTRA_SMALL_VOCAB_SIZE,
embedding_size,
input_length=1)
embeddings.append(weather_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
# Terrain
embedding_size = POKEMON_EXTRA_SMALL_EMBEDDINGS_DIM
terrain_embedding = Embedding(POKEMON_EXTRA_SMALL_VOCAB_SIZE,
embedding_size,
input_length=1)
embeddings.append(terrain_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
# Room
embedding_size = POKEMON_EXTRA_SMALL_EMBEDDINGS_DIM
room_embedding = Embedding(POKEMON_EXTRA_SMALL_VOCAB_SIZE,
embedding_size,
input_length=1)
embeddings.append(room_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
# Effective p1 a
embedding_size = POKEMON_EXTRA_SMALL_EMBEDDINGS_DIM
effective_p1_a_embedding = Embedding(POKEMON_EXTRA_SMALL_VOCAB_SIZE,
embedding_size,
input_length=1)
embeddings.append(effective_p1_a_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
# Effective p2 a
embedding_size = POKEMON_EXTRA_SMALL_EMBEDDINGS_DIM
effective_p2_a_embedding = Embedding(POKEMON_EXTRA_SMALL_VOCAB_SIZE,
embedding_size,
input_length=1)
embeddings.append(effective_p2_a_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
# p1 Pending Attacks A
embedding_size = POKEMON_LARGE_EMBEDDINGS_DIM
seen_attacks_a_embedding = Embedding(POKEMON_MAX_VOCAB_SIZE,
embedding_size,
input_length=1)
embeddings.append(seen_attacks_a_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
# p2 Pending Attacks A
embedding_size = POKEMON_LARGE_EMBEDDINGS_DIM
seen_attacks_a_embedding = Embedding(POKEMON_MAX_VOCAB_SIZE,
embedding_size,
input_length=1)
embeddings.append(seen_attacks_a_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
# for each pokemon for player and agent
for i in range(6):
embedding_size = POKEMON_LARGE_EMBEDDINGS_DIM
player_pokemon_name_embedding = Embedding(POKEMON_MAX_VOCAB_SIZE,
embedding_size,
input_length=1)
embeddings.append(player_pokemon_name_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
embedding_size = POKEMON_EXTRA_SMALL_EMBEDDINGS_DIM
player_pokemon_status_embedding = Embedding(
POKEMON_EXTRA_SMALL_VOCAB_SIZE, embedding_size, input_length=1)
embeddings.append(player_pokemon_status_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
embedding_size = POKEMON_SMALL_EMBEDDINGS_DIM
player_pokemon_first_element_embedding = Embedding(
POKEMON_MAX_VOCAB_SIZE, embedding_size, input_length=1)
embeddings.append(player_pokemon_first_element_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
embedding_size = POKEMON_SMALL_EMBEDDINGS_DIM
player_pokemon_second_element_embedding = Embedding(
POKEMON_MAX_VOCAB_SIZE, embedding_size, input_length=1)
embeddings.append(player_pokemon_second_element_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
embedding_size = POKEMON_LARGE_EMBEDDINGS_DIM
player_pokemon_abilities_embedding = Embedding(
POKEMON_MEDIUM_VOCAB_SIZE, embedding_size, input_length=1)
embeddings.append(player_pokemon_abilities_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
embedding_size = POKEMON_LARGE_EMBEDDINGS_DIM
player_pokemon_items_embedding = Embedding(POKEMON_MAX_VOCAB_SIZE,
embedding_size,
input_length=1)
embeddings.append(player_pokemon_items_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
embedding_size = POKEMON_EXTRA_SMALL_EMBEDDINGS_DIM
player_pokemon_genders_embedding = Embedding(
POKEMON_EXTRA_SMALL_VOCAB_SIZE, embedding_size, input_length=1)
embeddings.append(player_pokemon_genders_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
# 4 attack slots
for j in range(4):
embedding_size = POKEMON_LARGE_EMBEDDINGS_DIM
player_attack_slot_1_embedding = Embedding(
POKEMON_MAX_VOCAB_SIZE, embedding_size, input_length=1)
embeddings.append(player_attack_slot_1_embedding)
embeddings_shape.append(
Reshape(target_shape=(embedding_size, )))
embedding_size = POKEMON_SMALL_EMBEDDINGS_DIM
player_attack_slot_1_element_embedding = Embedding(
POKEMON_SMALL_VOCAB_SIZE, embedding_size, input_length=1)
embeddings.append(player_attack_slot_1_element_embedding)
embeddings_shape.append(
Reshape(target_shape=(embedding_size, )))
embedding_size = 1
player_attack_slot_1_category_embedding = Embedding(
POKEMON_EXTRA_SMALL_VOCAB_SIZE,
embedding_size,
input_length=1)
embeddings.append(player_attack_slot_1_category_embedding)
embeddings_shape.append(
Reshape(target_shape=(embedding_size, )))
# for each pokemon for player and agent
for i in range(6):
embedding_size = POKEMON_LARGE_EMBEDDINGS_DIM
agent_pokemon_name_embedding = Embedding(POKEMON_MAX_VOCAB_SIZE,
embedding_size,
input_length=1)
embeddings.append(agent_pokemon_name_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
embedding_size = POKEMON_EXTRA_SMALL_EMBEDDINGS_DIM
agent_pokemon_status_embedding = Embedding(
POKEMON_EXTRA_SMALL_VOCAB_SIZE, embedding_size, input_length=1)
embeddings.append(agent_pokemon_status_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
embedding_size = POKEMON_SMALL_EMBEDDINGS_DIM
agent_pokemon_first_element_embedding = Embedding(
POKEMON_MAX_VOCAB_SIZE, embedding_size, input_length=1)
embeddings.append(agent_pokemon_first_element_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
embedding_size = POKEMON_SMALL_EMBEDDINGS_DIM
agent_pokemon_second_element_embedding = Embedding(
POKEMON_MAX_VOCAB_SIZE, embedding_size, input_length=1)
embeddings.append(agent_pokemon_second_element_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
embedding_size = POKEMON_LARGE_EMBEDDINGS_DIM
agent_pokemon_abilities_embedding = Embedding(
POKEMON_MEDIUM_VOCAB_SIZE, embedding_size, input_length=1)
embeddings.append(agent_pokemon_abilities_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
embedding_size = POKEMON_LARGE_EMBEDDINGS_DIM
agent_pokemon_items_embedding = Embedding(POKEMON_MAX_VOCAB_SIZE,
embedding_size,
input_length=1)
embeddings.append(agent_pokemon_items_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
embedding_size = POKEMON_EXTRA_SMALL_EMBEDDINGS_DIM
agent_pokemon_genders_embedding = Embedding(
POKEMON_EXTRA_SMALL_VOCAB_SIZE, embedding_size, input_length=1)
embeddings.append(agent_pokemon_genders_embedding)
embeddings_shape.append(Reshape(target_shape=(embedding_size, )))
# 4 attack slots
for j in range(4):
embedding_size = POKEMON_LARGE_EMBEDDINGS_DIM
agent_attack_slot_1_embedding = Embedding(
POKEMON_MAX_VOCAB_SIZE, embedding_size, input_length=1)
embeddings.append(agent_attack_slot_1_embedding)
embeddings_shape.append(
Reshape(target_shape=(embedding_size, )))
embedding_size = POKEMON_SMALL_EMBEDDINGS_DIM
agent_attack_slot_1_element_embedding = Embedding(
POKEMON_SMALL_VOCAB_SIZE, embedding_size, input_length=1)
embeddings.append(agent_attack_slot_1_element_embedding)
embeddings_shape.append(
Reshape(target_shape=(embedding_size, )))
embedding_size = 1
agent_attack_slot_1_category_embedding = Embedding(
POKEMON_EXTRA_SMALL_VOCAB_SIZE,
embedding_size,
input_length=1)
embeddings.append(agent_attack_slot_1_category_embedding)
embeddings_shape.append(
Reshape(target_shape=(embedding_size, )))
merged = Concatenate(axis=-1) #(embeddings)
self.conv1_1 = Conv1D(256, 10, activation='relu')
# conv1 = Conv1D(100, 10, activation='relu', batch_input_shape=(None, ob_space.shape[1]))(field_inputs_)
self.conv1_2 = Conv1D(256, 10, activation='relu')
self.max_1 = MaxPooling1D(8)
self.conv1_3 = Conv1D(128, 4, activation='relu')
self.conv1_4 = Conv1D(128, 4, activation='relu')
self.max_2 = MaxPooling1D(8)
self.conv1_5 = Conv1D(256, 10, activation='relu')
self.conv1_6 = Conv1D(256, 10, activation='relu')
self.glob_1 = GlobalAveragePooling1D()
self.drop = Dropout(0.3)
# This returns a tensor
non_category_data_input_keras = tf.keras.layers.Input(
POKEMON_FIELD_REMAINDER, name="non_category_data_input")
categorical_dense = tf.keras.layers.Dense(512,
activation='relu') #(merged)
# categorical_dense = Reshape(target_shape=(512,))(categorical_dense)
non_categorical_dense_1 = tf.keras.layers.Dense(
512, activation='relu') #(non_category_data_input_keras)
non_categorical_dense_2 = tf.keras.layers.Dense(
1024, activation='relu') #(non_category_data_input_keras)
non_categorical_dense_3 = tf.keras.layers.Dense(
512, activation='relu') #(non_category_data_input_keras)
combined_fields = Concatenate(
axis=-1) #([non_categorical_dense, categorical_dense])
self.combined_dense_1 = tf.keras.layers.Dense(256, activation='relu')
self.combined_dense_2 = tf.keras.layers.Dense(512, activation='relu')
self.combined_dense_3 = tf.keras.layers.Dense(256, activation='relu')
self.embeddings = embeddings
self.embeddings_shape = embeddings_shape
self.merged = merged
self.categorical_dense = categorical_dense
self.non_categorical_dense_1 = non_categorical_dense_1
self.non_categorical_dense_2 = non_categorical_dense_2
self.non_categorical_dense_3 = non_categorical_dense_3
self.non_category_data_input_keras = non_category_data_input_keras
self.combined_fields = combined_fields
# Note: no tf.get_variable(), just simple Keras API!
self.hidden1 = kl.Dense(256, activation='relu') #(combined_fields)
self.hidden2 = kl.Dense(128, activation='relu')
self.value = kl.Dense(1, name='value')
# Logits are unnormalized log probabilities.
self.logits = kl.Dense(num_actions, name='policy_logits')
self.dist = ProbabilityDistribution()
def __init__(self, num_classes=2):
super(Multi_Resnet_scalars, self).__init__()
activ = relu
activ_end = 'softmax'
init = 'he_normal'
drop_rate = 0.0
nfeat = 16
dil_rate = 1
self.conv1 = Conv2D(nfeat, (3, 3), kernel_initializer=init,
padding='same', name="conv1",
input_shape = (360, 360, 2))
self.bn1 = BatchNormalization(name="bn1")
self.activ1 = Activation(activ, name="activ1")
self.drop1 = Dropout(drop_rate, name = "drop1")
self.conv2 = Conv2D(nfeat * 2, (3, 3), kernel_initializer=init,
strides = 2, padding='same', name="conv2")
self.bn2 = BatchNormalization(name="bn2")
self.activ2 = Activation(activ, name="activ2")
self. conv3 = Conv2D(nfeat * 2, (3, 3), kernel_initializer=init,
padding='same', name="conv3")
self.bn3 = BatchNormalization(name="bn3")
self.conv_b1 = Conv2D(nfeat*2, (1, 1), kernel_initializer=init,
strides = 2, padding='same', name="conv_b1")
self.skip0 = Add(name="skip0")
self.activ3 = Activation(activ, name = "activ3")
self.drop2 = Dropout(drop_rate, name = "drop2")
self.conv4 = Conv2D(nfeat * 4, (3, 3), kernel_initializer=init,
strides = 2, padding='same', name="conv4")
self.bn4 = BatchNormalization(name="bn4")
self.activ4 = Activation(activ, name="activ4")
self.conv5 = Conv2D(nfeat * 4, (3, 3), kernel_initializer=init,
padding='same', name="conv5")
self.bn5 = BatchNormalization(name="bn5")
self.conv_b2 = Conv2D(nfeat * 4, (1, 1), kernel_initializer=init,
strides = 2, padding='same', name="conv_b2")
self.skip1 = Add(name="skip1")
self.activ5 = Activation(activ, name="activ5")
self.gap = GlobalAveragePooling2D(name = "gap")
self.gap_scalars = GlobalAveragePooling3D(name = "gap_scalars")
self.conc = Concatenate(axis = 0, name = "merge")
self.conc_scalars = Concatenate(axis = 1, name = "conc_scalars")
self.conv_final0 = Conv1D(64, 3, padding = "same",
name = "conv_final0", input_shape = (None, 64))
self.conv_final = Conv1D(64, 3,
padding = "valid", name= "conv_final",
input_shape = (None, 64))
self.gap1d = GlobalAveragePooling1D(name = "gap1d")
self.dense2 = Dense(50, activation=activ, name="dense2")
self.activ7 = Activation(activ, name="activ7")
self.drop3 = Dropout(drop_rate, name="drop3")
self.out = Dense(num_classes, kernel_initializer=init,
activation=activ_end, name="out1")
def _init_model(self):
# According to parameters to check data is '2D' or '3D'
self.dimstr = check_dims(self.input_dim_3, self.is_2D, self.is_3D)
if self.dimstr == '2D':
inputs = Input(shape=(self.input_dim_1, self.input_dim_2),
name='inputs')
elif self.dimstr == '3D':
inputs = Input(shape=(self.input_dim_1, self.input_dim_2,
self.input_dim_3),
name='inputs')
# loop for convolution layers
for i in range(self.n_conv_layers):
if i == 0:
res = self._basic_cnn(inputs, i)
else:
res = self._basic_cnn(res, i)
# whether or not use global average pooling layer
if self.use_global:
if self.dimstr == '2D':
res = GlobalAveragePooling1D(name='global_1')(res)
elif self.dimstr == '3D':
res = GlobalAveragePooling2D(name='global_1')(res)
else: # if not global average pooling or Flatten Conv result
res = Flatten()(res)
# whether or not use Dense layer
if self.use_dnn:
for _ in range(self.n_dnn_layers):
res = Dense(self.dnn_units, name='dense_1')(res)
if self.use_batch:
res = BatchNormalization(name='dense_batch_1')(res)
res = Activation(self.activation)(res)
if self.use_dropout:
res = Dropout(self.drop_ratio, name='dense_drop_1')(res)
# this is method private function to check whether or not loss is not given, then use default loss
def _check_loss(model, loss, metrics, optimizer):
if loss is not None:
model.compile(loss=loss,
metrics=[metrics],
optimizer=optimizer)
return model
if self.n_classes == 2: # this is binary class problem.
out = Dense(self.n_classes, activation='sigmoid')(res)
model = Model(inputs, out)
if self.loss is None:
model.compile(loss='binary_crossentropy',
metrics=[self.metrics],
optimizer=self.optimizer)
else:
_check_loss(model, self.loss, self.metrics, self.optimizer)
elif self.n_classes >= 2: # this is multiclass problem.
out = Dense(self.n_classes, activation='softmax')(res)
model = Model(inputs, out)
if self.loss is None:
model.compile(loss='categorical_crossentropy',
metrics=[self.metrics],
optimizer=self.optimizer)
else:
_check_loss(model, self.loss, self.metrics, self.optimizer)
elif self.n_classes == -1: # this is regression problem
out = Dense(1)(res)
model = Model(inputs, out)
if self.loss is None:
model.compile(loss='mse',
metrics=[self.metrics],
optimizer=self.optimizer)
else:
_check_loss(model, self.loss, self.metrics, self.optimizer)
else:
raise AttributeError(
"Parameter 'n_classes' should be -1, 2 or up 2!")
print('Model structure summary:')
model.summary()
return model
#Padding the sequences. Padding type is "post".
pdf = pd.DataFrame(padded_sequences)
from sklearn.model_selection import train_test_split
train_x, test_x, train_y, test_y = train_test_split(pdf, train_y, test_size = 0.1, random_state = 10, shuffle = True)
#Splitting the dataset into train_data and test_data
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Embedding, GlobalAveragePooling1D
model = Sequential([ #Creating a sequential model that uses embeddings of dimension 16
Embedding(num_words, embedding_dim, input_length = max_length),
GlobalAveragePooling1D(), #GlobalAveragePooling1D() instead of Flatten() as the former results in less trainable parameters
Dense(16, activation = 'relu'),
Dense(1, activation = 'sigmoid')
])
model.compile(loss = 'binary_crossentropy', optimizer = 'adam', metrics = ['accuracy'])
history = model.fit(train_x, train_y, epochs = 50, validation_data = (test_x, test_y), verbose = 1)
test_file = "test_oJQbWVk.csv"
sub_file = "sample_submission_LnhVWA4.csv"
tdf = pd.read_csv(path + test_file).fillna(' ')
sdf = pd.read_csv(path + sub_file).fillna(' ')
tdf.head()
maxlen=max_len, num_words=vocab_size)
x_train = sequence.pad_sequences(x_train, maxlen=max_len)
x_test = sequence.pad_sequences(x_test, maxlen=max_len)
x_train_masks = tf.equal(x_train, 0)
x_test_masks = tf.equal(x_test, 0)
y_train = to_categorical(y_train)
y_test = to_categorical(y_test)
print('Model building ... ')
inputs = Input(shape=(max_len,), name="inputs")
masks = Input(shape=(max_len,), name='masks')
embeddings = Embedding(vocab_size, model_dim)(inputs)
encodings = PositionEncoding(model_dim)(embeddings)
encodings = Add()([embeddings, encodings])
x = MultiHeadAttention(8, 64)([encodings, encodings, encodings, masks])
x = GlobalAveragePooling1D()(x)
x = Dropout(0.2)(x)
x = Dense(10, activation='relu')(x)
outputs = Dense(2, activation='softmax')(x)
model = Model(inputs=[inputs, masks], outputs=outputs)
model.compile(optimizer=Adam(beta_1=0.9, beta_2=0.98, epsilon=1e-9),
loss='categorical_crossentropy', metrics=['accuracy'])
print("Model Training ... ")
es = EarlyStopping(patience=5)
model.fit([x_train, x_train_masks], y_train,
batch_size=batch_size, epochs=epochs, validation_split=0.2, callbacks=[es])
test_metrics = model.evaluate(
[x_test, x_test_masks], y_test, batch_size=batch_size, verbose=0)
def _init_model(self):
# Check given parameters to judge data is '2D' or '3D'
self.dimstr = check_dims(self.input_dim_3, self.is_2D, self.is_3D)
if self.dimstr == '2D':
inputs = Input(shape=(self.input_dim_1, self.input_dim_2))
elif self.dimstr == '3D':
inputs = Input(shape=(self.input_dim_1, self.input_dim_2,
self.input_dim_3))
# Whether first use Convolutional layer, can be choosen.
if self.first_conv:
for _ in range(self.n_first_convs):
if self.dimstr == '2D':
res_conv = Conv1D(self.conv_units,
self.kernel_size,
self.strides,
padding=self.padding)(inputs)
elif self.dimstr == '3D':
res_conv = Conv2D(self.conv_units,
self.kernel_size,
self.strides,
padding=self.padding)(inputs)
else:
res_conv = inputs
# Here is Wide & Deep model Block
for i in range(self.n_wide_layers):
if i == 0:
res = self._wide_deep_block(res_conv)
else:
res = self._wide_deep_block(res)
# Whether to use global avarega pooling or just Flatten concolutional result
if self.use_global:
if self.dimstr == '2D':
res = GlobalAveragePooling1D()(res)
elif self.dimstr == '3D':
res = GlobalAveragePooling2D()(res)
else:
res = Flatten()(res)
# Whether to use Dense layers
if self.use_dnn:
for _ in range(self.n_dnn_layers):
res = Dense(self.dnn_units)(res)
if self.use_batch:
res = BatchNormalization()(res)
res = Activation(self.activation)(res)
if self.use_dropout:
res = Dropout(self.drop_ratio)(res)
# this is method private function to check whether or not loss is not given, then use default loss
def _check_loss(model, loss, metrics, optimizer):
if loss is not None:
model.compile(loss=loss,
metrics=[metrics],
optimizer=optimizer)
return model
if self.n_classes == 2: # this is binary class problem.
out = Dense(self.n_classes, activation='sigmoid')(res)
model = Model(inputs, out)
if self.loss is None:
model.compile(loss='binary_crossentropy',
metrics=[self.metrics],
optimizer=self.optimizer)
else:
_check_loss(model, self.loss, self.metrics, self.optimizer)
elif self.n_classes >= 2: # this is multiclass problem.
out = Dense(self.n_classes, activation='softmax')(res)
model = Model(inputs, out)
if self.loss is None:
model.compile(loss='categorical_crossentropy',
metrics=[self.metrics],
optimizer=self.optimizer)
else:
_check_loss(model, self.loss, self.metrics, self.optimizer)
elif self.n_classes == -1: # this is regression problem
out = Dense(1)(res)
model = Model(inputs, out)
if self.loss is None:
model.compile(loss='mse',
metrics=[self.metrics],
optimizer=self.optimizer)
else:
_check_loss(model, self.loss, self.metrics, self.optimizer)
else:
raise AttributeError(
"Parameter 'n_classes' should be -1, 2 or up 2!")
print('Model structure summary:')
model.summary()
return model
tokenizer.fit_on_texts(''.join(x_train))
x_train = tokenizer.texts_to_sequences(x_train)
x_train = keras.preprocessing.sequence.pad_sequences(x_train, maxlen=MAX_LEN)
y_train = a_list
y_train = tokenizer.texts_to_sequences(y_train)
y_train = keras.preprocessing.sequence.pad_sequences(y_train, maxlen=MAX_LEN)
q_inputs = Input(shape=(None, ), dtype='int32')
a_inputs = Input(shape=(None, ), dtype='int32')
embeddings = Embedding(NUM_WORDS, 128)(q_inputs)
embeddings = Embedding(NUM_WORDS, 128)(a_inputs)
O_seq = Attention(16, 16)([embeddings, embeddings, embeddings])
O_seq = GlobalAveragePooling1D()(O_seq)
O_seq = Dropout(0.5)(O_seq)
outputs = Dense(1, activation='sigmoid')(O_seq)
model = Model(inputs=[q_inputs, a_inputs], outputs=outputs)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
print('Train...')
hist = model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=5,
