def model(reader):
known_in = L.Input(shape=(None, ), dtype='int32')
unknown_in = L.Input(shape=(None, ), dtype='int32')
embedding = L.Embedding(len(reader.vocabulary_above_cutoff) + 2, 5)
known_emb = embedding(known_in)
unknown_emb = embedding(unknown_in)
if 'device:GPU' in str('str(device_lib.list_local_devices())'):
gru = L.CuDNNGRU(200)
else:
gru = L.GRU(200)
features_known = gru(known_emb)
features_unknown = gru(unknown_emb)
abs_diff = L.merge(inputs=[features_known, features_unknown],
mode=lambda x: abs(x[0] - x[1]),
output_shape=lambda x: x[0],
name='absolute_difference')
dense1 = L.Dense(500, activation='relu')(abs_diff)
pruned = L.Dropout(0.3)(dense1)
output = L.Dense(2, activation='softmax', name='output')(pruned)
model = Model(inputs=[known_in, unknown_in], outputs=output)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def model(reader):
if not reader.binary:
raise AttributeError('This network only works in binary mode.')
known_in = L.Input(shape=(None, ), name='Known_Input')
unknown_in = L.Input(shape=(None, ), name='Unknown_Input')
embedding = L.Embedding(len(reader.vocabulary_above_cutoff) + 2, 5)
known_emb = embedding(known_in)
unknown_emb = embedding(unknown_in)
if 'device:GPU' in str('str(device_lib.list_local_devices())'):
gru = L.CuDNNGRU(200)
else:
gru = L.GRU(200)
gru_known = gru(known_emb)
gru_unknown = gru(unknown_emb)
cos_distance = L.merge([gru_known, gru_unknown], mode='cos', dot_axes=1)
cos_distance = L.Reshape((1, ))(cos_distance)
cos_similarity = L.Lambda(lambda x: 1 - x)(cos_distance)
model = Model(inputs=[known_in, unknown_in], outputs=cos_similarity)
model.compile(
optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])
return model
def build_keras_model_mv03r00(emb_dims, vocab_size, max_len, emb_matrix):
ip = kl.Input(shape=(max_len, ))
x = kl.Embedding(vocab_size,
emb_dims,
weights=[emb_matrix],
trainable=True,
name='X_emb')(ip)
x = kl.SpatialDropout1D(0.5)(x)
x = kl.Bidirectional(kl.CuDNNGRU(200, return_sequences=True))(x)
x = kl.Bidirectional(kl.CuDNNGRU(200, return_sequences=True))(x)
x = kl.GlobalMaxPool1D()(x)
x = kl.Dense(100, activation="relu")(x)
x = kl.Dropout(0.5)(x)
op = kl.Dense(6, activation="sigmoid")(x)
model = km.Model(inputs=[ip], outputs=op)
return model
def build_keras_model(comprehensive, emb_dims, vocab_size, max_len,
emb_matrix):
ip = kl.Input(shape=(max_len, ))
x = kl.Embedding(input_dim=vocab_size,
output_dim=emb_dims,
weights=[emb_matrix],
trainable=True,
name='X_emb')(ip)
# x = kl.Activation('tanh')(x)
x = kl.SpatialDropout1D(0.4)(x)
x1 = kl.Bidirectional(kl.CuDNNGRU(200, return_sequences=True))(x)
x2 = kl.Bidirectional(kl.CuDNNGRU(200, return_sequences=True))(x1)
x = kl.concatenate([x1, x2, x])
x = AttentionWeightedAverage(name='attlayer', return_attention=False)(x)
x = kl.Dropout(0.4)(x)
op = kl.Dense(6 if comprehensive else 1, activation="sigmoid")(x)
model = km.Model(inputs=[ip], outputs=op)
return model
def RNN(input_shape, test_type, num_classes):
inputs = layers.Input(shape=input_shape)
x = layers.CuDNNGRU(units=256, return_sequences=True, name='rnn_layer_1')(inputs)
x = layers.CuDNNGRU(units=256, return_sequences=True, name='rnn_layer_2')(x)
x = layers.CuDNNGRU(units=256, return_sequences=False, name='rnn_out')(x)
x = layers.Dense(num_classes, name='logits')(x)
if test_type == 'sgc':
output_activation = 'softmax'
elif test_type == 'mgc':
output_activation = 'sigmoid'
elif test_type in ['cos', 'mse']:
output_activation = 'linear'
pred = layers.Activation(output_activation, name=output_activation)(x)
return Model(inputs=inputs, outputs=pred)
def GRU3_solo(rd_input,
kernels,
conv_window_len,
maxpooling_len,
stride,
BN=True,
DropoutRate=0.2):
initializer = 'glorot_uniform'
conv1 = layers.Conv1D(kernels[0], 1, strides= stride , padding='same', \
kernel_initializer=initializer)(rd_input)
output = layers.Bidirectional(layers.CuDNNGRU(
32, return_sequences=True))(conv1)
output = layers.Bidirectional(layers.CuDNNGRU(
32, return_sequences=True))(output)
output = layers.Bidirectional(layers.CuDNNGRU(
32, return_sequences=True))(output)
output = layers.TimeDistributed(layers.Dense(8,
activation="softmax"))(output)
model = models.Model(rd_input, output)
return model
def UNet_GRU3(rd_input,
kernels,
conv_window_len,
maxpooling_len,
stride,
BN=True,
DropoutRate=0.2):
unet_module_output = UNet_module(rd_input, kernels, conv_window_len,
maxpooling_len, stride, BN, DropoutRate)
output = layers.Bidirectional(layers.CuDNNGRU(
32, return_sequences=True))(unet_module_output)
output = layers.Bidirectional(layers.CuDNNGRU(
32, return_sequences=True))(output)
output = layers.Bidirectional(layers.CuDNNGRU(
32, return_sequences=True))(output)
output = layers.TimeDistributed(layers.Dense(8,
activation="softmax"))(output)
model = models.Model(rd_input, output)
return model
def create_LSTM(units, return_sequences, name):
if MerLConfig.GPU:
return kl.CuDNNGRU(units=units, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal',
bias_initializer='zeros', kernel_regularizer=None, recurrent_regularizer=None,
bias_regularizer=None, activity_regularizer=None, kernel_constraint=None,
recurrent_constraint=None, bias_constraint=None, return_sequences=return_sequences,
return_state=False, stateful=False, name=name)
else:
return kl.GRU(units=units,
activation='tanh', recurrent_activation='hard_sigmoid',
use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal',
bias_initializer='zeros', kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None,
activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None,
dropout=0.0, recurrent_dropout=0.0, implementation=1, return_sequences=False, return_state=False,
go_backwards=False, stateful=False, unroll=False, name=name)
def make_model(embedding_matrix):
inps = nn.Input(shape=(MAXLEN, ))
x = nn.Embedding(*embedding_matrix.shape,
weights=[embedding_matrix],
trainable=False)(inps)
x = nn.SpatialDropout1D(0.1)(x)
x = nn.Bidirectional(nn.CuDNNGRU(256, return_sequences=True))(x)
gap = nn.GlobalAveragePooling1D()(x)
gmp = nn.GlobalMaxPooling1D()(x)
x = nn.concatenate([gap, gmp])
x = nn.Dropout(0.5)(x)
x = nn.BatchNormalization()(x)
x = nn.Dense(5, activation='sigmoid')(x)
model = Model(inputs=inps, outputs=x)
model.compile(loss='binary_crossentropy', optimizer=Adamax(decay=1e-6))
return model
vel_pred = y_pred[:, :, 128 + 2]
vel_true = y_true[:, :, 128 + 2]
time_pred = y_pred[:, :, -1]
time_true = y_true[:, :, -1]
loss_notes = kls.categorical_crossentropy(notes_true, notes_pred) * 1
loss_vel = K.abs(vel_true - vel_pred) * 1
loss_time = ((time_true - time_pred)**2) * 0.01
return loss_notes + loss_vel + loss_time
li = kl.Input(shape=(None, 128 + 2 + 1 + 1))
l = li
for i in range(3):
l = kl.CuDNNGRU(512, return_sequences=True)(l)
l = kl.Dense(512, activation='elu')(l)
ln = kl.Dense(128 + 2, activation='softmax')(l)
lv = kl.Dense(1, activation='sigmoid')(l)
lt = kl.Dense(1, activation='relu', use_bias=False)(l)
l = kl.Concatenate()([ln, lv, lt])
model = Model(li, l)
model.compile("adamax", myloss)
def generator(batch_size, seq_length):
batch = []
while True:
mid_index = np.random.choice(len(padded_mids))
cmid = padded_mids[mid_index]
if (len(cmid) - seq_length - 1) <= 0:
plot_train_validation_loss(
history,
'Conv1D, Dropout, BatchNormalization, CuDNNLSTM, Kernel Reguliarzer, Recurrent Regularizer'
)
# ## CuDNNGRU
# ### 학습하기(CuDNNGRU)
# In[29]:
start_time = time.time()
model = Sequential()
model.add(layers.CuDNNGRU(32, input_shape=(None, weather_df_value.shape[-1])))
model.add(layers.Dense(1))
model.compile(optimizer=RMSprop(), loss='mae')
history = model.fit_generator(train_gen,
steps_per_epoch=400,
epochs=20,
validation_data=val_gen,
validation_steps=val_steps)
print("--- %s seconds ---" % (time.time() - start_time))
# ### Train Loss, Validation Loss 그래프
# In[ ]:
def URNet(rd_input,
kernels,
conv_window_len,
maxpooling_len,
stride,
BN=True,
DropoutRate=0.2):
initializer = 'glorot_uniform' #
#stride=1
#kernels =[64, 128, 256, 512]
#kernels =[8, 16, 32, 64]
##################### Conv1 #########################
conv1 = layers.Conv1D(kernels[0], conv_window_len, strides= stride , padding='same', \
kernel_initializer=initializer)(rd_input)
if BN: conv1 = layers.BatchNormalization()(conv1)
conv1 = layers.Activation('relu')(conv1)
conv1 = layers.Conv1D(kernels[0], conv_window_len, strides= stride, padding='same', \
kernel_initializer=initializer)(conv1)
if BN: conv1 = layers.BatchNormalization()(conv1)
conv1 = layers.Activation('relu')(conv1)
rnn1 = layers.CuDNNGRU(kernels[0], return_sequences=True)(conv1)
# use it for the next stage
pool1 = layers.MaxPooling1D(maxpooling_len[0])(rnn1)
##################### Conv2 ##########################
conv2 = layers.Conv1D(kernels[1], 3, strides= stride, padding='same',\
kernel_initializer=initializer)(pool1)
if BN: conv2 = layers.BatchNormalization()(conv2)
conv2 = layers.Activation('relu')(conv2)
conv2 = layers.Conv1D(kernels[1], 3, strides= stride, padding='same',\
kernel_initializer=initializer)(conv2)
if BN: conv2 = layers.BatchNormalization()(conv2)
conv2 = layers.Activation('relu')(conv2)
rnn2 = layers.CuDNNGRU(kernels[1], return_sequences=True)(conv2)
#use it for the next stage
pool2 = layers.MaxPooling1D(maxpooling_len[1])(rnn2)
##################### conv3 ###########################
conv3 = layers.Conv1D(kernels[2], 3, strides= stride, padding='same',\
kernel_initializer=initializer)(pool2)
if BN: conv3 = layers.BatchNormalization()(conv3)
conv3 = layers.Activation('relu')(conv3)
conv3 = layers.Conv1D(kernels[2], 3, strides= stride, padding='same',\
kernel_initializer=initializer)(conv3)
if BN: conv3 = layers.BatchNormalization()(conv3)
conv3 = layers.Activation('relu')(conv3)
if DropoutRate > 0:
drop3 = layers.Dropout(DropoutRate)(conv3)
else:
drop3 = conv3
rnn3 = layers.CuDNNGRU(kernels[2], return_sequences=True)(drop3)
# use it for the next stage
pool3 = layers.MaxPooling1D(maxpooling_len[2])(rnn3)
#################### conv4 (U bottle) #####################
conv4 = layers.Conv1D(kernels[3], 3, strides= 1, padding='same',\
kernel_initializer=initializer)(pool3)
if BN: conv4 = layers.BatchNormalization()(conv4)
conv4 = layers.Activation('relu')(conv4)
conv4 = layers.Conv1D(kernels[3], 3, strides= 1, padding='same',\
kernel_initializer=initializer)(conv4)
if BN: conv4 = layers.BatchNormalization()(conv4)
conv4 = layers.Activation('relu')(conv4)
if DropoutRate > 0:
drop4 = layers.Dropout(DropoutRate)(conv4)
else:
drop4 = conv4
################### upSampling, upConv5 ##########################
# up5 = layers.UpSampling1D(maxpooling_len[2])(drop4)
up5 = Conv1DTranspose(drop4,
kernels[2],
3,
strides=maxpooling_len[2],
padding='same')
merge5 = layers.Concatenate(-1)([drop3, up5])
merge5 = layers.Concatenate(-1)([merge5, rnn3])
conv5 = layers.Conv1D(kernels[2], 3, padding='same', \
kernel_initializer=initializer)(merge5)
if BN: conv5 = layers.BatchNormalization()(conv5)
conv5 = layers.Activation('relu')(conv5)
conv5 = layers.Conv1D(kernels[2], 3, padding='same', \
kernel_initializer=initializer)(conv5)
if BN: conv5 = layers.BatchNormalization()(conv5)
conv5 = layers.Activation('relu')(conv5)
################### upConv 6 ##############################
#up6 = layers.UpSampling1D(maxpooling_len[1])(conv5)
up6 = Conv1DTranspose(conv5,
kernels[1],
3,
strides=maxpooling_len[1],
padding='same')
merge6 = layers.Concatenate(-1)([conv2, up6])
merge6 = layers.Concatenate(-1)([merge6, rnn2])
conv6 = layers.Conv1D(kernels[1], 3, padding='same', \
kernel_initializer=initializer)(merge6)
if BN: conv6 = layers.BatchNormalization()(conv6)
conv6 = layers.Activation('relu')(conv6)
conv6 = layers.Conv1D(kernels[1], 3, padding='same',\
kernel_initializer=initializer)(conv6)
if BN: conv6 = layers.BatchNormalization()(conv6)
conv6 = layers.Activation('relu')(conv6)
################### upConv 7 #########################
#up7 = layers.UpSampling1D(maxpooling_len[0])(conv6)
up7 = Conv1DTranspose(conv6,
kernels[0],
3,
strides=maxpooling_len[0],
padding='same')
merge7 = layers.Concatenate(-1)([conv1, up7])
merge7 = layers.Concatenate(-1)([merge7, rnn1])
conv7 = layers.Conv1D(kernels[0], 3, padding='same',\
kernel_initializer=initializer)(merge7)
if BN: conv7 = layers.BatchNormalization()(conv7)
conv7 = layers.Activation('relu')(conv7)
conv7 = layers.Conv1D(kernels[0], 3, padding='same', \
kernel_initializer=initializer)(conv7)
if BN: conv7 = layers.BatchNormalization()(conv7)
conv7 = layers.Activation('relu')(conv7)
################## final output ######################
conv8 = layers.Conv1D(kernels[0], 3, padding='same', \
kernel_initializer=initializer)(conv7)
if BN: conv8 = layers.BatchNormalization()(conv8)
conv8 = layers.Activation('relu')(conv8)
if DropoutRate > 0:
conv8 = layers.Dropout(DropoutRate)(conv8)
output = conv8
output = layers.Bidirectional(layers.CuDNNGRU(
64, return_sequences=True))(conv8)
output = layers.Bidirectional(layers.CuDNNGRU(
64, return_sequences=True))(output)
output = layers.Bidirectional(layers.CuDNNGRU(
64, return_sequences=True))(output)
output = layers.TimeDistributed(layers.Dense(8,
activation="softmax"))(output)
#conv9 = layers.Conv1D(8, 1, activation='softmax')(conv8)
model = models.Model(rd_input, output)
return model
# User options
args.add_argument('--output', type=int, default=1)
args.add_argument('--epochs', type=int, default=100)
args.add_argument('--batch', type=int, default=128)
args.add_argument('--strmaxlen', type=int, default=150)
args.add_argument('--embedding', type=int, default=256)
args.add_argument('--threshold', type=float, default=0.5)
config = args.parse_args()
#------------------------------------------------------------------
core_inputs = layers.Input((config.strmaxlen, ))
core_layer = layers.Embedding(252,
config.embedding,
input_length=config.strmaxlen)(core_inputs)
core_layer = layers.Bidirectional(
layers.CuDNNGRU(256, return_sequences=True))(core_layer)
core_layer = layers.Bidirectional(
layers.CuDNNGRU(256, return_sequences=False))(core_layer)
core_model = models.Model(inputs=core_inputs, outputs=core_layer)
inputs1 = layers.Input((config.strmaxlen, ))
inputs2 = layers.Input((config.strmaxlen, ))
layer1 = core_model(inputs1)
layer2 = core_model(inputs2)
main_layer = layers.Subtract()([layer1, layer2])
main_layer = layers.Lambda(lambda layer: tf.norm(
layer, ord=2, axis=-1, keep_dims=True))(main_layer)
main_layer = layers.Dense(1)(main_layer)
main_layer = layers.Activation('sigmoid')(main_layer)
main_model = models.Model(inputs=[inputs1, inputs2], outputs=main_layer)
# DONOTCHANGE: They are reserved for nsml
args.add_argument('--mode', type=str, default='train')
args.add_argument('--pause', type=int, default=0)
args.add_argument('--iteration', type=str, default='0')
# User options
args.add_argument('--output', type=int, default=1)
args.add_argument('--epochs', type=int, default=100)
args.add_argument('--batch', type=int, default=512)
args.add_argument('--strmaxlen', type=int, default=150)
args.add_argument('--embedding', type=int, default=256)
config = args.parse_args()
inputs = layers.Input((config.strmaxlen,))
layer = layers.Embedding(251, config.embedding, input_length=config.strmaxlen)(inputs)
layer = layers.Bidirectional(layers.CuDNNGRU(512, return_sequences=True))(layer)
layer = layers.Bidirectional(layers.CuDNNGRU(512, return_sequences=False))(layer)
layer1 = layers.Dense(3)(layer)
outputs1 = layers.Activation('softmax')(layer1)
layer2 = layers.Dense(1)(layer1)
outputs2 = layers.Activation('sigmoid')(layer2)
outputs2 = layers.Lambda(lambda layer: layer * 9 + 1)(outputs2)
model = models.Model(inputs=inputs, outputs=[outputs1, outputs2])
model.summary()
model.compile(optimizer=optimizers.Adam(lr=0.001, amsgrad=True, clipvalue=1.0), loss=['categorical_crossentropy', 'mse'], metrics=['accuracy'])
# DONOTCHANGE: Reserved for nsml use
bind_model(model)