def model(reader): known_in = L.Input(shape=(None, ), dtype='int32') unknown_in = L.Input(shape=(None, ), dtype='int32') embedding = L.Embedding(len(reader.vocabulary_above_cutoff) + 2, 5) known_emb = embedding(known_in) unknown_emb = embedding(unknown_in) if 'device:GPU' in str('str(device_lib.list_local_devices())'): gru = L.CuDNNGRU(200) else: gru = L.GRU(200) features_known = gru(known_emb) features_unknown = gru(unknown_emb) abs_diff = L.merge(inputs=[features_known, features_unknown], mode=lambda x: abs(x[0] - x[1]), output_shape=lambda x: x[0], name='absolute_difference') dense1 = L.Dense(500, activation='relu')(abs_diff) pruned = L.Dropout(0.3)(dense1) output = L.Dense(2, activation='softmax', name='output')(pruned) model = Model(inputs=[known_in, unknown_in], outputs=output) model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) return model
def model(reader): if not reader.binary: raise AttributeError('This network only works in binary mode.') known_in = L.Input(shape=(None, ), name='Known_Input') unknown_in = L.Input(shape=(None, ), name='Unknown_Input') embedding = L.Embedding(len(reader.vocabulary_above_cutoff) + 2, 5) known_emb = embedding(known_in) unknown_emb = embedding(unknown_in) if 'device:GPU' in str('str(device_lib.list_local_devices())'): gru = L.CuDNNGRU(200) else: gru = L.GRU(200) gru_known = gru(known_emb) gru_unknown = gru(unknown_emb) cos_distance = L.merge([gru_known, gru_unknown], mode='cos', dot_axes=1) cos_distance = L.Reshape((1, ))(cos_distance) cos_similarity = L.Lambda(lambda x: 1 - x)(cos_distance) model = Model(inputs=[known_in, unknown_in], outputs=cos_similarity) model.compile( optimizer='adam', loss='binary_crossentropy', metrics=['accuracy']) return model
def build_keras_model_mv03r00(emb_dims, vocab_size, max_len, emb_matrix): ip = kl.Input(shape=(max_len, )) x = kl.Embedding(vocab_size, emb_dims, weights=[emb_matrix], trainable=True, name='X_emb')(ip) x = kl.SpatialDropout1D(0.5)(x) x = kl.Bidirectional(kl.CuDNNGRU(200, return_sequences=True))(x) x = kl.Bidirectional(kl.CuDNNGRU(200, return_sequences=True))(x) x = kl.GlobalMaxPool1D()(x) x = kl.Dense(100, activation="relu")(x) x = kl.Dropout(0.5)(x) op = kl.Dense(6, activation="sigmoid")(x) model = km.Model(inputs=[ip], outputs=op) return model
def build_keras_model(comprehensive, emb_dims, vocab_size, max_len, emb_matrix): ip = kl.Input(shape=(max_len, )) x = kl.Embedding(input_dim=vocab_size, output_dim=emb_dims, weights=[emb_matrix], trainable=True, name='X_emb')(ip) # x = kl.Activation('tanh')(x) x = kl.SpatialDropout1D(0.4)(x) x1 = kl.Bidirectional(kl.CuDNNGRU(200, return_sequences=True))(x) x2 = kl.Bidirectional(kl.CuDNNGRU(200, return_sequences=True))(x1) x = kl.concatenate([x1, x2, x]) x = AttentionWeightedAverage(name='attlayer', return_attention=False)(x) x = kl.Dropout(0.4)(x) op = kl.Dense(6 if comprehensive else 1, activation="sigmoid")(x) model = km.Model(inputs=[ip], outputs=op) return model
def RNN(input_shape, test_type, num_classes): inputs = layers.Input(shape=input_shape) x = layers.CuDNNGRU(units=256, return_sequences=True, name='rnn_layer_1')(inputs) x = layers.CuDNNGRU(units=256, return_sequences=True, name='rnn_layer_2')(x) x = layers.CuDNNGRU(units=256, return_sequences=False, name='rnn_out')(x) x = layers.Dense(num_classes, name='logits')(x) if test_type == 'sgc': output_activation = 'softmax' elif test_type == 'mgc': output_activation = 'sigmoid' elif test_type in ['cos', 'mse']: output_activation = 'linear' pred = layers.Activation(output_activation, name=output_activation)(x) return Model(inputs=inputs, outputs=pred)
def GRU3_solo(rd_input, kernels, conv_window_len, maxpooling_len, stride, BN=True, DropoutRate=0.2): initializer = 'glorot_uniform' conv1 = layers.Conv1D(kernels[0], 1, strides= stride , padding='same', \ kernel_initializer=initializer)(rd_input) output = layers.Bidirectional(layers.CuDNNGRU( 32, return_sequences=True))(conv1) output = layers.Bidirectional(layers.CuDNNGRU( 32, return_sequences=True))(output) output = layers.Bidirectional(layers.CuDNNGRU( 32, return_sequences=True))(output) output = layers.TimeDistributed(layers.Dense(8, activation="softmax"))(output) model = models.Model(rd_input, output) return model
def UNet_GRU3(rd_input, kernels, conv_window_len, maxpooling_len, stride, BN=True, DropoutRate=0.2): unet_module_output = UNet_module(rd_input, kernels, conv_window_len, maxpooling_len, stride, BN, DropoutRate) output = layers.Bidirectional(layers.CuDNNGRU( 32, return_sequences=True))(unet_module_output) output = layers.Bidirectional(layers.CuDNNGRU( 32, return_sequences=True))(output) output = layers.Bidirectional(layers.CuDNNGRU( 32, return_sequences=True))(output) output = layers.TimeDistributed(layers.Dense(8, activation="softmax"))(output) model = models.Model(rd_input, output) return model
def create_LSTM(units, return_sequences, name): if MerLConfig.GPU: return kl.CuDNNGRU(units=units, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, return_sequences=return_sequences, return_state=False, stateful=False, name=name) else: return kl.GRU(units=units, activation='tanh', recurrent_activation='hard_sigmoid', use_bias=True, kernel_initializer='glorot_uniform', recurrent_initializer='orthogonal', bias_initializer='zeros', kernel_regularizer=None, recurrent_regularizer=None, bias_regularizer=None, activity_regularizer=None, kernel_constraint=None, recurrent_constraint=None, bias_constraint=None, dropout=0.0, recurrent_dropout=0.0, implementation=1, return_sequences=False, return_state=False, go_backwards=False, stateful=False, unroll=False, name=name)
def make_model(embedding_matrix): inps = nn.Input(shape=(MAXLEN, )) x = nn.Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False)(inps) x = nn.SpatialDropout1D(0.1)(x) x = nn.Bidirectional(nn.CuDNNGRU(256, return_sequences=True))(x) gap = nn.GlobalAveragePooling1D()(x) gmp = nn.GlobalMaxPooling1D()(x) x = nn.concatenate([gap, gmp]) x = nn.Dropout(0.5)(x) x = nn.BatchNormalization()(x) x = nn.Dense(5, activation='sigmoid')(x) model = Model(inputs=inps, outputs=x) model.compile(loss='binary_crossentropy', optimizer=Adamax(decay=1e-6)) return model
vel_pred = y_pred[:, :, 128 + 2] vel_true = y_true[:, :, 128 + 2] time_pred = y_pred[:, :, -1] time_true = y_true[:, :, -1] loss_notes = kls.categorical_crossentropy(notes_true, notes_pred) * 1 loss_vel = K.abs(vel_true - vel_pred) * 1 loss_time = ((time_true - time_pred)**2) * 0.01 return loss_notes + loss_vel + loss_time li = kl.Input(shape=(None, 128 + 2 + 1 + 1)) l = li for i in range(3): l = kl.CuDNNGRU(512, return_sequences=True)(l) l = kl.Dense(512, activation='elu')(l) ln = kl.Dense(128 + 2, activation='softmax')(l) lv = kl.Dense(1, activation='sigmoid')(l) lt = kl.Dense(1, activation='relu', use_bias=False)(l) l = kl.Concatenate()([ln, lv, lt]) model = Model(li, l) model.compile("adamax", myloss) def generator(batch_size, seq_length): batch = [] while True: mid_index = np.random.choice(len(padded_mids)) cmid = padded_mids[mid_index] if (len(cmid) - seq_length - 1) <= 0:
plot_train_validation_loss( history, 'Conv1D, Dropout, BatchNormalization, CuDNNLSTM, Kernel Reguliarzer, Recurrent Regularizer' ) # ## CuDNNGRU # ### 학습하기(CuDNNGRU) # In[29]: start_time = time.time() model = Sequential() model.add(layers.CuDNNGRU(32, input_shape=(None, weather_df_value.shape[-1]))) model.add(layers.Dense(1)) model.compile(optimizer=RMSprop(), loss='mae') history = model.fit_generator(train_gen, steps_per_epoch=400, epochs=20, validation_data=val_gen, validation_steps=val_steps) print("--- %s seconds ---" % (time.time() - start_time)) # ### Train Loss, Validation Loss 그래프 # In[ ]:
def URNet(rd_input, kernels, conv_window_len, maxpooling_len, stride, BN=True, DropoutRate=0.2): initializer = 'glorot_uniform' # #stride=1 #kernels =[64, 128, 256, 512] #kernels =[8, 16, 32, 64] ##################### Conv1 ######################### conv1 = layers.Conv1D(kernels[0], conv_window_len, strides= stride , padding='same', \ kernel_initializer=initializer)(rd_input) if BN: conv1 = layers.BatchNormalization()(conv1) conv1 = layers.Activation('relu')(conv1) conv1 = layers.Conv1D(kernels[0], conv_window_len, strides= stride, padding='same', \ kernel_initializer=initializer)(conv1) if BN: conv1 = layers.BatchNormalization()(conv1) conv1 = layers.Activation('relu')(conv1) rnn1 = layers.CuDNNGRU(kernels[0], return_sequences=True)(conv1) # use it for the next stage pool1 = layers.MaxPooling1D(maxpooling_len[0])(rnn1) ##################### Conv2 ########################## conv2 = layers.Conv1D(kernels[1], 3, strides= stride, padding='same',\ kernel_initializer=initializer)(pool1) if BN: conv2 = layers.BatchNormalization()(conv2) conv2 = layers.Activation('relu')(conv2) conv2 = layers.Conv1D(kernels[1], 3, strides= stride, padding='same',\ kernel_initializer=initializer)(conv2) if BN: conv2 = layers.BatchNormalization()(conv2) conv2 = layers.Activation('relu')(conv2) rnn2 = layers.CuDNNGRU(kernels[1], return_sequences=True)(conv2) #use it for the next stage pool2 = layers.MaxPooling1D(maxpooling_len[1])(rnn2) ##################### conv3 ########################### conv3 = layers.Conv1D(kernels[2], 3, strides= stride, padding='same',\ kernel_initializer=initializer)(pool2) if BN: conv3 = layers.BatchNormalization()(conv3) conv3 = layers.Activation('relu')(conv3) conv3 = layers.Conv1D(kernels[2], 3, strides= stride, padding='same',\ kernel_initializer=initializer)(conv3) if BN: conv3 = layers.BatchNormalization()(conv3) conv3 = layers.Activation('relu')(conv3) if DropoutRate > 0: drop3 = layers.Dropout(DropoutRate)(conv3) else: drop3 = conv3 rnn3 = layers.CuDNNGRU(kernels[2], return_sequences=True)(drop3) # use it for the next stage pool3 = layers.MaxPooling1D(maxpooling_len[2])(rnn3) #################### conv4 (U bottle) ##################### conv4 = layers.Conv1D(kernels[3], 3, strides= 1, padding='same',\ kernel_initializer=initializer)(pool3) if BN: conv4 = layers.BatchNormalization()(conv4) conv4 = layers.Activation('relu')(conv4) conv4 = layers.Conv1D(kernels[3], 3, strides= 1, padding='same',\ kernel_initializer=initializer)(conv4) if BN: conv4 = layers.BatchNormalization()(conv4) conv4 = layers.Activation('relu')(conv4) if DropoutRate > 0: drop4 = layers.Dropout(DropoutRate)(conv4) else: drop4 = conv4 ################### upSampling, upConv5 ########################## # up5 = layers.UpSampling1D(maxpooling_len[2])(drop4) up5 = Conv1DTranspose(drop4, kernels[2], 3, strides=maxpooling_len[2], padding='same') merge5 = layers.Concatenate(-1)([drop3, up5]) merge5 = layers.Concatenate(-1)([merge5, rnn3]) conv5 = layers.Conv1D(kernels[2], 3, padding='same', \ kernel_initializer=initializer)(merge5) if BN: conv5 = layers.BatchNormalization()(conv5) conv5 = layers.Activation('relu')(conv5) conv5 = layers.Conv1D(kernels[2], 3, padding='same', \ kernel_initializer=initializer)(conv5) if BN: conv5 = layers.BatchNormalization()(conv5) conv5 = layers.Activation('relu')(conv5) ################### upConv 6 ############################## #up6 = layers.UpSampling1D(maxpooling_len[1])(conv5) up6 = Conv1DTranspose(conv5, kernels[1], 3, strides=maxpooling_len[1], padding='same') merge6 = layers.Concatenate(-1)([conv2, up6]) merge6 = layers.Concatenate(-1)([merge6, rnn2]) conv6 = layers.Conv1D(kernels[1], 3, padding='same', \ kernel_initializer=initializer)(merge6) if BN: conv6 = layers.BatchNormalization()(conv6) conv6 = layers.Activation('relu')(conv6) conv6 = layers.Conv1D(kernels[1], 3, padding='same',\ kernel_initializer=initializer)(conv6) if BN: conv6 = layers.BatchNormalization()(conv6) conv6 = layers.Activation('relu')(conv6) ################### upConv 7 ######################### #up7 = layers.UpSampling1D(maxpooling_len[0])(conv6) up7 = Conv1DTranspose(conv6, kernels[0], 3, strides=maxpooling_len[0], padding='same') merge7 = layers.Concatenate(-1)([conv1, up7]) merge7 = layers.Concatenate(-1)([merge7, rnn1]) conv7 = layers.Conv1D(kernels[0], 3, padding='same',\ kernel_initializer=initializer)(merge7) if BN: conv7 = layers.BatchNormalization()(conv7) conv7 = layers.Activation('relu')(conv7) conv7 = layers.Conv1D(kernels[0], 3, padding='same', \ kernel_initializer=initializer)(conv7) if BN: conv7 = layers.BatchNormalization()(conv7) conv7 = layers.Activation('relu')(conv7) ################## final output ###################### conv8 = layers.Conv1D(kernels[0], 3, padding='same', \ kernel_initializer=initializer)(conv7) if BN: conv8 = layers.BatchNormalization()(conv8) conv8 = layers.Activation('relu')(conv8) if DropoutRate > 0: conv8 = layers.Dropout(DropoutRate)(conv8) output = conv8 output = layers.Bidirectional(layers.CuDNNGRU( 64, return_sequences=True))(conv8) output = layers.Bidirectional(layers.CuDNNGRU( 64, return_sequences=True))(output) output = layers.Bidirectional(layers.CuDNNGRU( 64, return_sequences=True))(output) output = layers.TimeDistributed(layers.Dense(8, activation="softmax"))(output) #conv9 = layers.Conv1D(8, 1, activation='softmax')(conv8) model = models.Model(rd_input, output) return model
# User options args.add_argument('--output', type=int, default=1) args.add_argument('--epochs', type=int, default=100) args.add_argument('--batch', type=int, default=128) args.add_argument('--strmaxlen', type=int, default=150) args.add_argument('--embedding', type=int, default=256) args.add_argument('--threshold', type=float, default=0.5) config = args.parse_args() #------------------------------------------------------------------ core_inputs = layers.Input((config.strmaxlen, )) core_layer = layers.Embedding(252, config.embedding, input_length=config.strmaxlen)(core_inputs) core_layer = layers.Bidirectional( layers.CuDNNGRU(256, return_sequences=True))(core_layer) core_layer = layers.Bidirectional( layers.CuDNNGRU(256, return_sequences=False))(core_layer) core_model = models.Model(inputs=core_inputs, outputs=core_layer) inputs1 = layers.Input((config.strmaxlen, )) inputs2 = layers.Input((config.strmaxlen, )) layer1 = core_model(inputs1) layer2 = core_model(inputs2) main_layer = layers.Subtract()([layer1, layer2]) main_layer = layers.Lambda(lambda layer: tf.norm( layer, ord=2, axis=-1, keep_dims=True))(main_layer) main_layer = layers.Dense(1)(main_layer) main_layer = layers.Activation('sigmoid')(main_layer) main_model = models.Model(inputs=[inputs1, inputs2], outputs=main_layer)
# DONOTCHANGE: They are reserved for nsml args.add_argument('--mode', type=str, default='train') args.add_argument('--pause', type=int, default=0) args.add_argument('--iteration', type=str, default='0') # User options args.add_argument('--output', type=int, default=1) args.add_argument('--epochs', type=int, default=100) args.add_argument('--batch', type=int, default=512) args.add_argument('--strmaxlen', type=int, default=150) args.add_argument('--embedding', type=int, default=256) config = args.parse_args() inputs = layers.Input((config.strmaxlen,)) layer = layers.Embedding(251, config.embedding, input_length=config.strmaxlen)(inputs) layer = layers.Bidirectional(layers.CuDNNGRU(512, return_sequences=True))(layer) layer = layers.Bidirectional(layers.CuDNNGRU(512, return_sequences=False))(layer) layer1 = layers.Dense(3)(layer) outputs1 = layers.Activation('softmax')(layer1) layer2 = layers.Dense(1)(layer1) outputs2 = layers.Activation('sigmoid')(layer2) outputs2 = layers.Lambda(lambda layer: layer * 9 + 1)(outputs2) model = models.Model(inputs=inputs, outputs=[outputs1, outputs2]) model.summary() model.compile(optimizer=optimizers.Adam(lr=0.001, amsgrad=True, clipvalue=1.0), loss=['categorical_crossentropy', 'mse'], metrics=['accuracy']) # DONOTCHANGE: Reserved for nsml use bind_model(model)