class_weight = {0: 1., 1: 300.}
#%% LSTM
model = Sequential()
model.add(
Conv2D(filters=256,
kernel_size=(7, 2),
activation='tanh',
input_shape=(x_train.shape[1], x_train.shape[2], 1),
padding='valid'))
model.add(Dropout(0.3))
model.add(
Conv2D(filters=256, kernel_size=(7, 1), activation='tanh', padding='same'))
model.add(Dropout(0.2))
model.add(TimeDistributed(Flatten()))
model.add(GRU(units=256, return_sequences=True))
model.add(GRU(units=256, return_sequences=True))
model.add(Flatten())
model.add(Dense(units=512, activation='tanh'))
model.add(Dropout(0.4))
model.add(Dense(units=128, activation='tanh'))
model.add(Dense(units=1, activation='sigmoid'))
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['accuracy'])
model_history = model.fit(x_train,
y_train,
batch_size=1024,
epochs=50,
def build_discriminator():
# one aproch
# https://towardsdatascience.com/an-approach-towards-convolutional-recurrent-neural-networks-f54cbeecd4a6
# vary setings
shape_in_x = int(320), int(8), int(1)
shape_in_y = int(720), int(1280), int(3)
shape_out = int(8134), int(120)
dropoutrate = 0.3
x_start = Input(shape=(shape_in_x))
x = x_start
for _i, _cnt in enumerate((2, 2)):
x = Conv2D(filters = 100, kernel_size=(2, 2), padding='same',)(x)
x = BatchNormalization(axis=1)(x)
#x = Activation('relu')(x)
x = LeakyReLU()(x)
#x = MaxPooling2D(pool_size=(2,2), dim_ordering="th" )(x)
x = MaxPooling2D(pool_size=2)(x)
x = Dropout(dropoutrate)(x)
x = Permute((2, 1, 3))(x)
x = Reshape((1, 16000))(x)
#out = Activation('sigmoid', name='strong_out')(x)
#audio_context = Model(inputs=x_start, outputs=out)
#audio_context.compile(optimizer='Adam', loss='binary_crossentropy',metrics = ['accuracy'])
#audio_context.summary()
y_start = Input(shape=(shape_in_y))
y = y_start
for _i, _cnt in enumerate((2, 2)):
y = Conv2D(filters = 100, kernel_size=(2, 2), padding='same',)(y)
y = BatchNormalization(axis=1)(y)
#y = Activation('relu')(y)
y = LeakyReLU()(y)
#y = MayPooling2D(pool_size=(2,2), dim_ordering="th" )(y)
y = MaxPooling2D(pool_size=2)(y)
y = Dropout(dropoutrate)(y)
y = Permute((2, 1, 3))(y)
y = Reshape((1, 5760000))(y)
#out = Activation('sigmoid', name='strong_out')(y)
#audio_context = Model(inputs=y_start, outputs=out)
#audio_context.compile(optimizer='Adam', loss='binary_crossentropy',metrics = ['accuracy'])
#audio_context.summary()
z = Concatenate()([x, y])
#out = Activation('sigmoid', name='strong_out')(z)
#audio_context = Model(inputs=[x_start,y_start], outputs=out)
#audio_context.compile(optimizer='Adam', loss='binary_crossentropy',metrics = ['accuracy'])
#audio_context.summary()
# The Gru/recurrent portion
# Get some knowledge
# http://colah.github.io/posts/2015-08-Understanding-LSTMs/
for r in (10,10):
z = Bidirectional(
GRU(r,
activation='tanh',
dropout=dropoutrate,
recurrent_dropout=dropoutrate,
return_sequences=True),
merge_mode='concat')(z)
for f in ((2,2)):
z = TimeDistributed(Dense(f))(z)
#z = Dropout(dropoutrate)(z)
#z = TimeDistributed(Dense(880))(z)
# arbitrary reshape may be a problem
z = Flatten()(z)
z = Dropout(dropoutrate)(z)
z = Dense(1,activation = 'sigmoid')(z)
#out = Activation('sigmoid', name='strong_out')(z)
descrim = Model(inputs=[x_start, y_start], outputs=z)
descrim.compile(optimizer='Adam', loss='binary_crossentropy',metrics = ['accuracy'])
#descrim.summary()
#ce = audio_context
return descrim
import numpy as np
x = np.array([[1, 2, 3], [2, 3, 4], [3, 4, 5], [4, 5, 6]])
y = np.array([4, 5, 6, 7])
print('x.shape : ', x.shape) #(4,3)
print('y.shape : ', y.shape) #(4,)
x = x.reshape(4, 3, 1)
#2. 모델구성
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, LSTM, GRU
model = Sequential()
model.add(GRU(10, activation='relu', input_shape=(3, 1))) #x변경 됐음
model.add(Dense(20))
model.add(Dense(10))
model.add(Dense(1)) #LSTM있는 곳에 디폴트가 탄젠트인것, output은 linear임
# 여기까지는 회귀값이다. 분류값 아님
#model.summary()
#3. 컴파일 , 훈련
model.compile(loss='mse', optimizer='adam')
model.fit(x, y, epochs=100, batch_size=1)
#4. 평가, 예측
loss = model.evaluate(x, y)
print(loss)
x_pred = np.array([5, 6, 7]) #(3,) 행은 하나 -> (1,3,1)
def build_attention_model(self):
if self.cell_type == "RNN":
# encoder
encoder_inputs = Input(shape=(None, len(self.srcChar2Int)))
encoder_outputs = encoder_inputs
for i in range(1, self.numEncoders + 1):
encoder = SimpleRNN(
self.latentDim,
return_state=True,
return_sequences=True,
dropout=self.dropout,
)
encoder_outputs, state = encoder(encoder_inputs)
if i == 1:
encoder_first_outputs= encoder_outputs
encoder_states = [state]
# decoder
decoder_inputs = Input(shape=(None, len(self.tgtChar2Int)))
decoder_outputs = decoder_inputs
for i in range(1, self.numDecoders + 1):
decoder = SimpleRNN(
self.latentDim,
return_sequences=True,
return_state=True,
dropout=self.dropout,
)
decoder_outputs, _ = decoder(decoder_inputs, initial_state=encoder_states)
if i == self.numDecoders:
decoder_first_outputs = decoder_outputs
attention_layer = AttentionLayer(name='attention_layer')
attention_out, attention_states = attention_layer([encoder_first_outputs, decoder_first_outputs])
decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_outputs, attention_out])
# dense
hidden = Dense(self.hidden, activation="relu")
hidden_time = TimeDistributed(hidden, name='time_distributed_layer')
hidden_outputs = hidden(decoder_concat_input)
decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax")
decoder_outputs = decoder_dense(hidden_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
return model
elif self.cell_type == "LSTM":
# encoder
encoder_inputs = Input(shape=(None, len(self.srcChar2Int)))
encoder_outputs = encoder_inputs
for i in range(1, self.numEncoders + 1):
encoder = LSTM(
self.latentDim,
return_state=True,
return_sequences=True,
dropout=self.dropout,
)
encoder_outputs, state_h, state_c = encoder(encoder_outputs)
if i == 1:
encoder_first_outputs= encoder_outputs
encoder_states = [state_h, state_c]
# decoder
decoder_inputs = Input(shape=(None, len(self.tgtChar2Int)))
decoder_outputs = decoder_inputs
for i in range(1, self.numDecoders + 1):
decoder = LSTM(
self.latentDim,
return_state=True,
return_sequences=True,
dropout=self.dropout,
)
decoder_outputs, _, _ = decoder(
decoder_outputs, initial_state=encoder_states
)
if i == self.numDecoders:
decoder_first_outputs = decoder_outputs
attention_layer = AttentionLayer(name='attention_layer')
attention_out, attention_states = attention_layer([encoder_first_outputs, decoder_first_outputs])
decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_outputs, attention_out])
# dense
hidden = Dense(self.hidden, activation="relu")
hidden_time = TimeDistributed(hidden, name='time_distributed_layer')
hidden_outputs = hidden(decoder_concat_input)
decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax")
decoder_outputs = decoder_dense(hidden_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
return model
elif self.cell_type == "GRU":
# encoder
encoder_inputs = Input(shape=(None, len(self.srcChar2Int)))
encoder_outputs = encoder_inputs
for i in range(1, self.numEncoders + 1):
encoder = GRU(
self.latentDim,
return_state=True,
return_sequences=True,
dropout=self.dropout,
)
encoder_outputs, state = encoder(encoder_inputs)
if i == 1:
encoder_first_outputs= encoder_outputs
encoder_states = [state]
# decoder
decoder_inputs = Input(shape=(None, len(self.tgtChar2Int)))
decoder_outputs = decoder_inputs
for i in range(1, self.numDecoders + 1):
decoder = GRU(
self.latentDim,
return_sequences=True,
return_state=True,
dropout=self.dropout,
)
decoder_outputs, _ = decoder(decoder_inputs, initial_state=encoder_states)
if i == self.numDecoders:
decoder_first_outputs = decoder_outputs
attention_layer = AttentionLayer(name='attention_layer')
attention_out, attention_states = attention_layer([encoder_first_outputs, decoder_first_outputs])
decoder_concat_input = Concatenate(axis=-1, name='concat_layer')([decoder_outputs, attention_out])
# dense
hidden = Dense(self.hidden, activation="relu")
hidden_time = TimeDistributed(hidden, name='time_distributed_layer')
hidden_outputs = hidden(decoder_concat_input)
decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax")
decoder_outputs = decoder_dense(hidden_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
return model
dencoder_seq = dencoder_tokenizer.texts_to_sequences(eng)
print(dencoder_seq[1])
dencoder_max_len = max([len(i) for i in dencoder_seq])
dencoder_pad = np.array(pad_sequences(dencoder_seq,
maxlen=dencoder_max_len + 1,
padding='post',
truncating='post'),
dtype='float')
dencoder_input_data = dencoder_pad[:, :-1]
dencoder_output_data = dencoder_pad[:, 1:]
dencoder_tokens = dict([(w, q) for q, w in dencoder_words.items()])
encoder_input = Input(shape=(None, ))
encoder_embedding = Embedding(total_words, 128)(encoder_input)
encoder_gru1 = GRU(128, return_sequences=True)(encoder_embedding)
encoder_gru2 = GRU(128, return_sequences=True)(encoder_gru1)
encoder_gru3 = GRU(128, return_sequences=False)(encoder_gru2)
# encoder_model=Sequential()
# encoder_model.add(Embedding(total_words, 128, input_length=encoder_max_len))
# encoder_model.add(Bidirectional(GRU(128,return_sequences = True)))
# encoder_model.add(Bidirectional(GRU(128,return_sequences = True)))
# encoder_outputs, state_h=encoder_model.add(Bidirectional(GRU(128,return_sequences = False,return_state=True)))
# encoder_outputs, state_h=encoder_model.output
dencoder_initial_state = Input(shape=(128, ))
dencoder_input = Input(shape=(None, ))
dencoder_embedding = Embedding(total_words, 128)(dencoder_input)
dencoder_gru1 = GRU(128, return_sequences=True)(dencoder_embedding,
initial_state=encoder_gru3)
dencoder_gru2 = GRU(128, return_sequences=True)(dencoder_gru1,
X = roll[:seq_len].reshape(seq_len) / float(len(alphabet))
y = utils.to_categorical(roll[seq_len % len(alphabet)], len(alphabet))
data["X"].append(X)
data["y"].append(y)
X = pad_sequences(data["X"],
maxlen=len(alphabet),
value=mask_value,
dtype=np.float32)
X = np.reshape(X, (X.shape[0], X.shape[1], 1))
y = np.array(data["y"])
y = np.reshape(y, (y.shape[0], y.shape[2]))
model = Sequential()
model.add(Masking(mask_value=mask_value, input_shape=(len(alphabet), 1)))
model.add(GRU(32))
model.add(Dense(len(alphabet), activation="softmax"))
model.compile(loss="categorical_crossentropy",
optimizer="nadam",
metrics=["accuracy"])
print(model.summary())
class EvaluationCallback(Callback):
def __init__(self, metric):
if metric == "loss":
self.metric = "loss"
else:
self.metric = "accuracy"
print("===> Metric is", metric)
dataset = dataset.reshape(40, 10, 2)
label = []
for i in range(5, 445):
if 94 < i <105 or 194 < i <205 or 294 < i <305 or 394 < i <405:
continue
y_data = [dataset_x[i]/100, dataset_y[i]/100]
label.append(y_data)
label = np.array(label, dtype=float)
label = label.reshape(40, 20)
# split data and labels into train and test
x_train, x_test, y_train, y_test = train_test_split(dataset, label, test_size=0.2, random_state=4)
print y_test.shape
model =Sequential()
model.add(GRU(units=50, return_sequences=True, input_shape = (x_train.shape[1],2)))
model.add(GRU(units=50))
model.add(Dense(20))
model.compile(loss='mean_absolute_error', optimizer='adam', metrics=['accuracy'])
model.summary()
history = model.fit(x_train, y_train, epochs=500, validation_data=(x_test, y_test))
results = model.predict(x_test)
results = results.reshape(results.shape[0],10,2)
y_test = y_test.reshape(8, 10, 2)
results = results.reshape(8, 10, 2)
a_axis=[]
b_axis=[]
a_r_axis = []
b_r_axis = []
for i in range(8):
def _build(self):
self.bigru = Bidirectional(
GRU(cfg.HIDDEN_DIM // 2, return_sequences=True, return_state=True))
vocab_size = len(tokenizer_obj.word_index) + 1
EMBEDDING_DIM = 100
max_length = max([len(s.split()) for s in x_all])
x_train_tokens = tokenizer_obj.texts_to_sequences(x_train)
x_test_tokens = tokenizer_obj.texts_to_sequences(x_test)
x_train_pad = pad_sequences(x_train_tokens, maxlen=max_length, padding='post')
x_test_pad = pad_sequences(x_test_tokens, maxlen=max_length, padding='post')
rnn_model = Sequential()
rnn_model.add(Embedding(vocab_size, EMBEDDING_DIM, input_length=max_length))
rnn_model.add(GRU(units=64, dropout=0.1, recurrent_dropout=0.1))
rnn_model.add(BatchNormalization())
rnn_model.add(Dense(1, activation='sigmoid'))
_optimizer = optimizers.RMSprop(lr=0.01)
rnn_model.compile(loss='binary_crossentropy', optimizer=_optimizer, metrics=['accuracy'])
print(rnn_model.summary())
print('Started training the model...')
early_stopping = tensorflow.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=10)
history = rnn_model.fit(x_train_pad, y_train,
batch_size=32,
epochs=15,
validation_data=(x_test_pad, y_test),
callbacks=[early_stopping],
verbose=2)
dataf = np.reshape(Train_data, [315*1000, 64])
rng_state = np.random.get_state()
np.random.shuffle(dataf)
np.random.set_state(rng_state)
np.random.shuffle(labelsf)
labels = np.reshape(labelsf, [315, 1000, 1])
Train_data = np.reshape(dataf, [315, 1000, 64])
X_train, X_test = train_test_split(Train_data,test_size=0.2)
y_train, y_test = train_test_split(labels,test_size=0.2)
lab = np.reshape(labels, [315*1000,])
class_weights = class_weight.compute_class_weight('balanced', np.unique(lab), lab)
training_in_shape = Train_data.shape[1:]
training_in = Input(shape = training_in_shape)
Var = GRU(1240, return_sequences=True, stateful = False)(training_in)
training_pred = Dense(3, activation = 'softmax')(Var)
training_model = Model(inputs = training_in, outputs = training_pred)
training_model.compile(loss = keras.losses.SparseCategoricalCrossentropy(),
optimizer = 'adam',
metrics = ['accuracy'])
training_model.summary()
results = training_model.fit(Train_data, labels, batch_size = 4, epochs = 20,
validation_split = 0.2)
streaming_in = Input(batch_shape=(1,None,64)) ## stateful ==> needs batch_shape specified
foo = GRU(1240, return_sequences=False, stateful=True )(streaming_in)
streaming_pred = Dense(3, activation = 'softmax')(foo)
def bert_tensorflow_test(X_train, X_test, Y_train, Y_test):
# Model
model = Sequential()
model.add(
Masking(mask_value=0., input_shape=(MAX_SEQUENCE_LEN, VECTOR_DIM)))
#forward_layer = LSTM(200, return_sequences=True)
forward_layer = GRU(10, return_sequences=False, dropout=0.5)
#backward_layer = LSTM(200, activation='relu', return_sequences=True,
backward_layer = GRU(10,
return_sequences=False,
dropout=0.5,
go_backwards=True)
model.add(
Bidirectional(forward_layer,
backward_layer=backward_layer,
input_shape=(MAX_SEQUENCE_LEN, VECTOR_DIM)))
#model.add(TimeDistributed(Dense(NUM_CLASSES)))
# Remove TimeDistributed() so that predictions are now made for the entire sentence
model.add(Dense(NUM_CLASSES))
model.add(Activation('softmax'))
#print('preds shape', model.predict(X_train[:3]).shape)
#print('Y_train shape', Y_train[:3].shape)
#print(list(Y_train[:3]))
classes = []
for y in Y_train:
cls = np.argmax(y)
classes.append(cls)
print(Counter(classes))
model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
print('compiled model')
model.fit(X_train, Y_train, batch_size=8,
epochs=10) #, validation_split=0.1)
print('fit model')
eval = model.evaluate(X_test, Y_test, batch_size=8)
#print('X_test[0]')
#print(X_test[0])
#print(X_train[0])
preds = model.predict_proba(X_test, verbose=1, batch_size=8)
print(preds)
num_correct = 0
num_incorrect = 0
TP = 0
TN = 0
FP = 0
FN = 0
# idiomatic = 2, non-idiomatic = 3
with open('preds_out_temp.txt', 'w') as tempoutf:
for pred, y in zip(preds, Y_test):
if np.argmax(y) == 2 or np.argmax(y) == 3:
if np.argmax(y) == np.argmax(pred):
num_correct += 1
else:
num_incorrect += 1
if np.argmax(pred) == 2 and np.argmax(y) == 2:
TP += 1
if np.argmax(pred) == 3 and np.argmax(y) == 3:
TN += 1
if np.argmax(pred) == 2 and np.argmax(y) == 3:
FP += 1
if np.argmax(pred) == 3 and np.argmax(y) == 2:
FN += 1
custom_accuracy = num_correct / (num_correct + num_incorrect)
print('custom accuracy is', num_correct / (num_correct + num_incorrect))
for y in Y_test:
cls = np.argmax(y)
classes.append(cls)
class_nums = Counter(classes)
print(class_nums)
default_acc = class_nums[2] / (class_nums[2] + class_nums[3])
print('default accuracy is', default_acc, 'or', 1 - default_acc)
return eval, custom_accuracy, default_acc, [TP, TN, FP, FN]
labels = np.array(temp)
# Divide the data into train, val and test data
x_train = data[:20000]
x_val = data[20000:24000]
x_test = data[24000:30000]
x_train_img = image_data[:20000]
x_val_img = image_data[20000:24000]
x_test_img = image_data[24000:30000]
y_train = labels[:20000]
y_val = labels[20000:24000]
y_test = labels[24000:30000]
# Build NwQM-w/oT model
page_input = Input(shape=(MAX_SECS, EMBED_DIM), dtype='float32')
l_lstm_sec = Bidirectional(GRU(100, return_sequences=True))(page_input)
l_att_sec = AttLayer(100)(l_lstm_sec)
img_inp = Input(shape=(2048, ))
img_rep = Dense(200, activation='relu')(img_inp)
norm1 = norm(l_att_sec - img_rep, ord=2, keepdims=True, axis=-1)
final_rep = concatenate([l_att_sec, img_rep, norm1], axis=-1)
preds = Dense(6, activation='softmax')(final_rep)
model = Model([page_input, img_inp], preds)
optim = Adam(learning_rate=args.learning_rate,
beta_1=0.9,
beta_2=0.999,
amsgrad=False)
def get_test_model_gru_stateful_optional(stateful):
"""Returns a test model for Gated Recurrent Unit (GRU) layers."""
input_shapes = [(17, 4), (1, 10)]
stateful_batch_size = 1
inputs = [
Input(batch_shape=(stateful_batch_size, ) + s) for s in input_shapes
]
outputs = []
for inp in inputs:
gru_sequences = GRU(stateful=stateful,
units=8,
recurrent_activation='relu',
reset_after=True,
return_sequences=True,
use_bias=True)(inp)
gru_regular = GRU(stateful=stateful,
units=3,
recurrent_activation='sigmoid',
reset_after=True,
return_sequences=False,
use_bias=False)(gru_sequences)
outputs.append(gru_regular)
gru_bidi_sequences = Bidirectional(
GRU(stateful=stateful,
units=4,
recurrent_activation='hard_sigmoid',
reset_after=False,
return_sequences=True,
use_bias=True))(inp)
gru_bidi = Bidirectional(
GRU(stateful=stateful,
units=6,
recurrent_activation='sigmoid',
reset_after=True,
return_sequences=False,
use_bias=False))(gru_bidi_sequences)
outputs.append(gru_bidi)
gru_gpu_regular = GRU(stateful=stateful,
units=3,
activation='tanh',
recurrent_activation='sigmoid',
reset_after=True,
use_bias=True)(inp)
gru_gpu_bidi = Bidirectional(
GRU(stateful=stateful,
units=3,
activation='tanh',
recurrent_activation='sigmoid',
reset_after=True,
use_bias=True))(inp)
outputs.append(gru_gpu_regular)
outputs.append(gru_gpu_bidi)
model = Model(inputs=inputs, outputs=outputs, name='test_model_gru')
model.compile(loss='mse', optimizer='nadam')
# fit to dummy data
training_data_size = stateful_batch_size
data_in = generate_input_data(training_data_size, input_shapes)
initial_data_out = model.predict(data_in)
data_out = generate_output_data(training_data_size, initial_data_out)
model.fit(data_in, data_out, batch_size=stateful_batch_size, epochs=10)
return model
test_gen = generator(df_values,
lookback=LOOKBACK,
delay=DELAY,
min_index=test_min_i,
max_index=test_max_i,
batch_size=BATCH_SIZE,
step=STEP,
shuffle=False)
# Instantiate Model
###################
clear_session()
model3 = Sequential()
model3.add(
GRU(32,
dropout=0.2,
recurrent_dropout=0.2,
input_shape=(None, df_values.shape[-1])))
model3.add(Dense(1))
model3.compile(
optimizer=RMSprop(),
loss='mae',
)
print(model3.summary())
# Train
#######
history3 = model3.fit(train_gen,
steps_per_epoch=500,
epochs=40,
validation_data=val_gen,
validation_steps=val_steps)
def flor(input_size, output_size, learning_rate=5e-4):
"""
Gated Convolucional Recurrent Neural Network by Flor et al.
"""
input_data = Input(name="input", shape=input_size)
cnn = Conv2D(filters=16,
kernel_size=(3, 3),
strides=(2, 2),
padding="same")(input_data)
cnn = PReLU(shared_axes=[1, 2])(cnn)
cnn = BatchNormalization(renorm=True)(cnn)
cnn = FullGatedConv2D(filters=16, kernel_size=(3, 3), padding="same")(cnn)
cnn = Conv2D(filters=32,
kernel_size=(3, 3),
strides=(1, 2),
padding="same")(cnn)
cnn = PReLU(shared_axes=[1, 2])(cnn)
cnn = BatchNormalization(renorm=True)(cnn)
cnn = FullGatedConv2D(filters=32, kernel_size=(3, 3), padding="same")(cnn)
cnn = Conv2D(filters=40,
kernel_size=(2, 4),
strides=(2, 2),
padding="same")(cnn)
cnn = PReLU(shared_axes=[1, 2])(cnn)
cnn = BatchNormalization(renorm=True)(cnn)
cnn = FullGatedConv2D(filters=40,
kernel_size=(3, 3),
padding="same",
kernel_constraint=MaxNorm(4, [0, 1, 2]))(cnn)
cnn = Dropout(rate=0.2)(cnn)
cnn = Conv2D(filters=48,
kernel_size=(3, 3),
strides=(1, 2),
padding="same")(cnn)
cnn = PReLU(shared_axes=[1, 2])(cnn)
cnn = BatchNormalization(renorm=True)(cnn)
cnn = FullGatedConv2D(filters=48,
kernel_size=(3, 3),
padding="same",
kernel_constraint=MaxNorm(4, [0, 1, 2]))(cnn)
cnn = Dropout(rate=0.2)(cnn)
cnn = Conv2D(filters=56,
kernel_size=(2, 4),
strides=(2, 2),
padding="same")(cnn)
cnn = PReLU(shared_axes=[1, 2])(cnn)
cnn = BatchNormalization(renorm=True)(cnn)
cnn = FullGatedConv2D(filters=56,
kernel_size=(3, 3),
padding="same",
kernel_constraint=MaxNorm(4, [0, 1, 2]))(cnn)
cnn = Dropout(rate=0.2)(cnn)
cnn = Conv2D(filters=64,
kernel_size=(3, 3),
strides=(1, 1),
padding="same")(cnn)
cnn = PReLU(shared_axes=[1, 2])(cnn)
cnn = BatchNormalization(renorm=True)(cnn)
cnn = MaxPooling2D(pool_size=(1, 2), strides=(1, 2), padding="valid")(cnn)
shape = cnn.get_shape()
bgru = Reshape((shape[1], shape[2] * shape[3]))(cnn)
bgru = Bidirectional(GRU(units=128, return_sequences=True,
dropout=0.5))(bgru)
bgru = TimeDistributed(Dense(units=128))(bgru)
bgru = Bidirectional(GRU(units=128, return_sequences=True,
dropout=0.5))(bgru)
output_data = TimeDistributed(
Dense(units=output_size, activation="softmax"))(bgru)
optimizer = RMSprop(learning_rate=learning_rate)
return (input_data, output_data, optimizer)
def DMN(n_answer, embedding, mask_zero=True, trainable=True):
context_shape = (MAX_CONTEXT_LENGTH, )
question_shape = (MAX_QUESTION_LENGTH, )
answer_shape = (MAX_ANSWER_LENGTH, )
in_context = Input(shape=context_shape, dtype='int32')
in_question = Input(shape=question_shape, dtype='int32')
in_answer = Input(shape=answer_shape, dtype='int32')
embedding_layer = PretrainedEmbedding(embeddings=embedding,
mask_zero=mask_zero,
rate=DROPOUT_RATE)
context, context_mask = embedding_layer(
in_context), embedding_layer.compute_mask(in_context)
question, question_mask = embedding_layer(
in_question), embedding_layer.compute_mask(in_question)
context, c_lst = GRU(units=INPUT_LAYER_UNIT,
return_sequences=True,
return_state=True)(context, mask=context_mask)
question = GRU(units=INPUT_LAYER_UNIT,
return_sequences=False,
return_state=False)(question, mask=question_mask)
# question = tf.expand_dims(question, axis=1)
initial_states = [question, question, question]
# TODO: this looks problematic. Maybe we should not pass m from the beginning to EPISODIC CELL?
m1, states = EpisodicModule(units=EPISODIC_LAYER_UNITS,
attention_layer_units=ATTENTION_LAYER_UNITS,
reg_scale=REG_SCALE,
trainable=trainable,
return_sequences=False,
return_state=True,
zero_output_for_mask=True,
unroll=True)(context,
initial_state=initial_states,
mask=context_mask)
# m1 = tf.expand_dims(m1, axis=1)
initial_states = [states, m1, question]
m2, states2 = EpisodicModule(units=EPISODIC_LAYER_UNITS,
attention_layer_units=ATTENTION_LAYER_UNITS,
reg_scale=REG_SCALE,
trainable=trainable,
return_sequences=False,
return_state=True,
zero_output_for_mask=True,
unroll=True)(context,
initial_state=initial_states,
mask=context_mask)
# m2 = tf.expand_dims(m2, axis=1)
initial_states = [states2, m2, question]
m = EpisodicModule(units=EPISODIC_LAYER_UNITS,
attention_layer_units=ATTENTION_LAYER_UNITS,
reg_scale=REG_SCALE,
trainable=trainable,
return_sequences=False,
return_state=False,
zero_output_for_mask=True,
unroll=True)(context,
initial_state=initial_states,
mask=context_mask)
input = tf.concat([m, question], axis=-1)
# Decode the predicted answer out
answer_outputs = LinearRegression(units=n_answer,
reg_scale=REG_SCALE,
trainable=trainable)(input)
model = Model([in_context, in_question], answer_outputs)
return model
def commandsmodel():
'''Contains the deep learning model for the commands classification
Returns:
model -- tensorflow.keras model instance'''
model = Sequential()
# first layer (Conv1D)
model.add(
Conv1D(8,
kernel_size=13,
strides=1,
padding='valid',
input_shape=(8000, 1)))
model.add(Activation('relu'))
model.add(MaxPooling1D(pool_size=3))
model.add(Dropout(0.3))
# Second layer(Second Conv1D layer)
model.add(Conv1D(16, kernel_size=11, padding='valid', strides=1))
model.add(Activation('relu'))
model.add(MaxPooling1D(pool_size=3))
model.add(Dropout(0.3))
# third layer(third Conv1D layer)
model.add(Conv1D(32, kernel_size=9, padding='valid', strides=1))
model.add(Activation('relu'))
model.add(MaxPooling1D(pool_size=3))
model.add(Dropout(0.3))
# fourth layer(fourth Conv1D layer)
model.add(Conv1D(64, kernel_size=9, padding='valid', strides=1))
model.add(Activation('relu'))
model.add(MaxPooling1D(pool_size=3))
model.add(Dropout(0.3))
# fifth layer a Gru layer
model.add(GRU(128, return_sequences=True))
model.add(Dropout(0.8))
model.add(BatchNormalization())
# sixth layer (GRU)
model.add(GRU(128, return_sequences=True))
model.add(Dropout(0.8))
model.add(BatchNormalization())
# flatten layer
model.add(Flatten())
# Dense layer 1
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.3))
# Dense layer 2
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.3))
# output layer
model.add(Dense(9, activation='softmax'))
opt = Adam()
model.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
model.summary()
return model
for line in f:
values = line.split();
word = values[0];
coefs = np.asarray(values[1:], dtype='float32');
embeddings_index[word] = coefs;
embeddings_matrix = np.zeros((vocab_size+1, embedding_dim));
for word, i in word_index.items():
embedding_vector = embeddings_index.get(word);
if embedding_vector is not None:
embeddings_matrix[i] = embedding_vector;
decoder_state=encoder_model.get_layer('dense')
decoder_initial_state=decoder_state.output
decoder_input=Input(shape=(None,),name='decoder_input')
decoder_embedding=Embedding(vocab_size+1, embedding_dim, input_length=decoder_max_len-1, weights=[embeddings_matrix], trainable=False)(decoder_input)
decoder_gru1=GRU(256,return_sequences = True)(decoder_embedding,initial_state=decoder_initial_state)
decoder_gru3=GRU(256,return_sequences = True)(decoder_gru1,initial_state=decoder_initial_state)
decoder_output=Dense(10000,activation='softmax',name='decoder_output')(decoder_gru3)
encoder_input=encoder_model.input
model_train=Model([encoder_input,decoder_input],[decoder_output])
encoder_input=model_train.get_layer('input_1')
encoder_input._name='encoder_input'
model_train.summary()
model_train.compile(loss='sparse_categorical_crossentropy',optimizer=RMSprop(lr=1e-3),metrics=['accuracy'])
path_checkpoint = r'C:\ml\image captioning\31296_39911_bundle_archive\checkpoint_weights'
callback_checkpoint = tf.keras.callbacks.ModelCheckpoint(filepath=path_checkpoint,
verbose=1,
save_weights_only=True)
callback_tensorboard = tf.keras.callbacks.TensorBoard(log_dir='C:\ml\image captioning\31296_39911_bundle_archive\callback_tensor',
from tensorflow.keras import Sequential, Input, Model
from tensorflow.keras.layers import Masking, Dense, Activation, GRU, Dropout, concatenate,LSTM
from time import time
# TF_CONFIG is the environment variable used to configure tf.distribute
# each worker will have a different number, an index entry in the nodes_endpoint list
# node 0 is master, by default
strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy()
# This implements the distributed stratedy for model
with strategy.scope():
## GRU branch
gru_input = Input(shape=(801,19), name='gru_input')
a = gru_input
a = Masking(mask_value=0.)(a)
a = GRU(units=50,activation='tanh')(a)
gruBranch = Dropout(0.2)(a)
hlf_input = Input(shape=(14), name='hlf_input')
b = hlf_input
hlfBranch = Dropout(0.2)(b)
c = concatenate([gruBranch, hlfBranch])
c = Dense(25, activation='relu')(c)
output = Dense(3, activation='softmax')(c)
model = Model(inputs=[gru_input, hlf_input], outputs=output)
## Compile model
optimizer = Adam(learning_rate=0.0005*number_workers)
loss = 'categorical_crossentropy'
def train_delta(matrix):
import numpy as np
import tensorflow as tf
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.preprocessing import MinMaxScaler
from tensorflow.keras import Model, Input
from tensorflow.keras.layers import (Dense, Dropout, GRU, Flatten,
GaussianNoise, concatenate)
from tensorflow.keras.models import load_model
from tensorflow.keras.callbacks import Callback
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.callbacks import ModelCheckpoint
from tensorflow.keras.optimizers import Adam
import supplemental_functions
from supplemental_functions import (sampling_fix, prepareinput,
prepareinput_nozero, prepareoutput)
tf.keras.backend.clear_session()
[
newdim, percent_drilled, start, stop, inc_layer1, inc_layer2,
data_layer1, data_layer2, dense_layer, range_max, memory, predictions,
drop1, drop2, lr, bs, ensemble_count
] = matrix
drop1 = drop1 / 100
drop2 = drop2 / 100
inc_layer2 = inc_layer2 / 1000
lr = lr / 10000
percent_drilled = percent_drilled / 100
df = pd.read_csv('F9ADepth.csv')
df_target = df.copy()
droplist = [
'nameWellbore', 'name', 'Pass Name unitless',
'MWD Continuous Inclination dega', 'Measured Depth m',
'MWD Continuous Azimuth dega', "Unnamed: 0", "Unnamed: 0.1"
]
for i in droplist:
df = df.drop(i, 1)
for i in list(df):
if df[i].count() < 1000:
del df[i]
info(f'dropped {i}')
start = start
stop = stop
step = 0.230876
X = np.arange(start, stop, step)
X = X.reshape(X.shape[0], 1)
X = np.arange(start, stop, step)
X = X.reshape(X.shape[0], 1)
my_data1 = sampling_fix(df_target, 'MWD Continuous Inclination dega',
start, stop, 1.7, 1, 0).predict(X)
data_array = []
for i in list(df):
sampled = sampling_fix(df_target, i, start, stop, 1.7, 3, 0).predict(X)
if np.isnan(np.sum(sampled)) == False:
data_array.append(sampled)
info(f'Using {i}')
data_array = np.asarray(data_array)
dftemp = pd.DataFrame()
dftemp['dinc'] = my_data1
dftemp['dinc'] = dftemp['dinc'].diff(1).rolling(3, center=True).mean()
my_data1 = dftemp['dinc'].ffill().bfill()
data_array = data_array.T
pre_PCA_scaler = MinMaxScaler()
data_array = pre_PCA_scaler.fit_transform(data_array)
from sklearn.decomposition import PCA
# =============================================================================
# pca = PCA().fit(data_array)
# plt.plot(np.cumsum(pca.explained_variance_ratio_))
# plt.xlabel('number of components')
# plt.ylabel('cumulative explained variance');
#
# plt.show()
# =============================================================================
sampcount = int(len(data_array) * percent_drilled)
pca = PCA(n_components=newdim).fit(data_array[:sampcount])
projected = pca.transform(data_array)
my_data = []
for i in range(newdim):
my_data.append(projected[:, i])
my_data1 = my_data1[:, np.newaxis]
my_data_newaxis = []
for i in my_data:
my_data_newaxis.append(i[:, np.newaxis])
temp_data1 = pd.DataFrame(my_data1.flatten())
temp_data1 = pd.DataFrame(my_data1)
range1 = temp_data1[0].diff(memory + predictions)
range2 = np.amax(range1)
RNN_scaler = MinMaxScaler()
my_data1 = RNN_scaler.fit_transform(my_data1)
my_data_scaled = []
for i in my_data_newaxis:
my_data_scaled.append(MinMaxScaler().fit_transform(i))
X1 = prepareinput(my_data1, memory)
Xdata = []
for i in my_data_scaled:
Xn = prepareinput_nozero(i, memory, predictions)
Xdata.append(Xn)
y_temp = prepareoutput(my_data1, memory, predictions)
stack = []
for i in range(memory):
stack.append(np.roll(my_data1, -i))
X_temp = np.hstack(stack)
y = y_temp
data_length = len(my_data1) - memory - predictions
testing_cutoff = 0.80
border1 = int((data_length) * (percent_drilled * 0.8))
border2 = int((data_length) * (percent_drilled))
border3 = int((data_length) * (percent_drilled + 0.2))
X1_train = X1[:border1]
X1_test = X1[border1:border2]
X1_test2 = X1[border2:border3]
Xdata_train = []
Xdata_test = []
Xdata_test2 = []
for i in Xdata:
Xdata_train.append(i[:border1])
Xdata_test.append(i[border1:border2])
Xdata_test2.append(i[border2:border3])
y_train, y_test, y_test2 = y[:border1], y[border1:border2], y[
border2:border3]
X1_train = X1_train.reshape((X1_train.shape[0], X1_train.shape[1], 1))
X1_test = X1_test.reshape((X1_test.shape[0], X1_test.shape[1], 1))
X1_test2 = X1_test2.reshape((X1_test2.shape[0], X1_test2.shape[1], 1))
Xdata_train_r = []
Xdata_test_r = []
Xdata_test2_r = []
for i in range(newdim):
Xdata_train_r.append(Xdata_train[i].reshape(
(Xdata_train[i].shape[0], Xdata_train[i].shape[1], 1)))
Xdata_test_r.append(Xdata_test[i].reshape(
(Xdata_test[i].shape[0], Xdata_test[i].shape[1], 1)))
Xdata_test2_r.append(Xdata_test2[i].reshape(
(Xdata_test2[i].shape[0], Xdata_test2[i].shape[1], 1)))
X_train_con = np.concatenate(Xdata_train_r, axis=2)
X_test_con = np.concatenate(Xdata_test_r, axis=2)
X_test2_con = np.concatenate(Xdata_test2_r, axis=2)
X_train = [X1_train, X_train_con]
X_test = [X1_test, X_test_con]
X_test2 = [X1_test2, X_test2_con]
input1 = Input(shape=(memory, 1))
input2 = Input(shape=(memory + predictions, newdim))
x1 = GaussianNoise(inc_layer2, input_shape=(memory, 1))(input1)
x1 = GRU(units=inc_layer1,
kernel_initializer='glorot_uniform',
recurrent_initializer='orthogonal',
bias_initializer='zeros',
kernel_regularizer='l2',
recurrent_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
kernel_constraint=None,
recurrent_constraint=None,
bias_constraint=None,
return_sequences=False,
return_state=False,
stateful=False)(x1)
x1 = Dropout(drop1)(x1)
x1 = Model(inputs=input1, outputs=x1)
x2 = Dense(data_layer1, input_shape=(memory + predictions, 3))(input2)
x2 = Dropout(drop2)(x2)
x2 = Flatten()(x2)
x2 = Dense(data_layer2)(x2)
x2 = Model(inputs=input2, outputs=x2)
combined = concatenate([x1.output, x2.output])
z = Dense(dense_layer, activation="relu")(combined)
z = Dense(predictions, activation="linear")(z)
#define the model
myadam = Adam(learning_rate=lr, beta_1=0.9, beta_2=0.999, amsgrad=False)
class PlotResuls(Callback):
def on_train_begin(self, logs={}):
self.i = 0
self.x = []
self.losses = []
self.val_losses = []
#self.fig = plt.figure()
self.logs = []
def on_epoch_end(self, epoch, logs={}):
self.logs.append(logs)
self.x.append(self.i)
self.losses.append(logs.get('loss'))
self.val_losses.append(logs.get('val_loss'))
self.i += 1
#print (".", end = '')
if (epoch % 14999 == 0) & (epoch > 0):
print(epoch)
plt.plot(self.x, np.log(self.losses), label="loss")
plt.plot(self.x, np.log(self.val_losses), label="val_loss")
plt.grid(color='gray', linestyle='-', linewidth=1, alpha=0.2)
plt.title("Loss")
plt.legend()
plt.show()
#mymanyplots(epoch, data, model)
#data = [X1, X2, X3, X4, y, X1_train,X_train, X_test, X1_test, border1, border2, y_train, y_test, memory, y_temp, predictions]
plot_results = PlotResuls()
es = EarlyStopping(monitor='val_loss', mode='min', verbose=0, patience=25)
ens_val_array = np.zeros(ensemble_count)
ens_test_array = np.zeros(ensemble_count)
for ens_no in range(ensemble_count):
tf.keras.backend.clear_session()
mc = ModelCheckpoint(f'best_model_ens_{ens_no}.h5',
monitor='val_loss',
mode='min',
save_best_only=True,
verbose=0)
model = Model(inputs=[x1.input, x2.input], outputs=z)
model.compile(optimizer=myadam, loss='mean_squared_error')
history = model.fit(X_train,
y_train,
validation_data=(X_test, y_test),
epochs=2000,
verbose=0,
batch_size=bs,
callbacks=[plot_results, es, mc])
model = load_model(f'best_model_ens_{ens_no}.h5')
valresult = np.log(model.evaluate(x=X_test, y=y_test, verbose=0))
testresult = np.log(model.evaluate(x=X_test2, y=y_test2, verbose=0))
ens_val_array[ens_no] = valresult
ens_test_array[ens_no] = testresult
winner = ens_val_array.argmin()
model = load_model(f'best_model_ens_{winner}.h5')
info(ens_val_array)
info(ens_test_array)
info(f'Validation winner {winner}')
sample_count = len(X_test2[0])
y_pred = model.predict(X_test2)
plt.plot(np.log(history.history['loss']), label='loss')
plt.plot(np.log(history.history['val_loss']), label='test')
plt.grid(color='gray', linestyle='-', linewidth=1, alpha=0.2)
plt.legend()
plt.clf()
plt.show()
for i in range(5):
rand = np.random.randint(0, len(X_test[0]))
y_test_descaled = RNN_scaler.inverse_transform(y_test[rand,
np.newaxis])
y_in_descaled = RNN_scaler.inverse_transform(X_test[0][rand, :])
y_in_descaled = y_in_descaled.flatten()
y_test_descaled = y_test_descaled.flatten()
y_pred = model.predict(X_test)
y_pred_descaled = RNN_scaler.inverse_transform(y_pred[rand,
np.newaxis])
y_pred_descaled = y_pred_descaled.flatten()
plt.plot(y_test_descaled, label="true")
plt.plot(y_pred_descaled, label="predicted")
plt.title('Inclination delta')
#plt.ylim(0,1)
plt.legend()
plt.show()
plt.figure(figsize=(5, 4))
x_after = np.linspace(0, 23, 100)
x_before = np.linspace(-23, -0.23, 100)
plt.plot(x_before,
np.cumsum(y_in_descaled),
label="measured",
linestyle="-",
c="black")
commonpoint = np.cumsum(y_in_descaled)[-1]
plt.plot(x_after,
commonpoint + np.cumsum(y_test_descaled),
label="actual",
linestyle='-.',
c='black')
plt.plot(x_after,
commonpoint + np.cumsum(y_pred_descaled),
label="predicted",
linestyle=':',
c='black')
#plt.title('')
plt.ylim(-1, 7)
plt.grid()
plt.tight_layout()
#plt.hlines(0, -23, 23, linewidth=0.5)
plt.xlabel("Distance to sensor [m]")
plt.ylabel("Inclination, local coordinates, [deg]")
plt.legend()
plt.tight_layout()
plt.savefig(f'Sample, {percent_drilled}, no.{i}.pdf')
plt.show()
# #### Different ensemble, voting ######
# ypred_array = []
# for i in range(ensemble_count):
# model = load_model(f'best_model_ens_{i}.h5')
# y_pred = model.predict(X_test2)
# ypred_array.append(y_pred)
# y_pred = np.average(ypred_array, axis=0)
# ######## Different ensemble ends here #
y_test_descaled = RNN_scaler.inverse_transform(y_test2)
y_pred = model.predict(X_test2)
y_pred_descaled = RNN_scaler.inverse_transform(y_pred)
error_matrix = np.cumsum(y_pred_descaled, axis=1) - np.cumsum(
y_test_descaled, axis=1)
def rand_jitter(arr):
stdev = .004 * (max(arr) - min(arr))
return arr + np.random.randn(len(arr)) * stdev
def jitter(x,
y,
s=20,
c='b',
marker='o',
cmap=None,
norm=None,
vmin=None,
vmax=None,
alpha=None,
linewidths=None,
verts=None,
**kwargs):
return plt.scatter(rand_jitter(x),
rand_jitter(y),
s=s,
c=c,
marker=marker,
cmap=cmap,
norm=norm,
vmin=vmin,
vmax=vmax,
alpha=alpha,
linewidths=linewidths,
verts=verts,
**kwargs)
# =============================================================================
# plt.figure(figsize=(5,5), dpi=200)
# for i in range(sample_count):
# _ = jitter(x_after,error_matrix[i], alpha=1,s=0.5,marker=".", c="black")
# plt.title(f"delta, drilled {percent_drilled}")
# plt.xlabel("Distance to sensor [m]")
# plt.ylabel("Prediction error [deg]")
# plt.grid()
# plt.tight_layout()
# plt.savefig(f'Birdflock, {percent_drilled}.pdf')
# plt.show()
# #plt.plot(np.median(error_matrix, axis=0), linewidth=8, alpha=1, c="white")
# #plt.plot(np.median(error_matrix, axis=0), linewidth=2, alpha=1, c="black")
# plt.scatter(np.arange(0,100,1),np.average(np.abs(error_matrix), axis=0),
# marker="o",s=40, alpha=0.7, c="white", zorder=2)
#
# =============================================================================
c_array = np.empty(100, dtype=object)
aae = np.average(np.abs(error_matrix), axis=0)
# =============================================================================
# for i in range(100):
# if aae[i] <= 0.4:
# c_array[i] = "green"
# elif aae[i] <= 0.8:
# c_array[i] = "orange"
# else:
# c_array[i] = "red"
#
# plt.scatter(np.arange(0,100,1),aae, marker=".",s=20, alpha=1, c=c_array,
# zorder=3, label="Average Absolute Error")
# plt.ylim((-3,3))
# plt.axhline(y=0, xmin=0, xmax=1, linewidth=2, c="black")
# plt.axhline(y=0.4, xmin=0, xmax=1, linewidth=1, c="black")
# plt.axhline(y=0.8, xmin=0, xmax=1, linewidth=1, c="black")
# plt.legend()
# plt.show()
# =============================================================================
model = load_model('best_model.h5')
#mymanyplots(-1, data, model)
#myerrorplots(data, model)
valresult = np.log(model.evaluate(x=X_test, y=y_test, verbose=0))
testresult = np.log(model.evaluate(x=X_test2, y=y_test2, verbose=0))
return valresult, testresult, aae
test_x, test_y = test[:, :-1], test[:, -1]
# 为了在LSTM中应用该数据，需要将其格式转化为3D format，即[Samples, timesteps, features]
train_X = train_x.reshape((train_x.shape[0], 1, train_x.shape[1]))
test_X = test_x.reshape((test_x.shape[0], 1, test_x.shape[1]))
model = Sequential()
model.add(
Conv1D(filters=32,
kernel_size=3,
strides=1,
padding="causal",
activation="relu"))
model.add(
GRU(32,
input_shape=(train_X.shape[1], train_X.shape[2]),
return_sequences=True))
model.add(GRU(16, input_shape=(train_X.shape[1], train_X.shape[2])))
model.add(Dense(16, activation="relu"))
model.add(Dropout(0.2))
model.add(Dense(1))
model.compile(loss=tf.keras.losses.Huber(), optimizer='adam', metrics=["mse"])
history = model.fit(train_X,
train_y,
epochs=50,
batch_size=72,
validation_data=(test_X, test_y),
verbose=2)
# 画图
plt.plot(history.history['loss'], label='train')
trainset = dataset_scaled[:len(dataset_train)]
testset = dataset_scaled[-len(dataset_test)-60:]
x_train = []
y_train = []
for i in range(60,1259):
x_train.append(trainset[i-60:i, 0])
y_train.append(trainset[i,0])
x_train,y_train = np.array(x_train),np.array(y_train)
x_train = np.reshape(x_train, (x_train.shape[0],x_train.shape[1],1))
# building model
regressor = Sequential()
regressor.add(GRU(units = 50,return_sequences = True,input_shape = (x_train.shape[1],1)))
regressor.add(Dropout(0.2))
regressor.add(GRU(units = 50,return_sequences = True))
regressor.add(Dropout(0.2))
regressor.add(GRU(units = 50))
regressor.add(Dropout(0.2))
regressor.add(Dense(units = 1))
regressor.compile(optimizer = 'adam',loss = 'mean_squared_error')
regressor.fit(x_train,y_train,epochs = 100, batch_size = 32)
# predict in test set
x_test = []
for i in range(60,185):
x_test.append(testset[i-60:i,0])
def build_configurable_model(self):
if self.cell_type == "RNN":
# encoder
encoder_inputs = Input(shape=(None, len(self.srcChar2Int)))
encoder_outputs = encoder_inputs
for i in range(1, self.numEncoders + 1):
encoder = SimpleRNN(
self.latentDim,
return_state=True,
return_sequences=True,
dropout=self.dropout,
)
encoder_outputs, state = encoder(encoder_inputs)
encoder_states = [state]
# decoder
decoder_inputs = Input(shape=(None, len(self.tgtChar2Int)))
decoder_outputs = decoder_inputs
for i in range(1, self.numDecoders + 1):
decoder = SimpleRNN(
self.latentDim,
return_sequences=True,
return_state=True,
dropout=self.dropout,
)
decoder_outputs, _ = decoder(decoder_inputs, initial_state=encoder_states)
# dense
hidden = Dense(self.hidden, activation="relu")
hidden_outputs = hidden(decoder_outputs)
decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax")
decoder_outputs = decoder_dense(hidden_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
return model
elif self.cell_type == "LSTM":
# encoder
encoder_inputs = Input(shape=(None, len(self.srcChar2Int)))
encoder_outputs = encoder_inputs
for i in range(1, self.numEncoders + 1):
encoder = LSTM(
self.latentDim,
return_state=True,
return_sequences=True,
dropout=self.dropout,
)
encoder_outputs, state_h, state_c = encoder(encoder_outputs)
encoder_states = [state_h, state_c]
# decoder
decoder_inputs = Input(shape=(None, len(self.tgtChar2Int)))
decoder_outputs = decoder_inputs
for i in range(1, self.numDecoders + 1):
decoder = LSTM(
self.latentDim,
return_state=True,
return_sequences=True,
dropout=self.dropout,
)
decoder_outputs, _, _ = decoder(
decoder_outputs, initial_state=encoder_states
)
# dense
hidden = Dense(self.hidden, activation="relu")
hidden_outputs = hidden(decoder_outputs)
decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax")
decoder_outputs = decoder_dense(hidden_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
return model
elif self.cell_type == "GRU":
# encoder
encoder_inputs = Input(shape=(None, len(self.srcChar2Int)))
encoder_outputs = encoder_inputs
for i in range(1, self.numEncoders + 1):
encoder = GRU(
self.latentDim,
return_state=True,
return_sequences=True,
dropout=self.dropout,
)
encoder_outputs, state = encoder(encoder_inputs)
encoder_states = [state]
# decoder
decoder_inputs = Input(shape=(None, len(self.tgtChar2Int)))
decoder_outputs = decoder_inputs
for i in range(1, self.numDecoders + 1):
decoder = GRU(
self.latentDim,
return_sequences=True,
return_state=True,
dropout=self.dropout,
)
decoder_outputs, _ = decoder(decoder_inputs, initial_state=encoder_states)
# dense
hidden = Dense(self.hidden, activation="relu")
hidden_outputs = hidden(decoder_outputs)
decoder_dense = Dense(len(self.tgtChar2Int), activation="softmax")
decoder_outputs = decoder_dense(hidden_outputs)
model = Model([encoder_inputs, decoder_inputs], decoder_outputs)
return model
print(y_target.shape) # (1065, 2, 1) from sklearn.model_selection import train_test_split x1_train, x1_test, x2_train, x2_test, y_train, y_test = train_test_split(x1_data[:-1], x2_data[:-1], y_target, test_size=0.2, random_state=45) x1_train, x1_val, x2_train, x2_val, y_train, y_val = train_test_split(x1_train, x2_train, y_train, test_size=0.2, random_state=45) print(x1_train.shape, x1_test.shape, x1_val.shape) print(x2_train.shape, x2_test.shape, x2_val.shape) print(y_train.shape, y_test.shape, y_val.shape) # 모델 from tensorflow.keras.models import Model from tensorflow.keras.layers import Input, Dense, LSTM, GRU, Conv1D, MaxPooling1D, Flatten, Dropout, Concatenate input1 = Input(shape=(x1_data.shape[1], x1_data.shape[2])) layer1 = GRU(64)(input1) layer1 = Dropout(0.2)(layer1) layer1 = Dense(32)(layer1) input2 = Input(shape=(x2_data.shape[1], x2_data.shape[2])) layer2 = Conv1D(16, 3, padding='same', strides=1, activation='relu')(input2) layer2 = MaxPooling1D(2)(layer2) layer2 = Dropout(0.2)(layer2) layer2 = Flatten()(layer2) layer2 = Dense(16)(layer2) merge = Concatenate()([layer1, layer2]) merge = Dense(64, activation='relu')(merge) merge = Dense(64, activation='relu')(merge) output1 = Dense(2, activation='relu')(merge)
#读取数据集
train_x, test_x, train_y, test_y = util.load_data2('dataset_all_420.csv')
# 训练集
print(train_x.shape)
print(train_y.shape)
print(test_x.shape)
epochs = 1000
batch_size = 5
# GRU参数: return_sequences=True GRU输出为一个序列。默认为False，输出一个值。
# input_dim： 输入单个样本特征值的维度
# input_length： 输入的时间点长度
model = Sequential()
model.add(
GRU(units=10,
return_sequences=True,
input_dim=train_x.shape[-1],
input_length=train_x.shape[1]))
model.add(GRU(units=50))
model.add(Dense(1))
model.compile(loss='mean_squared_error', optimizer='adam')
model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, verbose=1)
y_pred = model.predict(test_x)
rms = np.sqrt(np.mean(np.power((test_y - y_pred), 2)))
print(rms)
print(y_pred.shape)
print(test_y.shape)
x_axis = np.arange(1, np.shape(test_y)[0] + 1)
plt.xlabel('Time')
plt.ylabel('Price')
x_input = np.array([50, 60, 70]) # (3, ) print(x.shape) print(y.shape) print(x_input.shape) x = x.reshape(13, 3, 1) print(x.shape) # y = y.reshape(13) # print(y.shape) # 2. 모델구성 from tensorflow.keras.models import Sequential from tensorflow.keras.layers import Dense, LSTM, SimpleRNN, GRU model = Sequential() model.add(GRU(200, activation='relu', input_shape=(3, 1))) model.add(Dense(200, activation='relu')) model.add(Dense(200, activation='relu')) model.add(Dense(200, activation='relu')) model.add(Dense(200, activation='relu')) model.add(Dense(200, activation='relu')) model.add(Dense(200, activation='relu')) model.add(Dense(200, activation='relu')) model.add(Dense(200, activation='relu')) model.add(Dense(1)) model.summary() # 3. 컴파일, 훈련 model.compile(loss='mse', optimizer='adam', metrics='mae') model.fit(x, y, epochs=200, batch_size=1)
def main(job_dir, **args):
gcp = args['gcp']
latent_unit_count = 512
EPOCHS = 500
BATCH_SIZE = 96
dropout_rate = 0.3
opt = 'adam'
output_dir = f"final"
history_name = f'training_history.json'
if not os.path.exists(output_dir):
os.makedirs(output_dir)
path = lambda x: os.path.join(output_dir, x)
data_x, data_y, dataset_encoder = load_data(job_dir=job_dir,
gcp=gcp,
shuffle_data=True,
augment=True,
mode="crop")
with open('encoder.pickle', 'wb') as f:
pickle.dump(dataset_encoder, f, pickle.HIGHEST_PROTOCOL)
sys.exit(0)
total_n_songs = data_x.shape[0]
validation_split = 0.2
# test_split = 0.1
# test_split_idx = int(total_n_songs * (1 - test_split))
# test_x, test_y = data_x[test_split_idx:], data_y[test_split_idx:]
# data_x, data_y = data_x[:test_split_idx], data_y[:test_split_idx]
validation_idx = int(total_n_songs * (1 - validation_split))
train_x, train_y = data_x[:validation_idx], data_y[:validation_idx]
val_data = data_x[validation_idx:], data_y[validation_idx:]
train_gen = make_generator(data_x, data_y, BATCH_SIZE)
val_gen = make_generator(val_data[0], val_data[1], BATCH_SIZE)
n_songs, song_len, n_notes = train_x.shape
inputs = Input(shape=(song_len, n_notes), name="melody_input")
input_encoder = GRU(
units=latent_unit_count,
return_sequences=True,
return_state=True,
name="melody_encoder",
dropout=dropout_rate,
)
x, input_state = input_encoder(inputs)
alto_encoder = GRU(units=latent_unit_count,
return_sequences=True,
return_state=True,
name="alto_encoder",
dropout=dropout_rate)
tenor_encoder = GRU(units=latent_unit_count,
return_sequences=True,
return_state=True,
name="tenor_encoder",
dropout=dropout_rate)
bass_encoder = GRU(units=latent_unit_count,
return_sequences=True,
return_state=True,
name="bass_encoder",
dropout=dropout_rate)
a, a_state = alto_encoder(x, initial_state=input_state)
t, t_state = tenor_encoder(x, initial_state=input_state)
b, b_state = bass_encoder(x, initial_state=input_state)
alto_decoder = GRU(units=latent_unit_count,
return_sequences=True,
name="alto_decoder",
dropout=dropout_rate)
tenor_decoder = GRU(
units=latent_unit_count,
return_sequences=True,
name="tenor_decoder",
dropout=dropout_rate,
)
bass_decoder = GRU(
units=latent_unit_count,
return_sequences=True,
name="bass_decoder",
dropout=dropout_rate,
)
a = alto_decoder(a, initial_state=a_state)
t = tenor_decoder(t, initial_state=t_state)
b = bass_decoder(b, initial_state=b_state)
reshaper = Reshape((song_len, 1, latent_unit_count))
a = reshaper(a)
t = reshaper(t)
b = reshaper(b)
x = Concatenate(axis=2)([a, t, b])
merger = Dense(n_notes, name="fc_output")
x = merger(x)
outputs = Activation("softmax", name="softmax")(x)
model = Model(inputs=inputs, outputs=outputs, name="BachNet")
model.compile(loss="categorical_crossentropy", optimizer=opt)
model.summary()
try:
tf.keras.utils.plot_model(
model,
to_file=path("model.png"),
dpi=1200,
show_shapes=True,
show_layer_names=True,
rankdir="TB",
)
except ImportError:
print('Graphviz/pydot not installed, skipping model plot...')
sys.exit(0)
callbacks = [
tf.keras.callbacks.EarlyStopping(patience=3,
restore_best_weights=True,
min_delta=0,
monitor='val_loss'),
# tf.keras.callbacks.ModelCheckpoint(
# filepath=path("model.{epoch:02d}-{val_loss:.2f}.h5"),
# monitor="val_loss",
# save_best_only=True,
# ),
# tf.keras.callbacks.TensorBoard(log_dir="./logs"),
]
hist = model.fit(
train_gen,
validation_data=val_data,
# validation_split=validation_split,
steps_per_epoch=n_songs // BATCH_SIZE,
# batch_size=BATCH_SIZE,
epochs=EPOCHS,
callbacks=callbacks,
# shuffle=True,
)
if gcp:
# save the model on to google cloud storage
save_path = path('final_model.h5')
model.save(save_path)
# model.save_weights(path)
with file_io.FileIO(save_path, mode='rb') as in_f:
with file_io.FileIO(os.path.join(job_dir, save_path),
mode='wb+') as of:
of.write(in_f.read())
else:
import matplotlib.pyplot as plt
model.save(path("final_model.h5"))
plt.plot(hist.history["loss"], label="Training loss")
plt.plot(hist.history["val_loss"], label="Validation loss")
plt.xlabel('Epoch')
plt.ylabel('Cross-entropy loss')
plt.legend()
plt.savefig(path("history.png"), dpi=300)
with open(path(history_name), "w") as f:
json.dump(
{
"loss": hist.history["loss"],
},
f #"val_loss": hist.history["val_loss"]}, f
)
plt.show()
'helper function to create x, c and y with proper shapes'
x = data.filter(like='x-', axis=1).values[:, :, np.newaxis]
c = data[top_cities[1:]].to_numpy()
y = data.Amsterdam.values[:, np.newaxis]
return x, c, y
# create correct shapes for tensorflow
x_train, c_train, y_train = create_xy(train)
x_test, c_test, y_test = create_xy(test)
# deterministic
set_seed(random_state)
# As before, I start out by a pure autoregressive model.
model = Sequential(layers=[GRU(cells), Dense(units=1, activation='linear')])
model.compile(optimizer='adam', loss='mae')
history = model.fit(x=x_train, y=y_train, epochs=epochs, batch_size=None,
shuffle=True,
validation_split=validation_split)
# The final test loss is;
def inverseAms(data):
return (ct.named_transformers_['Amsterdam']
.inverse_transform(data)
)
modelmae = mean_absolute_error(inverseAms(model.predict(x_test)),
def _crnn_layers(self, input_shape, output_shape):
channel_axis = 3
melgram_input = Input(shape=input_shape, dtype='float32')
# Input block
padding = self.network_input_width - input_shape[1]
left_pad = int(padding / 2)
if padding % 2:
right_pad = left_pad + 1
else:
right_pad = left_pad
input_padding = ((0, 0), (left_pad, right_pad))
hidden = ZeroPadding2D(padding=input_padding)(melgram_input)
# Conv block 1
hidden = Conv2D(64, (3, 3), padding=self.padding, name='conv1')(hidden)
hidden = BatchNormalization(axis=channel_axis, name='bn1')(hidden)
hidden = Activation('elu', name='elu-1')(hidden)
hidden = MaxPooling2D(pool_size=(2, 2), strides=(2, 2),
name='pool1')(hidden)
hidden = Dropout(0.1, name='dropout1')(hidden)
# Conv block 2
hidden = Conv2D(128, (3, 3), padding=self.padding,
name='conv2')(hidden)
hidden = BatchNormalization(axis=channel_axis, name='bn2')(hidden)
hidden = Activation('elu', name='elu-2')(hidden)
hidden = MaxPooling2D(pool_size=(3, 3), strides=(3, 3),
name='pool2')(hidden)
hidden = Dropout(0.1, name='dropout2')(hidden)
# Conv block 3
hidden = Conv2D(128, (3, 3), padding=self.padding,
name='conv3')(hidden)
hidden = BatchNormalization(axis=channel_axis, name='bn3')(hidden)
hidden = Activation('elu', name='elu-3')(hidden)
hidden = MaxPooling2D(pool_size=(4, 4), strides=(4, 4),
name='pool3')(hidden)
hidden = Dropout(0.1, name='dropout3')(hidden)
# Conv block 4
hidden = Conv2D(128, (3, 3), padding=self.padding,
name='conv4')(hidden)
hidden = BatchNormalization(axis=channel_axis, name='bn4')(hidden)
hidden = Activation('elu', name='elu-4')(hidden)
hidden = MaxPooling2D(pool_size=(4, 4), strides=(4, 4),
name='pool4')(hidden)
hidden = Dropout(0.1, name='dropout4')(hidden)
# reshaping
hidden = Reshape((15, 128))(hidden)
# GRU block 1, 2, output
embed_size = 32
hidden = GRU(embed_size, return_sequences=True, name='gru1')(hidden)
hidden = GRU(embed_size, return_sequences=self.attention,
name='gru2')(hidden)
if self.attention:
attention = Dense(1)(hidden)
attention = Flatten()(attention)
attention_act = Activation('softmax')(attention)
attention = RepeatVector(embed_size)(attention_act)
attention = Permute((2, 1))(attention)
merged = Multiply()([hidden, attention])
hidden = Lambda(lambda xin: K.sum(xin, axis=1))(merged)
if self.output_dropout:
hidden = Dropout(self.output_dropout)(hidden)
output = Dense(output_shape, activation='sigmoid',
name='crnn_output')(hidden)
return melgram_input, output
y = []
print(len(data))
print("The last sample would be: " + str(len(data)) + " - " + str(T) + " = " +
str((len(data) - T)))
for i in range(len(data) - T):
x = data[i:i + T]
X.append(x)
y_temp = data[i + T]
y.append(y_temp)
X = np.array(X).reshape(-1, T, D)
y = np.array(y)
i_layer = Input(shape=(T, D))
h_layer = GRU(10, activation='tanh')(i_layer)
o_layer = Dense(1)(h_layer)
model = Model(i_layer, o_layer)
model.compile(loss='mse', optimizer=Adam(lr=0.2))
index = -N // 4
report = model.fit(X[:index],
y[:index],
epochs=50,
validation_data=(X[index:], y[index:]))
plt.plot(report.history['loss'], label='training_loss')
plt.plot(report.history['val_loss'], label='validation_loss')
plt.legend()
y_test = y[index:]
