def joint_learning():
X2 = fun2('number_age_col85tran_v2.csv')
print(X2.shape)
y = fun2('number_category.csv')
nb_x_train = fun3(f_in='nb_x_train_%d.dat' % (data_version))
nb_x_test = fun3(f_in='nb_x_test_%d.dat' % (data_version))
X_train, X_test, y_train, y_test, = get_train_test_data(
X2,
y,
nb_x_train,
nb_x_test,
)
temp_1_10 = fun2('front_1_10_temp.csv')
temp_2_11 = fun2('front_2_11_temp.csv')
temp_1_10_train, temp_1_10_test, temp_2_11_train, temp_2_11_test = get_train_test_data(
temp_1_10, temp_2_11, nb_x_train, nb_x_test)
temp_1_10_train = reshape_dataset(temp_1_10_train)
temp_2_11_train = reshape_dataset(temp_2_11_train)
temp_1_10_test = reshape_dataset(temp_1_10_test)
temp_2_11_test = reshape_dataset(temp_2_11_test)
print(temp_1_10_train.shape)
# x_train represents list temperature
# x2_train represents test parameter
x_train = X_train[:, in_file_length + 1:]
x2_train = X_train[:, 0:in_file_length]
x_test = X_test[:, in_file_length + 1:]
x2_test = X_test[:, 0:in_file_length]
print((sum(y_test) - len(y_test)) / len(y_test))
y_train = category_to_target(y_train)
y_test = category_to_target(y_test)
model = joint_learning_model()
mse_list = []
acc_list = []
for epoch in range(nb_epochs):
print('Train...')
model.fit([temp_1_10_train, x_train, x2_train],
[temp_2_11_train, y_train],
batch_size=batch_size,
nb_epoch=1,
shuffle=True)
loss, mse, acc = model.evaluate([temp_1_10_test, x_test, x2_test],
[temp_2_11_test, y_test],
batch_size=batch_size)
print('loss:', loss)
print('mse:', mse)
print('acc:', acc)
def dense_temp(f_in):
X = diff_length_dat(f_in)
X = np.array(X, dtype=float)
y = fun2('number_category.csv')
x_train, x_test, y_train, y_test = cross_validation.train_test_split(
X, y, test_size=0.2)
y_train = category_to_target(y_train)
y_test = category_to_target(y_test)
model = Sequential()
model.add(Dense(nb_classes, input_shape=(15, )))
model.add(Dropout(0.1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
acc_list = []
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
acc_list.append(acc)
for epoch in range(nb_epochs):
print('Train...')
model.fit(x_train,
y_train,
batch_size=batch_size,
nb_epoch=1,
validation_split=0.05)
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
acc_list.append(acc)
# print(model.predict_proba(X))
print('\nTest score:', score)
print('Test accuracy:', acc)
plot(range(0, nb_epochs + 1), acc_list)
show()
def pure_temp_prediction():
X = diff_length_csv('tiwencheck.csv')
X = np.array(X)
y = fun2('number_category.csv')
X_train, X_test, y_train, y_test = cross_validation.train_test_split(
X, y, test_size=0.2)
y_train = category_to_target(y_train)
y_test = category_to_target(y_test)
x_train = reshape_dataset(X_train)
x_test = reshape_dataset(X_test)
model = Sequential()
model.add(LSTM(16, input_shape=(12, 1)))
model.add(Dropout(0.25))
model.add(Dense(nb_classes, bias=False))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
print('Train...')
model.fit(x_train,
y_train,
batch_size=batch_size,
nb_epoch=5,
validation_split=0.05)
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
print(model.predict_proba(reshape_dataset(X)))
print('Test score:', score)
print('Test accuracy:', acc)
def para_prediction(f_x='number_age_col85tran_v2.csv',in_file_length=87):
X = fun2(f_x)
y = fun2('number_category.csv')
nb_x_train = fun3(f_in='nb_x_train_%d.dat' % data_version)
nb_x_test = fun3(f_in='nb_x_test_%d.dat' % data_version)
x_train, x_test, y_train, y_test = get_train_test_data(X, y, nb_x_train, nb_x_test, )
probability_test = (sum(y_test) - len(y_test)) / len(y_test)
print('probability_test:', probability_test)
y_train = category_to_target(y_train)
y_test = category_to_target(y_test)
# get_model
model = get_model(in_file_length)
print('Train...')
acc_list = []
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
acc_list.append(acc)
for epoch in range(nb_epochs):
# print('Train...')
model.fit(x_train, y_train, batch_size=batch_size, nb_epoch=1, validation_split=0.05)
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
acc_list.append(acc)
print('Test score:', score)
print('Test accuracy:', acc)
results = ""
for i, acc in enumerate(acc_list):
if acc > 0.72:
if acc > 0.74:
results += '\033[1;31m' + str(i + 1) + ':' + str(acc) + '\033[0m' + '; '
else:
results += '\033[1;34m' + str(i + 1) + ':' + str(acc) + '\033[0m' + '; '
else:
results += str(i + 1) + ':' + str(acc) + '; '
print(results)
subplot(2,3,data_version)
title('para_prediction_dataset_%d'%(data_version))
plot(range(0, nb_epochs + 1), acc_list,label='length:%d'%(in_file_length))
acc_list = sorted(acc_list, reverse=True)
print(acc_list)
print("top-K mean: %.3f" % np.mean(np.array(acc_list[:10])))
def temp_lstm(f_in='5_day_50_check.csv'):
if f_in[-1] == 't':
X = diff_length_dat(f_in)
else:
X = diff_length_csv(f_in)
X = pad_sequences(X,
maxlen=maxlen,
padding='post',
truncating='post',
value=0,
dtype=float)
X = np.array(X, dtype=float)
# print(X[0])
y = fun2('number_category.csv')
nb_x_train = fun3(f_in='nb_x_train_%d.dat' % (data_version))
nb_x_test = fun3(f_in='nb_x_test_%d.dat' % (data_version))
x_train, x_test, y_train, y_test = get_train_test_data(
X, y, nb_x_train, nb_x_test)
probability_test = (sum(y_test) - len(y_test)) / len(y_test)
print('probability_test:', probability_test)
y_train = category_to_target(y_train)
y_test = category_to_target(y_test)
#get_model
model = conv_lstm_model()
#evaluation
acc_list = []
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
acc_list.append(acc)
print('Train...')
for epoch in range(nb_epochs):
model.fit(x_train,
y_train,
batch_size=batch_size,
verbose=0,
nb_epoch=1,
shuffle=True)
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
acc_list.append(acc)
print('Test score:', score)
print('Test accuracy:', acc)
subplot(2, 3, data_version)
plot(range(0, nb_epochs + 1),
[probability_test for i in range(0, nb_epochs + 1)])
plot(range(0, nb_epochs + 1), acc_list)
acc_list = sorted(acc_list, reverse=True)
title('temp_pre_conv_dataset_%d' % (data_version))
print("top-K mean: %.3f" % np.mean(np.array(acc_list[:10])))
def temp(f_in='5_day_50_check.csv'):
if f_in[-1] == 't':
X = diff_length_dat(f_in)
else:
X = diff_length_csv(f_in)
X = np.array(X, dtype=float)
print(X.shape)
y = fun2('number_category.csv')
x_train, x_test, y_train, y_test = cross_validation.train_test_split(
X, y, test_size=0.2)
y_train = category_to_target(y_train)
y_test = category_to_target(y_test)
model = Sequential()
model.add(
Reshape(input_shape=(temp_length, ), target_shape=(temp_length, 1)))
model.add(
Convolution1D(border_mode='same', filter_length=2,
nb_filter=nb_filter))
model.add(Reshape(target_shape=(temp_length * nb_filter, )))
model.add(Dense(32))
model.add(Dropout(0.5))
model.add(Activation('relu'))
model.add(Dense(nb_classes))
model.add(Activation('sigmoid'))
# model2=Sequential()
# model2.add(Dense(input_shape=(50,),output_dim=3))
# model2.add(Merge([model,model2]))
model.compile(loss='binary_crossentropy',
optimizer=Adam(lr=0.001),
metrics=['accuracy'])
model.summary()
for epoch in range(nb_epochs):
print('Train...')
model.fit(x_train,
y_train,
batch_size=batch_size,
nb_epoch=1,
validation_split=0.05)
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
# print(model.predict_proba(X))
print('\nTest score:', score)
print('Test accuracy:', acc)
def temp_lstm_2(f_in='5_day_50_check.csv'):
if f_in[-1] == 't':
X = diff_length_dat(f_in)
else:
X = diff_length_csv(f_in)
X = np.array(X, dtype=float)
print(X.shape)
y = fun2('number_category.csv')
y = y - 1
x_train, x_test, y_train, y_test = cross_validation.train_test_split(
X, y, test_size=0.2)
model = Sequential()
model.add(
Reshape(input_shape=(temp_length, ), target_shape=(temp_length, 1)))
model.add(LSTM(16, ))
model.add(Dense(32, ))
model.add(Dropout(0.4))
model.add(Activation('relu'))
model.add(Dense(1))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer=Adam(lr=0.002),
metrics=['accuracy'])
model.summary()
acc_list = []
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
acc_list.append(acc)
for epoch in range(nb_epochs):
print('Train...')
model.fit(x_train,
y_train,
batch_size=batch_size,
nb_epoch=1,
validation_split=0.05)
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
acc_list.append(acc)
print(model.predict_classes(X))
# print(model.predict_proba(X))
print('\nTest score:', score)
print('Test accuracy:', acc)
plot(range(0, nb_epochs + 1), acc_list)
show()
def length_temp_prediction():
X = diff_length_csv('5_day_50_check.csv')
# X=fun2('5_day_50_check.csv')
# X = pad_sequences(X, maxlen=maxlen, padding='post', truncating='post', dtype='float')
X = np.array(X, dtype=float)
y = fun2('number_category.csv')
X_train, X_test, y_train, y_test = cross_validation.train_test_split(
X, y, test_size=0.2)
y_train = category_to_target(y_train)
y_test = category_to_target(y_test)
x_train = reshape_dataset(X_train)
x_test = reshape_dataset(X_test)
model = Sequential()
# model.add(Masking(mask_value=0, input_shape=(maxlen, 1)))
model.add(LSTM(3, input_shape=(maxlen, 1), return_sequences=True))
model.add(Dropout(0.25))
model.add(Reshape((maxlen * 3, )))
model.add(Dense(nb_classes, b_regularizer=l1(0.01)))
model.add(Dropout(0.25))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.summary()
print('Train...')
model.fit(x_train,
y_train,
batch_size=batch_size,
nb_epoch=30,
validation_split=0.05)
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
print(model.predict_proba(reshape_dataset(X)))
print('Test score:', score)
print('Test accuracy:', acc)
def fft_prediction(f_in='5_day_50_check.csv'):
if f_in[-1] == 't':
X = diff_length_dat(f_in)
else:
X = diff_length_csv(f_in)
X = np.array(X, dtype=float)
y = fun2('number_category.csv')
nb_x_train = fun3(f_in='nb_train_%d.dat' % (data_version))
nb_x_test = fun3(f_in='nb_test_%d.dat' % (data_version))
nb_x_validation = fun3(f_in='nb_validation_%d.dat' % (data_version))
x_train, x_test, x_validation, y_train, y_test, y_validation = get_train_validation_test_data(
X, y, nb_x_train, nb_x_test, nb_x_validation)
x_train, x_test, x_validation = fft_function(x_train, x_test, x_validation)
probability_validation = (sum(y_validation) -
len(y_validation)) / len(y_validation)
probability_test = (sum(y_test) - len(y_test)) / len(y_test)
print('probability_validation:', probability_validation)
y_train = category_to_target(y_train)
y_test = category_to_target(y_test)
y_validation = category_to_target(y_validation)
#get_model
model = fft_model()
#evaluation
best_epoch = 0
best_acc = probability_validation
acc_list = []
score, acc = model.evaluate(x_validation,
y_validation,
batch_size=batch_size)
acc_list.append(acc)
print('Train...')
for epoch in range(nb_epochs):
model.fit(x_train,
y_train,
batch_size=batch_size,
verbose=0,
nb_epoch=1,
shuffle=True)
score, acc = model.evaluate(x_validation,
y_validation,
batch_size=batch_size)
acc_list.append(acc)
print('Test score:', score)
print('Test accuracy:', acc)
if acc > best_acc:
best_acc = acc
model.save_weights('temp_dataset_%d_epoch_%d.h5' %
(data_version, epoch))
if best_epoch != 0:
os.remove('temp_dataset_%d_epoch_%d.h5' %
(data_version, best_epoch))
best_epoch = epoch
plot(range(0, nb_epochs + 1), acc_list)
plot(range(0, nb_epochs + 1),
[probability_validation for i in range(0, nb_epochs + 1)])
savefig('fft_temp_dataset_%d.png' % (data_version))
show()
print(best_epoch)
print(best_acc)
#test
model.load_weights('temp_dataset_%d_epoch_%d.h5' %
(data_version, best_epoch))
score, acc = model.evaluate(x_test, y_test, batch_size=batch_size)
print('Test score:', score)
print('probabitity_test:', probability_test)
print('Test accuracy:', acc)
loss_weights={
'dense_softmax': 0.1,
'dense_temp': 1
})
model.summary()
return model
if __name__ == '__main__':
model = merge_predict_next_temp_model()
temp_x, temp_y = get_train_fun()
# X = fun2('number_age_col85tran_v2.csv')
# X = fun2('number_age_col71tran_v2.csv')
X = fun2('cap_feature_2.csv')
print(X.shape)
y = fun2('number_category.csv')
nb_x_train = fun3(f_in='nb_x_train_%d.dat' % (data_version))
nb_x_test = fun3(f_in='nb_x_test_%d.dat' % (data_version))
X_train, X_test, y_train, y_test, = get_train_test_data(
X,
y,
nb_x_train,
nb_x_test,
)
print(X_train.shape)
y_train = category_to_target(y_train)
y_test = category_to_target(y_test)
))
# model.add(Dense(nb_classes,W_regularizer=l2(0.01)))
# model.add(Dense(16, activation='relu'))
# model.add(Dropout(0.1))
# model.add(Dense(2))
# model.add(Dense(nb_classes,input_dim=in_file_length,W_regularizer=l2(0.01), b_regularizer=l2(0.01)))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=0.0005),
metrics=['accuracy'])
model.summary()
return model
if __name__ == '__main__':
X = fun2('number_age_col85tran.csv')
y = fun2('number_category.csv')
nb_x_train = fun3(f_in='nb_train_%d.dat' % data_version)
nb_x_test = fun3(f_in='nb_test_%d.dat' % data_version)
nb_x_validation = fun3(f_in='nb_validation_%d.dat' % (data_version))
x_train, x_test, x_validation, y_train, y_test, y_validation = get_train_validation_test_data(
X, y, nb_x_train, nb_x_test, nb_x_validation)
probability_validation = (sum(y_validation) -
len(y_validation)) / len(y_validation)
probability_test = (sum(y_test) - len(y_test)) / len(y_test)
print('probability_validation:', probability_validation)
print('probability_test:', probability_test)
y_train = category_to_target(y_train)
y_test = category_to_target(y_test)
y_validation = category_to_target(y_validation)