import pandas as pd
import preprocessing
import config
import numpy as np
from sklearn.model_selection import train_test_split
import time
yyyymm_list = preprocessing.get_yyyymm_list()
sido_nm_list = preprocessing.get_sido_nm_list()
features_nm_list = preprocessing.get_fetures_nm_list()
def load_data_set(percentage, random_state, mode=1):
start_time = time.time()
df_61 = pd.read_csv('data\\kab61.csv')
df_71 = pd.read_csv('data\\kab71.csv')
df_61_filtered = df_61[['AreaName', 'YYYYMMDD', 'InfoType2', 'Values']]
df_71_filtered = df_71[['AreaName', 'YYYYMMDD', 'InfoType2', 'Values']]
df_x = df_61_filtered.append(df_71_filtered)
df_x = df_x.assign(YYYYMM=lambda x: x.YYYYMMDD.str[0:7])
df_x = df_x[['AreaName', 'YYYYMM', 'InfoType2', 'Values']]
df_y = pd.read_csv('data\\y.txt', delimiter='\t', encoding='MS949')
n_acceptable_yyyymm = len(yyyymm_list)-config.N_MONTH_TO_PREDICT-config.N_TIME_WINDOW+2
l_training_feature = np.zeros([n_acceptable_yyyymm * len(sido_nm_list), config.N_TIME_WINDOW, config.N_FEATURES + 1])
l_training_label = np.zeros([n_acceptable_yyyymm * len(sido_nm_list)])
idx_yyyymm_inserting = 0
for idx_yyyymm, yyyymm in enumerate(yyyymm_list):
d1 = Dense(32)(d_features)
d2 = Dense(32)(d1)
d3 = Dense(32)(d2)
output = Dense(3, activation='softmax')(d3)
model = Model(inputs=d_features, outputs=output)
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
# training
model.fit([
np.array(X_train[:, :, 0:config.N_FEATURES]),
np.array([
m[preprocessing.get_sido_nm_list()[int(r)]]
for r in X_train[:, 0, config.N_FEATURES]
])
],
np.array(keras.utils.to_categorical(y_train, 3)),
epochs=20,
batch_size=10)
# save weight
model.save_weights('{}.hdf5'.format(MODEL_FILE_NAME))
# finished and predict using training model above!!
score = model.evaluate([
np.array(X_test[:, :, 0:config.N_FEATURES]),
np.array([
m[preprocessing.get_sido_nm_list()[int(r)]]
d_features = Input(shape=(config.N_TIME_WINDOW, config.N_FEATURES), name="features")
# d_sido = Input(shape=(len(preprocessing.get_sido_nm_list()),), name="sido_onehot")
d1 = Dense(32)(d_features)
d2 = Dense(32)(d1)
d3 = Dense(32)(d2)
output = Dense(3, activation='softmax')(d3)
model = Model(inputs=d_features,outputs=output)
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
# training
model.fit(
[np.array(X_train[:,:,0:config.N_FEATURES]), np.array([m[preprocessing.get_sido_nm_list()[int(r)]] for r in X_train[:,0,config.N_FEATURES]]) ],
np.array(keras.utils.to_categorical(y_train, 3)), epochs=20, batch_size=10)
# save weight
model.save_weights('{}.hdf5'.format(MODEL_FILE_NAME))
# finished and predict using training model above!!
score = model.evaluate(
[np.array(X_test[:,:,0:config.N_FEATURES]), np.array([m[preprocessing.get_sido_nm_list()[int(r)]] for r in X_test[:,0,config.N_FEATURES]]) ],
np.array(keras.utils.to_categorical(y_test, 3)), verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
else:
X_train, X_test, y_train, y_test = util.load_data_set(30, 33)
m = preprocessing.get_sido_onehot_map()
f = open('{}.pickle'.format(config.PICKLE_FILE_NAME), 'wb')
pickle.dump([X_train, X_test, y_train, y_test, m], f)
print('finished to dump pickle file from disk')
"""
merge two other neural networks
https://statcompute.wordpress.com/2017/01/08/an-example-of-merge-layer-in-keras/
https://nhanitvn.wordpress.com/2016/09/27/a-keras-layer-for-one-hot-encoding/
"""
# d_features = Input(shape=(1, config.N_FEATURES, config.N_TIME_WINDOW), name="features")
# d_sido = Input(shape=(len(preprocessing.get_sido_nm_list()), ), name="sido_onehot")
d_features = Input(shape=(config.N_TIME_WINDOW, config.N_FEATURES),
name="features")
d_sido = Input(shape=(len(preprocessing.get_sido_nm_list()), ),
name="sido_onehot")
l1 = Conv1D(32, kernel_size=(12), activation='relu')(d_features)
l2 = Conv1D(64, kernel_size=(6), activation='relu')(l1)
l2_flat = Flatten()(l2)
# d_concatenated = Concatenate([l2_flat, d_sido])
# output = Dense(3, activation='softmax')(d_concatenated)
output = Dense(3, activation='softmax')(l2_flat)
model = Model(inputs=[d_features, d_sido], outputs=output)
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),
metrics=['accuracy'])
else:
X_train, X_test, y_train, y_test = util.load_data_set(30, 33)
m = preprocessing.get_sido_onehot_map()
f = open('{}.pickle'.format(PICKLE_FILE_NAME), 'wb')
pickle.dump([X_train, X_test, y_train, y_test, m], f)
print('finished to dump pickle file from disk')
"""
merge two other neural networks
https://statcompute.wordpress.com/2017/01/08/an-example-of-merge-layer-in-keras/
https://nhanitvn.wordpress.com/2016/09/27/a-keras-layer-for-one-hot-encoding/
"""
# d_features = Input(shape=(1, config.N_FEATURES, config.N_TIME_WINDOW), name="features")
# d_sido = Input(shape=(len(preprocessing.get_sido_nm_list()), ), name="sido_onehot")
d_features = Input(shape=(config.N_TIME_WINDOW, config.N_FEATURES), name="features")
d_sido = Input(shape=(len(preprocessing.get_sido_nm_list()),), name="sido_onehot")
l1 = Conv1D(32, kernel_size=(12), activation='relu')(d_features)
l2 = Conv1D(64, kernel_size=(6), activation='relu')(l1)
l2_flat = Flatten()(l2)
d_concatenated = Concatenate(axis=1)([l2_flat, d_sido])
d_1 = Dense(200)(d_concatenated)
d_2 = Dense(100)(d_1)
output = Dense(3, activation='softmax')(d_2)
# output = Dense(3, activation='softmax')(l2_flat)
model = Model(inputs=[d_features,d_sido],outputs=output)
model.compile(loss=keras.losses.categorical_crossentropy,
optimizer=keras.optimizers.Adadelta(),