import pandas as pd import preprocessing import config import numpy as np from sklearn.model_selection import train_test_split import time yyyymm_list = preprocessing.get_yyyymm_list() sido_nm_list = preprocessing.get_sido_nm_list() features_nm_list = preprocessing.get_fetures_nm_list() def load_data_set(percentage, random_state, mode=1): start_time = time.time() df_61 = pd.read_csv('data\\kab61.csv') df_71 = pd.read_csv('data\\kab71.csv') df_61_filtered = df_61[['AreaName', 'YYYYMMDD', 'InfoType2', 'Values']] df_71_filtered = df_71[['AreaName', 'YYYYMMDD', 'InfoType2', 'Values']] df_x = df_61_filtered.append(df_71_filtered) df_x = df_x.assign(YYYYMM=lambda x: x.YYYYMMDD.str[0:7]) df_x = df_x[['AreaName', 'YYYYMM', 'InfoType2', 'Values']] df_y = pd.read_csv('data\\y.txt', delimiter='\t', encoding='MS949') n_acceptable_yyyymm = len(yyyymm_list)-config.N_MONTH_TO_PREDICT-config.N_TIME_WINDOW+2 l_training_feature = np.zeros([n_acceptable_yyyymm * len(sido_nm_list), config.N_TIME_WINDOW, config.N_FEATURES + 1]) l_training_label = np.zeros([n_acceptable_yyyymm * len(sido_nm_list)]) idx_yyyymm_inserting = 0 for idx_yyyymm, yyyymm in enumerate(yyyymm_list):
d1 = Dense(32)(d_features) d2 = Dense(32)(d1) d3 = Dense(32)(d2) output = Dense(3, activation='softmax')(d3) model = Model(inputs=d_features, outputs=output) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy']) # training model.fit([ np.array(X_train[:, :, 0:config.N_FEATURES]), np.array([ m[preprocessing.get_sido_nm_list()[int(r)]] for r in X_train[:, 0, config.N_FEATURES] ]) ], np.array(keras.utils.to_categorical(y_train, 3)), epochs=20, batch_size=10) # save weight model.save_weights('{}.hdf5'.format(MODEL_FILE_NAME)) # finished and predict using training model above!! score = model.evaluate([ np.array(X_test[:, :, 0:config.N_FEATURES]), np.array([ m[preprocessing.get_sido_nm_list()[int(r)]]
d_features = Input(shape=(config.N_TIME_WINDOW, config.N_FEATURES), name="features") # d_sido = Input(shape=(len(preprocessing.get_sido_nm_list()),), name="sido_onehot") d1 = Dense(32)(d_features) d2 = Dense(32)(d1) d3 = Dense(32)(d2) output = Dense(3, activation='softmax')(d3) model = Model(inputs=d_features,outputs=output) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy']) # training model.fit( [np.array(X_train[:,:,0:config.N_FEATURES]), np.array([m[preprocessing.get_sido_nm_list()[int(r)]] for r in X_train[:,0,config.N_FEATURES]]) ], np.array(keras.utils.to_categorical(y_train, 3)), epochs=20, batch_size=10) # save weight model.save_weights('{}.hdf5'.format(MODEL_FILE_NAME)) # finished and predict using training model above!! score = model.evaluate( [np.array(X_test[:,:,0:config.N_FEATURES]), np.array([m[preprocessing.get_sido_nm_list()[int(r)]] for r in X_test[:,0,config.N_FEATURES]]) ], np.array(keras.utils.to_categorical(y_test, 3)), verbose=0) print('Test loss:', score[0]) print('Test accuracy:', score[1])
else: X_train, X_test, y_train, y_test = util.load_data_set(30, 33) m = preprocessing.get_sido_onehot_map() f = open('{}.pickle'.format(config.PICKLE_FILE_NAME), 'wb') pickle.dump([X_train, X_test, y_train, y_test, m], f) print('finished to dump pickle file from disk') """ merge two other neural networks https://statcompute.wordpress.com/2017/01/08/an-example-of-merge-layer-in-keras/ https://nhanitvn.wordpress.com/2016/09/27/a-keras-layer-for-one-hot-encoding/ """ # d_features = Input(shape=(1, config.N_FEATURES, config.N_TIME_WINDOW), name="features") # d_sido = Input(shape=(len(preprocessing.get_sido_nm_list()), ), name="sido_onehot") d_features = Input(shape=(config.N_TIME_WINDOW, config.N_FEATURES), name="features") d_sido = Input(shape=(len(preprocessing.get_sido_nm_list()), ), name="sido_onehot") l1 = Conv1D(32, kernel_size=(12), activation='relu')(d_features) l2 = Conv1D(64, kernel_size=(6), activation='relu')(l1) l2_flat = Flatten()(l2) # d_concatenated = Concatenate([l2_flat, d_sido]) # output = Dense(3, activation='softmax')(d_concatenated) output = Dense(3, activation='softmax')(l2_flat) model = Model(inputs=[d_features, d_sido], outputs=output) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(), metrics=['accuracy'])
else: X_train, X_test, y_train, y_test = util.load_data_set(30, 33) m = preprocessing.get_sido_onehot_map() f = open('{}.pickle'.format(PICKLE_FILE_NAME), 'wb') pickle.dump([X_train, X_test, y_train, y_test, m], f) print('finished to dump pickle file from disk') """ merge two other neural networks https://statcompute.wordpress.com/2017/01/08/an-example-of-merge-layer-in-keras/ https://nhanitvn.wordpress.com/2016/09/27/a-keras-layer-for-one-hot-encoding/ """ # d_features = Input(shape=(1, config.N_FEATURES, config.N_TIME_WINDOW), name="features") # d_sido = Input(shape=(len(preprocessing.get_sido_nm_list()), ), name="sido_onehot") d_features = Input(shape=(config.N_TIME_WINDOW, config.N_FEATURES), name="features") d_sido = Input(shape=(len(preprocessing.get_sido_nm_list()),), name="sido_onehot") l1 = Conv1D(32, kernel_size=(12), activation='relu')(d_features) l2 = Conv1D(64, kernel_size=(6), activation='relu')(l1) l2_flat = Flatten()(l2) d_concatenated = Concatenate(axis=1)([l2_flat, d_sido]) d_1 = Dense(200)(d_concatenated) d_2 = Dense(100)(d_1) output = Dense(3, activation='softmax')(d_2) # output = Dense(3, activation='softmax')(l2_flat) model = Model(inputs=[d_features,d_sido],outputs=output) model.compile(loss=keras.losses.categorical_crossentropy, optimizer=keras.optimizers.Adadelta(),