def exp6(): # {{{ input_data, ans, date = load_wu([ 'SAR', 'yb_price', 'pa', 'humidity_low', 'wind_max_dir', 'wu_day_price', 'wu_day_amount' ]) for input_size in range(1, 15): mses = [] for i in range(5): processed_data = preprocess(input_data, ans, date, 30, 30, input_size, 1) data, _ = split(processed_data, tr_ratio=0.8, va_ratio=0.1, te_ratio=0.1) npad = ((0, 0), (0, 3), (0, 0)) data['train_x'] = np.pad(data['train_x'], npad, 'constant', constant_values=0) data['valid_x'] = np.pad(data['valid_x'], npad, 'constant', constant_values=0) data['test_x'] = np.pad(data['test_x'], npad, 'constant', constant_values=0) mse = train_and_eval_model('test', 'base', **data, drop_model=True) mses.append(mse) print(f'inupt_size={input_size}\nmse={np.mean(mse)}')
def load_data(target, mode): if 'wu' == target: input_data, ans, date = load_wu([ 'AED', 'pf_price', 'temp', 'humidity', 'wind_speed', 'wu_ex_weight', 'wu_ex_price', 'wu_month_price' ]) if 'day' == mode: processed_data = preprocess(input_data, ans, date, 1, 1, 11, 3) elif 'week' == mode: processed_data = preprocess(input_data, ans, date, 7, 7, 8, 2) elif 'chi' == target: input_data, ans, date = load_chi( ['wind_max', 'chi_day_price', 'chi_month_amount']) if 'day' == mode: processed_data = preprocess(input_data, ans, date, 1, 1, 11, 3) elif 'week' == mode: processed_data = preprocess(input_data, ans, date, 7, 7, 8, 2) data, time = split(processed_data, tr_ratio=0.8, va_ratio=0.1, te_ratio=0.1) return data, time
def test(): input_data, ans, date = load_wu([ 'AED', 'pf_price', 'temp', 'humidity', 'wind_speed', 'wu_ex_weight', 'wu_ex_price', 'wu_month_price' ]) processed_data = preprocess(input_data, ans, date, 1, 1, 7, 1) data, _ = split(processed_data, tr_ratio=0.8, va_ratio=0.1, te_ratio=0.1) res = train_and_eval_model('test', 'base', **data) print(res)
def exp3(): # {{{ # testing for results of exp2, results of selection target = argv[2] if 'wu' == target: input_data, ans, date = load_wu() elif 'chi' == target: input_data, ans, date = load_chi() else: raise Exception(f'Error target: {target}') with open(f'logs/selec_res_{target}', 'r') as f: cols_set = [ line.strip()[11:-1].replace('\'', '').split(' ') for line in f.readlines() ] for use_cols in cols_set: input_data = input_data[use_cols] processed_data = preprocess(input_data, ans, date, 1, 1, 7, 1) data, _ = split(processed_data, tr_ratio=0.8, va_ratio=0.1, te_ratio=0.1) res_base = [] res_large = [] for i in range(5): res1 = train_and_eval_model('test', 'base', **data, drop_model=True) res2 = train_and_eval_model('test', 'large', **data, drop_model=True) res_base.append(res1) res_large.append(res2) with open(f'logs/exp3_eval_selection_{target}', 'a+') as f: f.write(f'1.\n') f.write(f'Use columns: {use_cols}\n') f.write(f'base model: {res_base}\n') f.write(f'base mean: {np.mean(res_base)}\n') f.write(f'large model: {res_large}\n') f.write(f'large mean: {np.mean(res_large)}\n\n')
def exp2(): # {{{ # which features to use if 'wu' == argv[2]: input_data, ans, date = load_wu() elif 'chi' == argv[2]: input_data, ans, date = load_chi() else: exit() columns = input_data.columns.to_numpy() columns_history = [] mse_history = [] ori_columns = columns.copy() for _ in range(len(columns) - 2): res_list = [] for i in range(len(columns)): mask = np.ones(len(columns), dtype=bool) mask[i] = False data_mask = columns[mask] processed_data = preprocess(input_data[data_mask], ans, date, 1, 1, 7, 1) data, _ = split(processed_data, tr_ratio=0.8, va_ratio=0.1, te_ratio=0.1) res = train_and_eval_model('test', 'base', **data, drop_model=True) res_list.append(res) idx = np.argmin(res_list) columns = np.delete(columns, idx) columns_history.append(columns) mse_history.append(res_list) print('The origin columns: ') print(ori_columns) print('The final columns: ') print(columns) for i in range(len(columns_history)): print('res: ', mse_history[i]) print('columns: ', columns_history[i]) print('')