def call_data(): loading = instant_data() df, mode = loading.hourly_instant(), 'hour' df = df[start_p:stop_p] data = df data = data.interpolate(limit=300000000, limit_direction='both').astype( 'float32' ) #interpolate neighbor first, for rest NA fill with mean() #.apply(lambda x: x.fillna(x.mean()),axis=0) data['Day'] = data.index.dayofyear # # MARS mars_cutoff = 0.3 data_mar = call_mar(data, target, mode, cutoff=mars_cutoff) data_mar = move_column_inplace(data_mar, target, 0) return data_mar
def datapreprocess(): loading = instant_data() df,mode = loading.hourly_instant(),'hour' df = df[start_p:stop_p] data = df data = data.interpolate(limit=300000000,limit_direction='both').astype('float32')#interpolate neighbor first, for rest NA fill with mean() #.apply(lambda x: x.fillna(x.mean()),axis=0) data[target].plot() # # MARS mars_cutoff = 0.3 data_mar = call_mar(data,target,mode,cutoff=mars_cutoff) data_mar = move_column_inplace(data_mar,target,0) # # SCALE # scaler = MinMaxScaler() # df_scaled = pd.DataFrame(scaler.fit_transform(data_mar), columns=data_mar.columns,index=data_mar.index) scaler_tar = MinMaxScaler() scaler_tar.fit(data[target].to_numpy().reshape(-1,1)) return data_mar,scaler_tar
return X, y, xlabels def inti_data(df): global start_p, stop_p data = df[start_p:stop_p].astype('float32') # data['Day'] = data.index.dayofyear #add day data = data.interpolate(limit=30000000, limit_direction='both').astype( 'float32') #interpolate neighbor first, for rest NA fill with mean() data[target] = data[target].shift(-out_t_step) data.dropna(inplace=True) return data #--------------------------------------------------------# loading = instant_data() df, mode = loading.hourly_instant(), 'hour' # df,mode = loading.daily_instant(),'day' st = 'CPY012' target, start_p, stop_p, host_path = station_sel(st, mode) if mode == 'hour': n_past, n_future = 24 * 6, 72 elif mode == 'day': n_past, n_future = 60, 30 n_pca = 4 split_date = '2016-11-01' #----------------------------------------# save_path = host_path + 'ML_svr/' if not os.path.exists(save_path): os.makedirs(save_path) ###########################################