import numpy as np

import pandas as pd

import missingno as msno

import matplotlib.pyplot as plt

import statsmodels.api as sm

import statsmodels.formula.api as smf

import matplotlib.pyplot as plt

import plotly.plotly as py

import seaborn as sns

from sklearn.metrics import mean_squared_error #, cross_val_score

from sklearn.model_selection import train_test_split, cross_val_score

from sklearn.linear_model import LassoCV, LinearRegression, RidgeCV, Ridge

from fancyimpute import SimpleFill, KNN, IterativeSVD, IterativeImputer

from plotly.graph_objs import *

from sklearn.preprocessing import scale

from statsmodels.stats.outliers_influence import variance_inflation_factor

## Data Scrubber

def data_scrubber(income_df, percent_good):

return(income_df.fillna(income_df.mean()))

def join_df_weather(dfmeter_hh, dfweather_h):

return merg # .drop(columns = extradata, inplace = True)

def break_by_meter(df, unique_meters):

return meter_df_list

## plot Scatter_matrix()

## Discuss corollations

def plot_scatter_matrix(df):

plot1 = pd.plotting.scatter_matrix(df)

plt.savefig('images/Scatter_matrix_of_{}.png'.format(df['LCLid'][0]))#Jane says: this is elegant; it automatically names the file appropriately, rather than asking for user input

## Mean energy by block

## build function

## Plot heat map of blocks

## Mean energy by meter in each block

def mean_meter_all(meter_df_list):

return mean, name

def split_data_single_meter(single_meter_df, drop_list):

return X_train, X_test, y_train, y_test, df1

def linear_reg_single_meter(X_train, X_test, y_train, y_test):

return ridge_cv, lasso_cv, linear

def run_models(meter_df_list, meter,list_to_drop):

return ridge_cv_meter, lasso_cv_meter, linear_meter, X_train_meter, X_test_meter, y_train_meter, y_test_meter

def Ridge_plot_updated(X_train, X_test, y_train, y_test):

ax.legend(loc="upper right")

## This needs to be fixed. Log issue??

def Ridge_plot(X_train, X_test, y_train, y_test):

return plot_x, plot_y, plot_y_test

## Not working on the data ?? need to fix

def OLS_model(X_train,y_train):

plt.show()

def OLS_model_noplot(X_train,y_train):

ols_model = sm.OLS(endog=y_train, exog=X_train).fit()

print(ols_model.summary())

## Split all data into train and test. Only test on test

## Need function to read and write new files to split data

## Place in train and test file sets for easyc iterations

## should probably build a database (SQL) with tables by block number

## Add cost benifit matrix from confuion matrix see this: /Documents/galvanize/Lectures/lecture_profit-curve-imbal-classes

## For each block test data

## Set x and y values

## X is average meter data

## Y will be the weather and time (month and day) data

## split into train and test data

## Do LinearRegression with Kfolds

## QQ plot

## do logisticRegression with X above or below average use

## ROC plot

#return linear errors

def vifs(x):

return vifs

def runNmeters(meter_df_list, n ):

return scores

def plt_v_time(y_test, y_train, y_train_pred, y_test_pred):

plt.show()

if __name__ == '__main__':

plt.rcParams.update({

'font.size' : 20.0,

'axes.titlesize' : 'large',

'axes.labelsize' : 'medium',

'xtick.labelsize' : 'medium',

'ytick.labelsize' : 'medium',

'legend.fontsize' : 'large',

})

dfmeter = pd.read_csv("data/smart_meters_london/daily_dataset/block_0.csv")

dfweather = pd.read_csv("data/smart_meters_london/weather_daily_darksky.csv")

dfmeter_hh = pd.read_csv("data/smart_meters_london/halfhourly_dataset/block_0.csv")##, dtype={'energy(kWh/hh)': float} )

dfweather_h = pd.read_csv("data/smart_meters_london/weather_hourly_darksky.csv")

df_meter_weather_hourly = join_df_weather(dfmeter_hh, dfweather_h) ## joins the two data frames on hour of time. Removes all non matching half hour data

df_meter_weather_hourly['energy'] = pd.to_numeric(df_meter_weather_hourly['energy(kWh/hh)']) ## converts exisitng energy colum to correct numeric values

unique_meters = df_meter_weather_hourly['LCLid'].unique() ## gets unique meters in block

meter_df_list = break_by_meter(df_meter_weather_hourly, unique_meters)

meter_num = 0

meter_df_list[meter_num] = pd.get_dummies(meter_df_list[meter_num],columns = ['precipType', 'icon','summary'])

## This is all for data with no Time parameter

print('Initial Reg without time')

list_to_drop = ['energy','energy(kWh/hh)','time', 'tstp', 'date_start_time', 'LCLid','dewPoint', 'apparentTemperature']

X_train_meter, X_test_meter, y_train_meter, y_test_meter, df = split_data_single_meter(meter_df_list[meter_num],drop_list = list_to_drop)

ridge_cv_meter, lasso_cv_meter, linear_meter = linear_reg_single_meter(X_train_meter, X_test_meter, y_train_meter, y_test_meter)

#OLS_model(X_train_meter,y_train_meter)

# OLS_model_noplot(X_train_meter,y_train_meter)

print('\n')

print('Initial Reg without time removing first set of Lasso coefficients = 0')

list_to_drop1 = list(X_test_meter.columns[lasso_cv_meter.coef_ == 0])

list_to_drop = list_to_drop1 + list_to_drop

X_train_meter1, X_test_meter1, y_train_meter1, y_test_meter1, df = split_data_single_meter(meter_df_list[meter_num], drop_list = list_to_drop )

ridge_cv_meter1, lasso_cv_meter1, linear_meter1 = linear_reg_single_meter(X_train_meter1, X_test_meter1, y_train_meter1, y_test_meter1)

#OLS_model(X_train_meter1,y_train_meter1)

# OLS_model_noplot(X_train_meter1,y_train_meter1)

print('\n')

print('Initial Reg without time removing second set of Lasso coefficients = 0')

list_to_drop2 = list(X_test_meter1.columns[lasso_cv_meter1.coef_ == 0])

list_to_drop = list_to_drop + list_to_drop2

X_train_meter2, X_test_meter2, y_train_meter2, y_test_meter2, df = split_data_single_meter(meter_df_list[meter_num], drop_list = list_to_drop )

ridge_cv_meter2, lasso_cv_meter2, linear_meter2 = linear_reg_single_meter(X_train_meter2, X_test_meter2, y_train_meter2, y_test_meter2)

#OLS_model(X_train_meter2,y_train_meter2)

# OLS_model_noplot(X_train_meter2,y_train_meter2)

print('\n')

## Now add in the time parameter

print('Initial Reg with time')

meter_df_list[meter_num]['hour_column'] = [d.hour for d in meter_df_list[meter_num]['date_start_time']]

meter_df_list[meter_num]['day_week'] = [d.dayofweek for d in meter_df_list[meter_num]['date_start_time']]

meter_df_list[meter_num] = pd.get_dummies(meter_df_list[meter_num] , columns = ['day_week','hour_column'])

list_to_drop = ['energy','energy(kWh/hh)','time', 'tstp', 'LCLid','dewPoint', 'apparentTemperature', 'date_start_time']

X_train_meter3, X_test_meter3, y_train_meter3, y_test_meter3, df = split_data_single_meter(meter_df_list[meter_num], drop_list = list_to_drop)

ridge_cv_meter3, lasso_cv_meter3, linear_meter3 = linear_reg_single_meter(X_train_meter3, X_test_meter3, y_train_meter3, y_test_meter3)

#OLS_model(X_train_meter,y_train_meter)

# OLS_model_noplot(X_train_meter3,y_train_meter3)

print('\n')

print('Initial Reg with time removing first set of Lasso coefficients = 0')

list_to_drop1 = list(X_test_meter.columns[lasso_cv_meter.coef_ == 0])

list_to_drop = list_to_drop1 + list_to_drop

X_train_meter4, X_test_meter4, y_train_meter4, y_test_meter4, df = split_data_single_meter(meter_df_list[meter_num], drop_list = list_to_drop )

ridge_cv_meter4, lasso_cv_meter4, linear_meter4 = linear_reg_single_meter(X_train_meter4, X_test_meter4, y_train_meter4, y_test_meter4)

#OLS_model(X_train_meter1,y_train_meter1)

# OLS_model_noplot(X_train_meter4,y_train_meter4)

print('\n')

list_to_drop2 = list(X_test_meter1.columns[lasso_cv_meter1.coef_ == 0])

list_to_drop = list_to_drop + list_to_drop2

X_train_meter5, X_test_meter5, y_train_meter5, y_test_meter5, df = split_data_single_meter(meter_df_list[meter_num], drop_list = list_to_drop )

ridge_cv_meter5, lasso_cv_meter5, linear_meter5 = linear_reg_single_meter(X_train_meter5, X_test_meter5, y_train_meter5, y_test_meter5)

#OLS_model(X_train_meter5,y_train_meter5)

# OLS_model_noplot(X_train_meter5,y_train_meter5)

print('\n')

# print('use this')

# ridge_cv,lasso_cv,linear,X_train,X_test,y_train,y_test = run_models(meter_df_list, meter_num,list_to_drop)

OLS_model_noplot(X_train_meter5,y_train_meter5)

y_train_pred = lasso_cv_meter5.predict(X_train_meter5)

y_test_pred = lasso_cv_meter5.predict(X_test_meter5)

plt_v_time(y_test_meter5, y_train_meter5, y_train_pred, y_test_pred)

#meterScores = runNmeters(meter_df_list, 5)

#plot_x_alphas, plot_y, plot_y_test = ridgeCV_plot(X_train, X_test, y_train, y_test)

mean_meter_enengy, mean_meter_enengy_name = mean_meter_all(meter_df_list)

#heat_map_plot(mean_meter_enengy, mean_meter_enengy_name)

#plt.show()