loss_file = '{}/{}.csv'.format(csv_dir, loss_file_name)
return [y_pred_file, ts_plot_path, loss_file, loss_plot_path]
else:
return [y_pred_file, ts_plot_path]
#model options: VAR, RNN , R2N2, Baselime, Perfect
model = 'Perfect'
y_pred_csv_name = 'train_y'
plot_suffix = 'train'
loss_file_name = None
#set options: 'train', 'cross_val', or 'test'
set = 'train'
file_list = get_file_names(model= model, y_pred_csv_name= y_pred_csv_name,
plot_suffix= plot_suffix, loss_file_name= loss_file_name)
check_model = eval_model(y_pred_file= file_list[0], set= set)
check_model.backtest(printer=True)
check_model.plot_backtest(file_path=file_list[1])
if loss_file_name is not None:
check_model.plot_loss(loss_csv_path= file_list[2], loss_plot_path= file_list[3])
check_model.accuracy(printer= True)
VAR_model = VAR(X_train_df) results = VAR_model.fit(1) Y_train_pred_df, train_auc, train_mse = fit_VAR(results, 'train') Y_dev_pred_df, dev_auc, dev_mse = fit_VAR(results, 'cross_val') model_name = 'VAR_int' # run eval class check_model = eval_model(y_pred_df= Y_train_pred_df, y_actual_df= Y_train_df) check_model.backtest(printer=False) check_model.accuracy(printer=False) train_metrics_dict = check_model.metrics train_acc_score = check_model.accuracy_score train_conf_list = check_model.confusion_matrix # do for both train and dev set metrics # run eval class
import numpy as np
import os
import pandas as pd
from Data.scripts.data import data
from sklearn.metrics import mean_squared_error, roc_auc_score, roc_curve, auc
from Models.Evaluation.eval import eval_model
_, Y_actual = data.import_data(set='test')
y_pred_df = Y_actual.shift(1)
y_pred_df = y_pred_df.dropna()
Y_actual = Y_actual.loc[y_pred_df.index]
tester = eval_model(y_pred_df=y_pred_df, y_actual_df=Y_actual)
tester.backtest(printer=False)
out_dict = tester.metrics
out_dict['mse'] = mean_squared_error(Y_actual.as_matrix(),
y_pred_df.as_matrix())
flat_pred = np.clip(y_pred_df.as_matrix().flatten() + 0.5, 0, 1)
flat_actual = np.where(Y_actual.as_matrix().flatten() > 0, 1, 0)
out_dict['auc'] = roc_auc_score(flat_actual, flat_pred)
model = 'Test'
print(out_dict)
import torch
from torch.autograd import Variable
from Data.scripts.data import data
from Models.Extra.scripts.RNN import RNN
from Models.Evaluation.eval import eval_model
from statsmodels.tsa.api import VAR
from sklearn.metrics import mean_squared_error, roc_auc_score, roc_curve, auc
X_test, Y_test = data.import_data(set='test')
Y_pred_df = ((-1) * Y_test.shift(1)).dropna()
Y_test = Y_test.loc[Y_pred_df.index]
check_model = eval_model(y_pred_df=Y_pred_df, y_actual_df=Y_test)
check_model.backtest(printer=False)
check_model.accuracy(printer=False)
dev_metrics_dict = check_model.metrics
dev_acc_score = check_model.accuracy_score
dev_conf_list = check_model.confusion_matrix
print('Dev')
print(dev_metrics_dict)
print('Acc: {}'.format(dev_acc_score))
from torch.autograd import Variable
from Data.scripts.data import data
from Models.Extra.scripts.RNN import RNN
from Models.Evaluation.eval import eval_model
import matplotlib.pyplot as plt
from statsmodels.tsa.api import VAR
from sklearn.metrics import mean_squared_error, roc_auc_score, roc_curve, auc
X, Y = data.import_data(set='test')
Y_pred = pd.read_csv(
'/Users/ianshaw/Downloads/GitHub/deepcrypto/Models/VAR/predicted_values_VAR_test_shifted.csv',
index_col=0)
check_model = eval_model(y_pred_df=Y_pred, y_actual_df=Y)
check_model.backtest(printer=False)
check_model.accuracy(printer=False)
check_model.strat_series.plot()
plt.show()
train_metrics_dict = check_model.metrics
train_acc_score = check_model.accuracy_score
train_conf_list = check_model.confusion_matrix
print('Train')
