class TestPredictionPerformance(unittest.TestCase):
def setUp(self):
self.symbol = 'GOOG'
self.learn_start_date = dt.datetime(2004,9,1)
self.learn_end_date = dt.datetime(2011,1,1)
self.stock_period_start = dt.datetime(2000,1,1)
now = dt.datetime.now()
today = dt.datetime(now.year, now.month, now.day)
# stock_period contains important information like
# stock period start and the market symbol that was used in training and discolvery
self.stock_period = StockPeriod(self.symbol, self.stock_period_start, today)
self.mach_learn = EnsembleLearner.machine_learner_for_features_string(self.symbol,
GOOGF[0],
self.stock_period)
self.mach_learn.learn_period(self.learn_start_date, self.learn_end_date)
self.predictions = self.mach_learn.predict_period(dt.datetime(2012,1,1), dt.datetime(2013,1,1))
self.market_relative_close_data = self.mach_learn.feats.relative_data['close'][self.symbol]
self.perf = PredictionPerformance(self.predictions, self.market_relative_close_data, 2)
def test_initialize(self):
self.assertEqual(self.perf.end_date, self.perf.predictions.index[-1])
self.assertEqual(self.perf.start_date, self.perf.predictions.index[0])
self.assertEqual(self.perf.mRel_close.index[0], self.perf.start_date)
self.assertEqual(self.perf.mRel_close.index[-1], self.perf.end_date)
print self.perf.predictions[:][:30]
def test_return_neg_std(self):
print self.perf.return_neg_std()
def __init__(self, y, absolute_y, predictions, close_data, hold_days):
self.y = y
self.absolute_y = absolute_y
self.predictions = predictions
self.close_data = close_data
self.length = float(len(y))
self.calc_scoring_stats()
self.prediction_perf = PredictionPerformance(predictions, close_data, hold_days)
def setUp(self):
self.symbol = 'GOOG'
self.learn_start_date = dt.datetime(2004,9,1)
self.learn_end_date = dt.datetime(2011,1,1)
self.stock_period_start = dt.datetime(2000,1,1)
now = dt.datetime.now()
today = dt.datetime(now.year, now.month, now.day)
# stock_period contains important information like
# stock period start and the market symbol that was used in training and discolvery
self.stock_period = StockPeriod(self.symbol, self.stock_period_start, today)
self.mach_learn = EnsembleLearner.machine_learner_for_features_string(self.symbol,
GOOGF[0],
self.stock_period)
self.mach_learn.learn_period(self.learn_start_date, self.learn_end_date)
self.predictions = self.mach_learn.predict_period(dt.datetime(2012,1,1), dt.datetime(2013,1,1))
self.market_relative_close_data = self.mach_learn.feats.relative_data['close'][self.symbol]
self.perf = PredictionPerformance(self.predictions, self.market_relative_close_data, 2)
from simple_order_gen import SimpleOrderGen
from prediction_performance import PredictionPerformance
import matplotlib.pyplot as plt
symbol = "GOOG"
learn_start_date = dt.datetime(2004, 9, 1)
learn_end_date = dt.datetime(2011, 1, 1)
now = dt.datetime.now()
today = dt.datetime(now.year, now.month, now.day)
stock_period = StockPeriod(symbol, dt.datetime(2000, 1, 1), today)
learners = map(
lambda fs: EnsembleLearner.machine_learner_for_features_string(symbol, fs, stock_period),
ensemble_feats.goog_lower_bound8,
)
ensemble = EnsembleLearner(symbol, learn_start_date, learn_end_date, stock_period, learners)
ensemble.learn()
predictions = ensemble.predict_period(dt.datetime(2012, 1, 1), dt.datetime(2013, 1, 1), 0.1)
pp = PredictionPerformance(predictions, ensemble.learners[0].feats.relative_data["close"][symbol], 2)
print "tot return:", pp.tot_return()
print "mean return:", pp.return_mean()
print "std return:", pp.return_std()
print "sharpe's ratio:", pp.sharpe_ratio()
pp.predictions["tot_return"].plot()
plt.show()
class LearnScorer():
def __init__(self, y, absolute_y, predictions, close_data, hold_days):
self.y = y
self.absolute_y = absolute_y
self.predictions = predictions
self.close_data = close_data
self.length = float(len(y))
self.calc_scoring_stats()
self.prediction_perf = PredictionPerformance(predictions, close_data, hold_days)
def calc_scoring_stats(self):
self.num_equal = 0.0
self.bets = 0.0
self.good_bets = 0.0
self.really_good_bets = 0.0
self.really_bad_bets = 0.0
for p,y,yvv in zip(self.predictions.values, self.y.values, self.absolute_y):
if p[0] == 1 or p[2] == 1:
self.bets += 1
if yvv == p[0]:
self.good_bets += 1
elif (y[0] == 1 or y[2] == 1) and y[0] != p[0]: # not all(map(lambda s,d: s == d, p,y)):
self.really_bad_bets += 1
if sum(p) == 0 and y[1] == 1:
self.num_equal += 1
if all(map(lambda s,d: s == d, p,y)):
self.num_equal += 1
if p[0] == 1 or p[2] == 1:
self.really_good_bets += 1
def percent_correct(self):
return self.num_equal / self.length
def percent_good_bets(self):
if self.good_bets == 0:
return 0
else:
return float(self.good_bets) / self.bets
def percent_really_good_bets(self):
if self.really_good_bets == 0:
return 0
else:
return float(self.really_good_bets) / self.bets
def percent_really_bad_bets(self):
if self.really_bad_bets == 0:
return 0
else:
return float(self.really_bad_bets) / self.bets
def percent_really_good_to_good(self):
if self.good_bets == 0:
return 0
else:
return float(self.really_good_bets) / self.good_bets
def overall_score(self):
if self.bets == 0:
return 0
else:
return ((self.percent_good_bets() + self.percent_really_good_to_good()) / 2.0) - self.percent_really_bad_bets()
def fitness_score(self):
return (self.percent_really_good_bets() + 2 * self.percent_good_bets() + 2 * (1 - self.percent_really_bad_bets())) / 5.0
def fitness_score2(self):
return self.sharpe_ratio() / 5.0
def fitness_score3(self):
return self.tot_return() / 2.0
def fitness_score4(self):
return self.sortino_ratio() / 10.0
def tot_return(self):
return self.prediction_perf.tot_return()
def sharpe_ratio(self):
return self.prediction_perf.sharpe_ratio()
def sortino_ratio(self):
return self.prediction_perf.sortino_ratio()
def count_summary(self):
return {"bets":self.bets,
"length":self.length,
"num_equal": self.num_equal,
"good_bets": self.good_bets,
"really_good_bets": self.really_good_bets,
"really_bad_bets": self.really_bad_bets }
def stats(self):
return {"percent_correct": round(self.percent_correct(),5),
"percent_good_bets": round(self.percent_good_bets(),5),
"percent_really_good_bets": round(self.percent_really_good_bets(),5),
"percent_really_bad_bets": round(self.percent_really_bad_bets(),5),
#"mean_ret": round(self.prediction_perf.return_mean(),5),
#"std_ret": round(self.prediction_perf.return_std(),5),
"tot_return": round(self.tot_return(),5),
"sharpe_ratio": round(self.sharpe_ratio(),5),
"sortino_ratio": round(self.sortino_ratio(),5),
"overall_score": round(self.overall_score(),5)
}
first_learners = map(lambda fs: EnsembleLearner.machine_learner_for_features_string(symbol, fs, stock_period),
map(lambda x: x.feature_string, found_ensemble.found_forests))
prediction_performances = []
val_prediction_performances = []
another_domain_pp = []
use_another_domain = (settings.predict_cache_end_time == dt.datetime(2013,2,1))
for found_ensemble in gse.found_ensembles:
all_predictions = map(lambda ff: ff.test_predictions, found_ensemble.found_forests)
#all_predictions = [found_ensemble.found_forests[0].test_predictions]
joined_cached_predicitons = EnsembleLearner.process_predictions(all_predictions, 0.1)
test_period_predicitons = joined_cached_predicitons[settings.test_start_time:settings.test_end_time]
#test_period_predicitons = joined_cached_predicitons[settings.val_end_time:(settings.val_end_time + relativedelta(months=1) )]
pp = PredictionPerformance(test_period_predicitons, first_learners[0].feats.relative_data['close'][symbol], 2)
prediction_performances.append(pp)
val_period_predicitons = joined_cached_predicitons[settings.val_start_time:settings.val_end_time]
val_pp = PredictionPerformance(val_period_predicitons, first_learners[0].feats.relative_data['close'][symbol], 2)
val_prediction_performances.append(val_pp)
if use_another_domain:
another_domain_predictions = joined_cached_predicitons[settings.test_end_time:(settings.test_end_time + relativedelta(months=1))]
#another_domain_predictions = joined_cached_predicitons[(settings.val_end_time + relativedelta(months=1)):(settings.val_end_time + relativedelta(months=2) )]
another_pp = PredictionPerformance(another_domain_predictions, first_learners[0].feats.relative_data['close'][symbol], 2)
another_domain_pp.append(another_pp)
data_holder = []
