class LearnScorer(): def __init__(self, y, absolute_y, predictions, close_data, hold_days): self.y = y self.absolute_y = absolute_y self.predictions = predictions self.close_data = close_data self.length = float(len(y)) self.calc_scoring_stats() self.prediction_perf = PredictionPerformance(predictions, close_data, hold_days) def calc_scoring_stats(self): self.num_equal = 0.0 self.bets = 0.0 self.good_bets = 0.0 self.really_good_bets = 0.0 self.really_bad_bets = 0.0 for p,y,yvv in zip(self.predictions.values, self.y.values, self.absolute_y): if p[0] == 1 or p[2] == 1: self.bets += 1 if yvv == p[0]: self.good_bets += 1 elif (y[0] == 1 or y[2] == 1) and y[0] != p[0]: # not all(map(lambda s,d: s == d, p,y)): self.really_bad_bets += 1 if sum(p) == 0 and y[1] == 1: self.num_equal += 1 if all(map(lambda s,d: s == d, p,y)): self.num_equal += 1 if p[0] == 1 or p[2] == 1: self.really_good_bets += 1 def percent_correct(self): return self.num_equal / self.length def percent_good_bets(self): if self.good_bets == 0: return 0 else: return float(self.good_bets) / self.bets def percent_really_good_bets(self): if self.really_good_bets == 0: return 0 else: return float(self.really_good_bets) / self.bets def percent_really_bad_bets(self): if self.really_bad_bets == 0: return 0 else: return float(self.really_bad_bets) / self.bets def percent_really_good_to_good(self): if self.good_bets == 0: return 0 else: return float(self.really_good_bets) / self.good_bets def overall_score(self): if self.bets == 0: return 0 else: return ((self.percent_good_bets() + self.percent_really_good_to_good()) / 2.0) - self.percent_really_bad_bets() def fitness_score(self): return (self.percent_really_good_bets() + 2 * self.percent_good_bets() + 2 * (1 - self.percent_really_bad_bets())) / 5.0 def fitness_score2(self): return self.sharpe_ratio() / 5.0 def fitness_score3(self): return self.tot_return() / 2.0 def fitness_score4(self): return self.sortino_ratio() / 10.0 def tot_return(self): return self.prediction_perf.tot_return() def sharpe_ratio(self): return self.prediction_perf.sharpe_ratio() def sortino_ratio(self): return self.prediction_perf.sortino_ratio() def count_summary(self): return {"bets":self.bets, "length":self.length, "num_equal": self.num_equal, "good_bets": self.good_bets, "really_good_bets": self.really_good_bets, "really_bad_bets": self.really_bad_bets } def stats(self): return {"percent_correct": round(self.percent_correct(),5), "percent_good_bets": round(self.percent_good_bets(),5), "percent_really_good_bets": round(self.percent_really_good_bets(),5), "percent_really_bad_bets": round(self.percent_really_bad_bets(),5), #"mean_ret": round(self.prediction_perf.return_mean(),5), #"std_ret": round(self.prediction_perf.return_std(),5), "tot_return": round(self.tot_return(),5), "sharpe_ratio": round(self.sharpe_ratio(),5), "sortino_ratio": round(self.sortino_ratio(),5), "overall_score": round(self.overall_score(),5) }
from simple_order_gen import SimpleOrderGen from prediction_performance import PredictionPerformance import matplotlib.pyplot as plt symbol = "GOOG" learn_start_date = dt.datetime(2004, 9, 1) learn_end_date = dt.datetime(2011, 1, 1) now = dt.datetime.now() today = dt.datetime(now.year, now.month, now.day) stock_period = StockPeriod(symbol, dt.datetime(2000, 1, 1), today) learners = map( lambda fs: EnsembleLearner.machine_learner_for_features_string(symbol, fs, stock_period), ensemble_feats.goog_lower_bound8, ) ensemble = EnsembleLearner(symbol, learn_start_date, learn_end_date, stock_period, learners) ensemble.learn() predictions = ensemble.predict_period(dt.datetime(2012, 1, 1), dt.datetime(2013, 1, 1), 0.1) pp = PredictionPerformance(predictions, ensemble.learners[0].feats.relative_data["close"][symbol], 2) print "tot return:", pp.tot_return() print "mean return:", pp.return_mean() print "std return:", pp.return_std() print "sharpe's ratio:", pp.sharpe_ratio() pp.predictions["tot_return"].plot() plt.show()
if use_another_domain: another_domain_predictions = joined_cached_predicitons[settings.test_end_time:(settings.test_end_time + relativedelta(months=1))] #another_domain_predictions = joined_cached_predicitons[(settings.val_end_time + relativedelta(months=1)):(settings.val_end_time + relativedelta(months=2) )] another_pp = PredictionPerformance(another_domain_predictions, first_learners[0].feats.relative_data['close'][symbol], 2) another_domain_pp.append(another_pp) data_holder = [] if use_another_domain: for pp, pp_val, pp_another in iter.izip(prediction_performances, val_prediction_performances, another_domain_pp): data_holder.append([pp.tot_return(), pp.return_mean(), pp.return_std(), pp.sharpe_ratio(), pp_val.tot_return(), pp_another.tot_return()]) chart_df = pd.DataFrame(data= data_holder, columns = ["tot_return", "mean", "std", "sharpe", "val_return", "cur_return"]) # chart_df["cur_return"] = ((1 + chart_df["cur_return"]) ** 12) - 1 chart_df["cur_return"] = chart_df["cur_return"] * 12 #chart_df["tot_return"] = chart_df["tot_return"] * 12 else: for pp, pp_val in iter.izip(prediction_performances, val_prediction_performances): data_holder.append([pp.tot_return(), pp.return_mean(), pp.return_std(), pp.sharpe_ratio(), pp_val.tot_return()]) chart_df = pd.DataFrame(data= data_holder, columns = ["tot_return", "mean", "std", "sharpe", "val_return"])