def get_analysis(self):
tdf = self.tdf
rs_oos = self.rs
mr_df = compare_monthly_returns(rs_oos,
tdf).loc[self.startdate:self.enddate]
cp_df = compare_series(rs_oos, tdf).loc[self.startdate:self.enddate]
rs = RSeries()
rs.load_custom_series(cp_df[["ACTUAL"]].pct_change())
actual_metrics = rs.get_metrics()["Overall"]
rs = RSeries()
rs.load_custom_series(cp_df[["OOS"]].pct_change())
allo_metrics = rs.get_metrics()["Overall"]
# allo_metrics = self.rs.get_metrics()["Overall"]
metrics_df = pd.DataFrame(
dict(actual=actual_metrics, replica=allo_metrics)
) # may not be meaningful! (because of linear interpolation on actual series)
mr_df = mr_df.dropna()
rmse = np.sqrt(mean_squared_error(mr_df["ACTUAL"], mr_df["OOS"]))
mae = mean_absolute_error(mr_df["ACTUAL"], mr_df["OOS"])
oos_ret = mr_df["OOS"]
mean_neg = np.mean(oos_ret[oos_ret < 0])
print("RMSE:", rmse)
print("mae:", mae)
print("mean_neg:", mean_neg)
return mr_df, cp_df, metrics_df
def download_yahoo(asset_list):
MS = MongoStorage()
z = MS.Find({
"Name": {
"$in": asset_list
},
"SeriesType": {
"$in": ["yahoo"]
}
})
exist_asset = [j["Name"] for j in z]
to_download_asset_list = [j for j in asset_list if j not in exist_asset]
to_download_asset_list
for asset in to_download_asset_list:
try:
df = query_data(asset)
df[asset] = df["Adj Close"].pct_change()
df = df.dropna()
rdf = df[[asset]]
if rdf.shape[0] > 100:
rs = RSeries()
rs.load_custom_series(rdf,
custom_series_name=asset,
series_type="yahoo")
rs.SaveSeries()
else:
print("Not enough data:", asset)
except Exception as err:
print(err)
print("Error:", asset)
def construct_entire_series(self):
df = pd.concat([j["forward_df"] for j in self.output])
df["pcret"] = cumulative_returns(df["pret"])
rs = RSeries()
rdf = df[["pret"]]
rs.load_custom_series(rdf,
custom_series_name="custom_allocate_series",
series_type="CustomAllo")
self.fdf = df
self.rs = rs
def Load(self, id_list, db_name = "AnalysisEvo", coll_name = "Strategy"):
if not isinstance(id_list, list):
id_list = [id_list]
db = self.client[db_name]
coll = db[coll_name]
id_list = [ObjectId(j) for j in id_list]
doc_list = [r for r in coll.find({"_id": {"$in":id_list}})]
rs_list = []
for doc in doc_list:
rs = RSeries()
rs.load_from_mongo(doc = doc)
rs_list.append(rs)
return rs_list
def AddCustomSeries(self, df, custom_series_name = None):
rseries = RSeries()
rseries.load_custom_series(df, custom_series_name)
self.AddSeries(rseries)
def SelectAndAllocateOverTime(self, select_method, allocate_method, weight_lookback, weight_forward, select_lookback, startdate = None, lastdate = None, **kwargs):
### INITIALIZE DATA ###
last_index = kwargs.get("last_index")
filter_seriestype = kwargs.get("filter_seriestype")
filter_seriestype = ["Strategy"] if filter_seriestype is None else filter_seriestype
filter_strategy_list = self.FilterStrategyList(series_type_list = filter_seriestype)
df = self.get_df_combined(strategy_list = filter_strategy_list).fillna(0).copy()
# Setting the startdate and lastdate (change lastdate for robustness testing)
if startdate is None:
startdate = df.first_valid_index()
else:
startdate = startdate - datetime.timedelta(weight_lookback)
if lastdate is None:
lastdate = df.last_valid_index()
df = df.loc[startdate:lastdate]
Result = dict()
### END OF INITIALIZATION ###
### SPLIT DATA INTO N INTERVALS ###
### EACH INTERVAL CONTAINS A LOOKBACK FOR SELECTION AND ALLOCATION, AND A FORWARD EVALUATION PERIOD ###
### LOOP ###
result_date_dict_list = generate_allocation_date(startdate, lastdate, weight_lookback, weight_forward, select_lookback)
result_date_dict_list = result_date_dict_list[0:last_index]
counter = 0
for dd in result_date_dict_list:
s1, s2 = dd.get("select_back_startdate"), dd.get("select_back_enddate")
a1, a2 = dd.get("allocate_back_startdate"), dd.get("allocate_back_enddate")
f1, f2 = dd.get("fwd_startdate"), dd.get("fwd_enddate")
rs_df = self.all_rseries_df.copy()
rs_df["Selected"] = 1
bwd_df = df.loc[a1:a2]
fwd_df = df.loc[f1:f2]
# FILTER
rs_df.loc[~rs_df["Name"].isin(filter_strategy_list), "Selected"] = 0
dd["Filter_Selected"] = filter_strategy_list
# EXCLUSION
rs_df, exclusion_selected_strategy_list = self.Exclude(rs_df, f1, f2)
dd["Exclusion_Selected"] = exclusion_selected_strategy_list
# SELECTION
rs_df, selected_strategy_list = self.Select(select_method, rs_df, s1, s2, **kwargs)
dd["Selected"] = selected_strategy_list
# ALLOCATION
weights, rs_df = self.Allocate(allocate_method, rs_df, a1, a2, **kwargs)
dd["rs_df"] = rs_df
dd["Weights"] = weights
# WEIGHTS ADJUSTMENT
# EVALUATION
weights_list = [j for j in weights.values()]
if check_equal_weight(weights_list):
print("WARNING: OPTIMIZER DOES NOT CONVERGE - STUCK AT EQUAL WEIGHTS! Counter:", counter)
final_selected_strategy_list = rs_df.loc[rs_df["Selected"] == 1, "Name"].values
dd["interval_optimized_fwd_df"] = weights_list*fwd_df[final_selected_strategy_list]
dd["interval_fwd_sharpe"] = P_SHARPE(weights_list, fwd_df[final_selected_strategy_list])
dd["interval_bwd_sharpe"] = P_SHARPE(weights_list, bwd_df[final_selected_strategy_list])
counter = counter + 1
Result["Intervals"] = result_date_dict_list
### END OF LOOP ###
### COMBINE INTERVALS ###
### Generate Portfolio Equity Curve ###
try:
Result["Portfolio_DF"] = self.GetPortfolioDF(result_date_dict_list)
Result["Portfolio_Returns"] = Result["Portfolio_DF"].sum(axis = 1)
Result["Portfolio_Equity"] = cumulative_returns(Result["Portfolio_Returns"])
Result["Portfolio_Equity"] = pd.Series(Result["Portfolio_Equity"], index = Result["Portfolio_Returns"].index)
rs = RSeries()
df = pd.DataFrame(Result["Portfolio_Returns"], columns = ["Portfolio_Returns"])
rs.load_custom_series(df, custom_series_name = "Portfolio", series_type = "Portfolio")
Result["RSeries"] = rs
Result["Metrics"] = rs.get_metrics_by_date(d1 = startdate, d2 = lastdate)
Result["SetupName"] = kwargs.get("setup_name")
except Exception as err:
print(err)
### END OF COMBINE ###
self.Result = Result
return Result
def SingleSimulation(self,
select_lookback,
allocate_lookback,
evaluate_forward,
startdate,
enddate,
filter_dict={},
custom_name="",
**sim_kwargs):
# custom_name = "Sim_{}_{}".format(sim_kwargs.get("sim_name"), sim_kwargs.get("seed"))
### SPLIT DATA INTO N INTERVALS ###
### EACH INTERVAL CONTAINS A LOOKBACK FOR SELECTION AND ALLOCATION, AND A FORWARD EVALUATION PERIOD ###
### LOOP ###
result_date_dict_list = generate_allocation_date(
startdate, enddate, allocate_lookback, evaluate_forward,
select_lookback)
oos_portfolio_df_list = []
is_portfolio_df_list = []
original_seed = sim_kwargs["seed"]
counter = 0
sim_obj_list = []
for dd in result_date_dict_list:
s1, s2 = dd.get("select_back_startdate"), dd.get(
"select_back_enddate")
a1, a2 = dd.get("allocate_back_startdate"), dd.get(
"allocate_back_enddate")
f1, f2 = dd.get("fwd_startdate"), dd.get("fwd_enddate")
sim_kwargs["seed"] = original_seed + counter
date_kwargs = dict(s1=s1, s2=s2, a1=a1, a2=a2, f1=f1, f2=f2)
interval_kwargs = {**date_kwargs, **sim_kwargs}
PS = None
PS = PortfolioSimulation(kwargs=interval_kwargs,
filter_dict=filter_dict)
sim_obj_list.append(PS)
oos_portfolio_df_list.append(PS.portfolio_df.copy()[["pret"]])
is_portfolio_df_list.append(PS.is_portfolio_df.copy()[["pret"]])
self.log(custom_name, f2, PS.logs.copy())
counter = counter + 1
sim_kwargs["original_seed"] = original_seed
### END OF LOOP ###
# oos_portfolio_df = pd.concat(oos_portfolio_df_list).sort_index()
# is_portfolio_df = pd.concat(is_portfolio_df_list).sort_index()
oos_portfolio_df = pd.concat(oos_portfolio_df_list)
oos_portfolio_df = oos_portfolio_df.groupby(
oos_portfolio_df.index).first().sort_index()
is_portfolio_df = pd.concat(is_portfolio_df_list)
is_portfolio_df = is_portfolio_df.groupby(
is_portfolio_df.index).first().sort_index()
rs_oos = RSeries()
rs_oos.load_custom_series(
oos_portfolio_df[["pret"]],
custom_series_name="OOS_{}".format(custom_name),
series_type="Allocation")
rs_is = RSeries()
rs_is.load_custom_series(
is_portfolio_df[["pret"]],
custom_series_name="IS_{}".format(custom_name),
series_type="Allocation")
if custom_name != "":
oos_kw, is_kw = sim_kwargs.copy(), sim_kwargs.copy()
oos_kw["Sample"], is_kw["Sample"] = "OOS", "IS"
rs_oos.SaveSeries(user=custom_name, extra=oos_kw.copy())
rs_is.SaveSeries(user=custom_name, extra=is_kw.copy())
self.rs_is, self.rs_oos, self.sim_obj_list = rs_is, rs_oos, sim_obj_list
print("Successful.")
def SingleMonthlySimulationWithTarget(self,
startdate,
enddate,
target_ret_dict,
filter_dict={},
**sim_kwargs):
custom_name = "Sim_{}_{}".format(sim_kwargs.get("sim_name"),
sim_kwargs.get("seed"))
### SPLIT DATA INTO N INTERVALS ###
### EACH INTERVAL CONTAINS A LOOKBACK FOR SELECTION AND ALLOCATION, AND A FORWARD EVALUATION PERIOD ###
### LOOP ###
result_date_dict_list = generate_monthly_allocation_date(
startdate, enddate)
oos_portfolio_df_list = []
is_portfolio_df_list = []
original_seed = sim_kwargs["seed"]
counter = 0
sim_obj_list = []
for dd in result_date_dict_list:
s1, s2 = dd.get("select_back_startdate"), dd.get(
"select_back_enddate")
a1, a2 = dd.get("allocate_back_startdate"), dd.get(
"allocate_back_enddate")
f1, f2 = dd.get("fwd_startdate"), dd.get("fwd_enddate")
target_ret = target_ret_dict.get(a2)
sim_kwargs["seed"] = original_seed + counter
date_kwargs = dict(s1=s1, s2=s2, a1=a1, a2=a2, f1=f1, f2=f2)
interval_kwargs = {**date_kwargs, **sim_kwargs}
interval_kwargs["target_ret"] = target_ret
PS = None
PS = PortfolioSimulation(kwargs=interval_kwargs,
filter_dict=filter_dict)
sim_obj_list.append(PS)
oos_portfolio_df_list.append(PS.portfolio_df.copy()[["pret"]])
is_portfolio_df_list.append(PS.is_portfolio_df.copy()[["pret"]])
self.log(custom_name, f2, PS.logs.copy())
counter = counter + 1
sim_kwargs["original_seed"] = original_seed
### END OF LOOP ###
# oos_portfolio_df = pd.concat(oos_portfolio_df_list).sort_index()
# is_portfolio_df = pd.concat(is_portfolio_df_list).sort_index()
# solve duplicated date index when allocate lookback != evaluation fwd
oos_portfolio_df = pd.concat(oos_portfolio_df_list)
oos_portfolio_df = oos_portfolio_df.groupby(
oos_portfolio_df.index).first().sort_index()
is_portfolio_df = pd.concat(is_portfolio_df_list)
is_portfolio_df = is_portfolio_df.groupby(
is_portfolio_df.index).first().sort_index()
rs_oos = RSeries()
rs_oos.load_custom_series(oos_portfolio_df[["pret"]],
custom_series_name=custom_name,
series_type="Simulation")
rs_is = RSeries()
rs_is.load_custom_series(is_portfolio_df[["pret"]],
custom_series_name=custom_name,
series_type="Simulation")
# if self.save:
# rs.SaveSeries(user="******", extra = sim_kwargs.copy(), db_name = "AnalysisEvo", coll_name = "Simulation")
self.rs_is, self.rs_oos, self.sim_obj_list = rs_is, rs_oos, sim_obj_list
print("Successful.")