def time_of_day_seasonality(self, data_frame, years = False):
calculations = Calculations()
if years is False:
return calculations.average_by_hour_min_of_day_pretty_output(data_frame)
set_year = set(data_frame.index.year)
year = sorted(list(set_year))
intraday_seasonality = None
commonman = CommonMan()
for i in year:
temp_seasonality = calculations.average_by_hour_min_of_day_pretty_output(data_frame[data_frame.index.year == i])
temp_seasonality.columns = commonman.postfix_list(temp_seasonality.columns.values, " " + str(i))
if intraday_seasonality is None:
intraday_seasonality = temp_seasonality
else:
intraday_seasonality = intraday_seasonality.join(temp_seasonality)
return intraday_seasonality
def plot_strategy_group_benchmark_pnl_yoy(self, strip = None, silent_plot = False):
style = self.create_style("", "Group Benchmark PnL YoY - cumulative")
keys = self._strategy_group_benchmark_ret_stats.keys()
yoy = []
for key in keys:
col = self._strategy_group_benchmark_ret_stats[key].yoy_rets()
col.columns = [key]
yoy.append(col)
calculations = Calculations()
ret_stats = calculations.pandas_outer_join(yoy)
ret_stats.index = ret_stats.index.year
if strip is not None: ret_stats.columns = [k.replace(strip, '') for k in ret_stats.columns]
# ret_stats = ret_stats.sort_index()
style.file_output = self.DUMP_PATH + self.FINAL_STRATEGY + ' (Group Benchmark PnL - YoY) ' + str(style.scale_factor) + '.png'
style.html_file_output = self.DUMP_PATH + self.FINAL_STRATEGY + ' (Group Benchmark PnL - YoY) ' + str(style.scale_factor) + '.html'
style.display_brand_label = False
style.date_formatter = "%Y"
chart = Chart(ret_stats * 100, engine=self.DEFAULT_PLOT_ENGINE, chart_type='bar', style=style)
if not (silent_plot): chart.plot()
return chart
def bus_day_of_month_seasonality(self, data_frame,
month_list = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], cum = True,
cal = "FX", partition_by_month = True, add_average = False, price_index = False):
calculations = Calculations()
filter = Filter()
if price_index:
data_frame = data_frame.resample('B') # resample into business days
data_frame = calculations.calculate_returns(data_frame)
data_frame.index = pandas.to_datetime(data_frame.index)
data_frame = filter.filter_time_series_by_holidays(data_frame, cal)
monthly_seasonality = calculations.average_by_month_day_by_bus_day(data_frame, cal)
monthly_seasonality = monthly_seasonality.loc[month_list]
if partition_by_month:
monthly_seasonality = monthly_seasonality.unstack(level=0)
if add_average:
monthly_seasonality['Avg'] = monthly_seasonality.mean(axis=1)
if cum is True:
if partition_by_month:
monthly_seasonality.loc[0] = numpy.zeros(len(monthly_seasonality.columns))
# monthly_seasonality.index = monthly_seasonality.index + 1 # shifting index
monthly_seasonality = monthly_seasonality.sort_index()
monthly_seasonality = calculations.create_mult_index(monthly_seasonality)
return monthly_seasonality
def compare_strategy_vs_benchmark(self, br, strategy_df, benchmark_df):
"""Compares the trading strategy we are backtesting against a benchmark
Parameters
----------
br : BacktestRequest
Parameters for backtest such as start and finish dates
strategy_df : pandas.DataFrame
Strategy time series
benchmark_df : pandas.DataFrame
Benchmark time series
"""
include_benchmark = False
calc_stats = False
if hasattr(br, 'include_benchmark'): include_benchmark = br.include_benchmark
if hasattr(br, 'calc_stats'): calc_stats = br.calc_stats
if include_benchmark:
ret_stats = RetStats()
risk_engine = RiskEngine()
filter = Filter()
calculations = Calculations()
# align strategy time series with that of benchmark
strategy_df, benchmark_df = strategy_df.align(benchmark_df, join='left', axis = 0)
# if necessary apply vol target to benchmark (to make it comparable with strategy)
if hasattr(br, 'portfolio_vol_adjust'):
if br.portfolio_vol_adjust is True:
benchmark_df = risk_engine.calculate_vol_adjusted_index_from_prices(benchmark_df, br = br)
# only calculate return statistics if this has been specified (note when different frequencies of data
# might underrepresent vol
# if calc_stats:
benchmark_df = benchmark_df.fillna(method='ffill')
ret_stats.calculate_ret_stats_from_prices(benchmark_df, br.ann_factor)
if calc_stats:
benchmark_df.columns = ret_stats.summary()
# realign strategy & benchmark
strategy_benchmark_df = strategy_df.join(benchmark_df, how='inner')
strategy_benchmark_df = strategy_benchmark_df.fillna(method='ffill')
strategy_benchmark_df = filter.filter_time_series_by_date(br.plot_start, br.finish_date, strategy_benchmark_df)
strategy_benchmark_df = calculations.create_mult_index_from_prices(strategy_benchmark_df)
self._benchmark_pnl = benchmark_df
self._benchmark_ret_stats = ret_stats
return strategy_benchmark_df
return strategy_df
def get_pnl_trades(self):
"""Gets P&L of each individual trade per signal
Returns
-------
pandas.Dataframe
"""
if self._pnl_trades is None:
calculations = Calculations()
self._pnl_trades = calculations.calculate_individual_trade_gains(self._signal, self._pnl)
return self._pnl_trades
def calculate_diagnostic_trading_PnL(self, asset_a_df, signal_df, further_df = [], further_df_labels = []):
"""Calculates P&L table which can be used for debugging purposes,
The table is populated with asset, signal and further dataframes provided by the user, can be used to check signalling methodology.
It does not apply parameters such as transaction costs, vol adjusment and so on.
Parameters
----------
asset_a_df : DataFrame
Asset prices
signal_df : DataFrame
Trade signals (typically +1, -1, 0 etc)
further_df : DataFrame
Further dataframes user wishes to output in the diagnostic output (typically inputs for the signals)
further_df_labels
Labels to append to the further dataframes
Returns
-------
DataFrame with asset, trading signals and returns of the trading strategy for diagnostic purposes
"""
calculations = Calculations()
asset_rets_df = calculations.calculate_returns(asset_a_df)
strategy_rets = calculations.calculate_signal_returns(signal_df, asset_rets_df)
reset_points = ((signal_df - signal_df.shift(1)).abs())
asset_a_df_entry = asset_a_df.copy(deep=True)
asset_a_df_entry[reset_points == 0] = numpy.nan
asset_a_df_entry = asset_a_df_entry.ffill()
asset_a_df_entry.columns = [x + '_entry' for x in asset_a_df_entry.columns]
asset_rets_df.columns = [x + '_asset_rets' for x in asset_rets_df.columns]
strategy_rets.columns = [x + '_strat_rets' for x in strategy_rets.columns]
signal_df.columns = [x + '_final_signal' for x in signal_df.columns]
for i in range(0, len(further_df)):
further_df[i].columns = [x + '_' + further_df_labels[i] for x in further_df[i].columns]
flatten_df =[asset_a_df, asset_a_df_entry, asset_rets_df, strategy_rets, signal_df]
for f in further_df:
flatten_df.append(f)
return calculations.pandas_outer_join(flatten_df)
def run_day_of_month_analysis(self, trading_model):
from finmarketpy.economics.seasonality import Seasonality
calculations = Calculations()
seas = Seasonality()
trading_model.construct_strategy()
pnl = trading_model.get_strategy_pnl()
# get seasonality by day of the month
pnl = pnl.resample('B').mean()
rets = calculations.calculate_returns(pnl)
bus_day = seas.bus_day_of_month_seasonality(rets, add_average = True)
# get seasonality by month
pnl = pnl.resample('BM').mean()
rets = calculations.calculate_returns(pnl)
month = seas.monthly_seasonality(rets)
self.logger.info("About to plot seasonality...")
style = Style()
# Plotting spot over day of month/month of year
style.color = 'Blues'
style.scale_factor = self.SCALE_FACTOR
style.file_output = self.DUMP_PATH + trading_model.FINAL_STRATEGY + ' seasonality day of month.png'
style.html_file_output = self.DUMP_PATH + trading_model.FINAL_STRATEGY + ' seasonality day of month.html'
style.title = trading_model.FINAL_STRATEGY + ' day of month seasonality'
style.display_legend = False
style.color_2_series = [bus_day.columns[-1]]
style.color_2 = ['red'] # red, pink
style.linewidth_2 = 4
style.linewidth_2_series = [bus_day.columns[-1]]
style.y_axis_2_series = [bus_day.columns[-1]]
self.chart.plot(bus_day, chart_type='line', style=style)
style = Style()
style.scale_factor = self.SCALE_FACTOR
style.file_output = self.DUMP_PATH + trading_model.FINAL_STRATEGY + ' seasonality month of year.png'
style.html_file_output = self.DUMP_PATH + trading_model.FINAL_STRATEGY + ' seasonality month of year.html'
style.title = trading_model.FINAL_STRATEGY + ' month of year seasonality'
self.chart.plot(month, chart_type='line', style=style)
return month
def calculate_vol_adjusted_index_from_prices(self, prices_df, br):
"""Adjusts an index of prices for a vol target
Parameters
----------
br : BacktestRequest
Parameters for the backtest specifying start date, finish data, transaction costs etc.
asset_a_df : pandas.DataFrame
Asset prices to be traded
Returns
-------
pandas.Dataframe containing vol adjusted index
"""
calculations = Calculations()
returns_df, leverage_df = self.calculate_vol_adjusted_returns(prices_df, br, returns=False)
return calculations.create_mult_index(returns_df)
def calculate_vol_adjusted_returns(self, returns_df, br, returns=True):
"""Adjusts returns for a vol target
Parameters
----------
br : BacktestRequest
Parameters for the backtest specifying start date, finish data, transaction costs etc.
returns_a_df : pandas.DataFrame
Asset returns to be traded
Returns
-------
pandas.DataFrame
"""
calculations = Calculations()
if not returns: returns_df = calculations.calculate_returns(returns_df)
if not (hasattr(br, 'portfolio_vol_resample_type')):
br.portfolio_vol_resample_type = 'mean'
if not (hasattr(br, 'portfolio_vol_resample_freq')):
br.portfolio_vol_resample_freq = None
if not (hasattr(br, 'portfolio_vol_period_shift')):
br.portfolio_vol_period_shift = 0
leverage_df = self.calculate_leverage_factor(returns_df,
br.portfolio_vol_target, br.portfolio_vol_max_leverage,
br.portfolio_vol_periods, br.portfolio_vol_obs_in_year,
br.portfolio_vol_rebalance_freq, br.portfolio_vol_resample_freq,
br.portfolio_vol_resample_type,
period_shift=br.portfolio_vol_period_shift)
vol_returns_df = calculations.calculate_signal_returns_with_tc_matrix(leverage_df, returns_df, tc=br.spot_tc_bp)
vol_returns_df.columns = returns_df.columns
return vol_returns_df, leverage_df
def monthly_seasonality(self, data_frame,
cum = True,
add_average = False, price_index = False):
calculations = Calculations()
if price_index:
data_frame = data_frame.resample('BM').mean() # resample into month end
data_frame = calculations.calculate_returns(data_frame)
data_frame.index = pandas.to_datetime(data_frame.index)
monthly_seasonality = calculations.average_by_month(data_frame)
if add_average:
monthly_seasonality['Avg'] = monthly_seasonality.mean(axis=1)
if cum is True:
monthly_seasonality.loc[0] = numpy.zeros(len(monthly_seasonality.columns))
monthly_seasonality = monthly_seasonality.sort_index()
monthly_seasonality = calculations.create_mult_index(monthly_seasonality)
return monthly_seasonality
def create_tech_ind(self,
data_frame_non_nan,
name,
tech_params,
data_frame_non_nan_early=None):
self._signal = None
self._techind = None
if tech_params.fillna:
data_frame = data_frame_non_nan.fillna(method="ffill")
else:
data_frame = data_frame_non_nan
if data_frame_non_nan_early is not None:
data_frame_early = data_frame_non_nan_early.fillna(method="ffill")
if name == "SMA":
if (data_frame_non_nan_early is not None):
# calculate the lagged sum of the n-1 point
if pd.__version__ < '0.17':
rolling_sum = pd.rolling_sum(
data_frame.shift(1).rolling,
window=tech_params.sma_period - 1)
else:
rolling_sum = data_frame.shift(1).rolling(
center=False, window=tech_params.sma_period - 1).sum()
# add non-nan one for today
rolling_sum = rolling_sum + data_frame_early
# calculate average = sum / n
self._techind = rolling_sum / tech_params.sma_period
narray = np.where(data_frame_early > self._techind, 1, -1)
else:
if pd.__version__ < '0.17':
self._techind = pd.rolling_sum(
data_frame, window=tech_params.sma_period)
else:
self._techind = data_frame.rolling(
window=tech_params.sma_period, center=False).mean()
narray = np.where(data_frame > self._techind, 1, -1)
self._signal = pd.DataFrame(index=data_frame.index, data=narray)
self._signal.loc[0:tech_params.sma_period] = np.nan
self._signal.columns = [
x + " SMA Signal" for x in data_frame.columns.values
]
self._techind.columns = [
x + " SMA" for x in data_frame.columns.values
]
elif name == "EMA":
# self._techind = pd.ewma(data_frame, span = tech_params.ema_period)
self._techind = data_frame.ewm(ignore_na=False,
span=tech_params.ema_period,
min_periods=0,
adjust=True).mean()
narray = np.where(data_frame > self._techind, 1, -1)
self._signal = pd.DataFrame(index=data_frame.index, data=narray)
self._signal.loc[0:tech_params.ema_period] = np.nan
self._signal.columns = [
x + " EMA Signal" for x in data_frame.columns.values
]
self._techind.columns = [
x + " EMA" for x in data_frame.columns.values
]
elif name == "ROC":
if (data_frame_non_nan_early is not None):
self._techind = data_frame_early / \
data_frame.shift(tech_params.roc_period) - 1
else:
self._techind = data_frame / \
data_frame.shift(tech_params.roc_period) - 1
narray = np.where(self._techind > 0, 1, -1)
self._signal = pd.DataFrame(index=data_frame.index, data=narray)
self._signal.loc[0:tech_params.roc_period] = np.nan
self._signal.columns = [
x + " ROC Signal" for x in data_frame.columns.values
]
self._techind.columns = [
x + " ROC" for x in data_frame.columns.values
]
elif name == "polarity":
self._techind = data_frame
narray = np.where(self._techind > 0, 1, -1)
self._signal = pd.DataFrame(index=data_frame.index, data=narray)
self._signal.columns = [
x + " Polarity Signal" for x in data_frame.columns.values
]
self._techind.columns = [
x + " Polarity" for x in data_frame.columns.values
]
elif name == "SMA2":
sma = data_frame.rolling(window=tech_params.sma_period,
center=False).mean()
sma2 = data_frame.rolling(window=tech_params.sma2_period,
center=False).mean()
narray = np.where(sma > sma2, 1, -1)
self._signal = pd.DataFrame(index=data_frame.index, data=narray)
self._signal.columns = [
x + " SMA2 Signal" for x in data_frame.columns.values
]
sma.columns = [x + " SMA" for x in data_frame.columns.values]
sma2.columns = [x + " SMA2" for x in data_frame.columns.values]
most = max(tech_params.sma_period, tech_params.sma2_period)
self._signal.loc[0:most] = np.nan
self._techind = pd.concat([sma, sma2], axis=1)
elif name in ['RSI']:
# delta = data_frame.diff()
#
# dUp, dDown = delta.copy(), delta.copy()
# dUp[dUp < 0] = 0
# dDown[dDown > 0] = 0
#
# rolUp = pd.rolling_mean(dUp, tech_params.rsi_period)
# rolDown = pd.rolling_mean(dDown, tech_params.rsi_period).abs()
#
# rsi = rolUp / rolDown
# Get the difference in price from previous step
delta = data_frame.diff()
# Get rid of the first row, which is NaN since it did not have a previous
# row to calculate the differences
delta = delta[1:]
# Make the positive gains (up) and negative gains (down) Series
up, down = delta.copy(), delta.copy()
up[up < 0] = 0
down[down > 0] = 0
# Calculate the EWMA
roll_up1 = pd.stats.moments.ewma(up, tech_params.rsi_period)
roll_down1 = pd.stats.moments.ewma(down.abs(),
tech_params.rsi_period)
# Calculate the RSI based on EWMA
RS1 = roll_up1 / roll_down1
RSI1 = 100.0 - (100.0 / (1.0 + RS1))
# Calculate the SMA
roll_up2 = up.rolling(window=tech_params.rsi_period,
center=False).mean()
roll_down2 = down.abs().rolling(window=tech_params.rsi_period,
center=False).mean()
# Calculate the RSI based on SMA
RS2 = roll_up2 / roll_down2
RSI2 = 100.0 - (100.0 / (1.0 + RS2))
self._techind = RSI2
self._techind.columns = [
x + " RSI" for x in data_frame.columns.values
]
signal = data_frame.copy()
sells = (signal.shift(-1) <
tech_params.rsi_lower) & (signal > tech_params.rsi_lower)
buys = (signal.shift(-1) >
tech_params.rsi_upper) & (signal < tech_params.rsi_upper)
# print (buys[buys == True])
# buys
signal[buys] = 1
signal[sells] = -1
signal[~(buys | sells)] = np.nan
signal = signal.fillna(method='ffill')
self._signal = signal
self._signal.loc[0:tech_params.rsi_period] = np.nan
self._signal.columns = [
x + " RSI Signal" for x in data_frame.columns.values
]
elif name in ["BB"]:
# calcuate Bollinger bands
mid = data_frame.rolling(center=False,
window=tech_params.bb_period).mean()
mid.columns = [x + " BB Mid" for x in data_frame.columns.values]
std_dev = data_frame.rolling(center=False,
window=tech_params.bb_period).std()
BB_std = tech_params.bb_mult * std_dev
lower = pd.DataFrame(data=mid.values - BB_std.values,
index=mid.index,
columns=data_frame.columns)
upper = pd.DataFrame(data=mid.values + BB_std.values,
index=mid.index,
columns=data_frame.columns)
# calculate signals
signal = data_frame.copy()
buys = signal > upper
sells = signal < lower
signal[buys] = 1
signal[sells] = -1
signal[~(buys | sells)] = np.nan
signal = signal.fillna(method='ffill')
self._signal = signal
self._signal.loc[0:tech_params.bb_period] = np.nan
self._signal.columns = [
x + " " + name + " Signal" for x in data_frame.columns.values
]
lower.columns = [
x + " BB Lower" for x in data_frame.columns.values
]
upper.columns = [x + " BB Mid" for x in data_frame.columns.values]
upper.columns = [
x + " BB Lower" for x in data_frame.columns.values
]
self._techind = pd.concat([lower, mid, upper], axis=1)
elif name == "long-only":
# have +1 signals only
self._techind = data_frame # the technical indicator is just "prices"
narray = np.ones((len(data_frame.index), len(data_frame.columns)))
self._signal = pd.DataFrame(index=data_frame.index, data=narray)
self._signal.columns = [
x + " Long Only Signal" for x in data_frame.columns.values
]
self._techind.columns = [
x + " Long Only" for x in data_frame.columns.values
]
elif name == "ATR":
# get all the asset names (assume we have names 'close', 'low', 'high' in the Data)
# keep ordering of assets
asset_name = list(
OrderedDict.fromkeys(
[x.split('.')[0] for x in data_frame.columns]))
df = []
# can improve the performance of this if vectorise more!
for a in asset_name:
close = [a + '.close']
low = [a + '.low']
high = [a + '.high']
# if we don't fill NaNs, we need to remove those rows and then
# calculate the ATR
if not (tech_params.fillna):
data_frame_short = data_frame[[close[0], low[0], high[0]]]
data_frame_short = data_frame_short.dropna()
else:
data_frame_short = data_frame
prev_close = data_frame_short[close].shift(1)
c1 = data_frame_short[high].values - \
data_frame_short[low].values
c2 = np.abs(data_frame_short[high].values - prev_close.values)
c3 = np.abs(data_frame_short[low].values - prev_close.values)
true_range = np.max((c1, c2, c3), axis=0)
true_range = pd.DataFrame(index=data_frame_short.index,
data=true_range,
columns=[close[0] + ' True Range'])
# put back NaNs into ATR if necessary
if (not (tech_params.fillna)):
true_range = true_range.reindex(data_frame.index,
fill_value=np.nan)
df.append(true_range)
calc = Calculations()
true_range = calc.pandas_outer_join(df)
self._techind = true_range.rolling(window=tech_params.atr_period,
center=False).mean()
# self._techind = true_range.ewm(ignore_na=False, span=tech_params.atr_period, min_periods=0, adjust=True).mean()
self._techind.columns = [x + ".close ATR" for x in asset_name]
elif name in ["VWAP"]:
asset_name = list(
OrderedDict.fromkeys(
[x.split('.')[0] for x in data_frame.columns]))
df = []
for a in asset_name:
high = [a + '.high']
low = [a + '.low']
close = [a + '.close']
volume = [a + '.volume']
if not tech_params.fillna:
df_mod = data_frame[[high[0], low[0], close[0], volume[0]]]
df_mod.dropna(inplace=True)
else:
df_mod = data_frame
l = df_mod[low].values
h = df_mod[high].values
c = df_mod[close].values
v = df_mod[volume].values
vwap = np.cumsum(((h + l + c) / 3) * v) / np.cumsum(v)
vwap = pd.DataFrame(index=df_mod.index,
data=vwap,
columns=[close[0] + ' VWAP'])
print(vwap.columns)
if not tech_params.fillna:
vwap = vwap.reindex(data_frame.index, fill_value=np.nan)
df.append(vwap)
calc = Calculations()
vwap = calc.pandas_outer_join(df)
self._techind = vwap
self._techind.columns = [x + ".close VWAP" for x in asset_name]
self.create_custom_tech_ind(data_frame_non_nan, name, tech_params,
data_frame_non_nan_early)
# TODO create other indicators
if hasattr(tech_params, 'only_allow_longs'):
self._signal[self._signal < 0] = 0
# TODO create other indicators
if hasattr(tech_params, 'only_allow_shorts'):
self._signal[self._signal > 0] = 0
# apply signal multiplier (typically to flip signals)
if hasattr(tech_params, 'signal_mult'):
self._signal = self._signal * tech_params.signal_mult
if hasattr(tech_params, 'strip_signal_name'):
if tech_params.strip_signal_name:
self._signal.columns = data_frame.columns
return self._techind, self._signal
class FXOptionsCurve(object):
"""Constructs continuous forwards time series total return indices from underlying forwards contracts.
"""
def __init__(
self,
market_data_generator=None,
fx_options_trading_tenor=market_constants.fx_options_trading_tenor,
roll_days_before=market_constants.fx_options_roll_days_before,
roll_event=market_constants.fx_options_roll_event,
construct_via_currency='no',
fx_options_tenor_for_interpolation=market_constants.
fx_options_tenor_for_interpolation,
base_depos_tenor=data_constants.base_depos_tenor,
roll_months=market_constants.fx_options_roll_months,
cum_index=market_constants.fx_options_cum_index,
strike=market_constants.fx_options_index_strike,
contract_type=market_constants.fx_options_index_contract_type,
premium_output=market_constants.fx_options_index_premium_output,
position_multiplier=1,
depo_tenor_for_option=market_constants.fx_options_depo_tenor,
freeze_implied_vol=market_constants.fx_options_freeze_implied_vol,
tot_label='',
cal=None,
output_calculation_fields=market_constants.
output_calculation_fields):
"""Initializes FXForwardsCurve
Parameters
----------
market_data_generator : MarketDataGenerator
Used for downloading market data
fx_options_trading_tenor : str
What is primary forward contract being used to trade (default - '1M')
roll_days_before : int
Number of days before roll event to enter into a new forwards contract
roll_event : str
What constitutes a roll event? ('month-end', 'quarter-end', 'year-end', 'expiry')
cum_index : str
In total return index, do we compute in additive or multiplicative way ('add' or 'mult')
construct_via_currency : str
What currency should we construct the forward via? Eg. if we asked for AUDJPY we can construct it via
AUDUSD & JPYUSD forwards, as opposed to AUDJPY forwards (default - 'no')
fx_options_tenor_for_interpolation : str(list)
Which forwards should we use for interpolation
base_depos_tenor : str(list)
Which base deposits tenors do we need (this is only necessary if we want to start inferring depos)
roll_months : int
After how many months should we initiate a roll. Typically for trading 1M this should 1, 3M this should be 3
etc.
tot_label : str
Postfix for the total returns field
cal : str
Calendar to use for expiry (if None, uses that of FX pair)
output_calculation_fields : bool
Also output additional data should forward expiries etc. alongside total returns indices
"""
self._market_data_generator = market_data_generator
self._calculations = Calculations()
self._calendar = Calendar()
self._filter = Filter()
self._fx_options_trading_tenor = fx_options_trading_tenor
self._roll_days_before = roll_days_before
self._roll_event = roll_event
self._construct_via_currency = construct_via_currency
self._fx_options_tenor_for_interpolation = fx_options_tenor_for_interpolation
self._base_depos_tenor = base_depos_tenor
self._roll_months = roll_months
self._cum_index = cum_index
self._contact_type = contract_type
self._strike = strike
self._premium_output = premium_output
self._position_multiplier = position_multiplier
self._depo_tenor_for_option = depo_tenor_for_option
self._freeze_implied_vol = freeze_implied_vol
self._tot_label = tot_label
self._cal = cal
self._output_calculation_fields = output_calculation_fields
def generate_key(self):
from findatapy.market.ioengine import SpeedCache
# Don't include any "large" objects in the key
return SpeedCache().generate_key(self, [
'_market_data_generator', '_calculations', '_calendar', '_filter'
])
def fetch_continuous_time_series(self,
md_request,
market_data_generator,
fx_options_trading_tenor=None,
roll_days_before=None,
roll_event=None,
construct_via_currency=None,
fx_options_tenor_for_interpolation=None,
base_depos_tenor=None,
roll_months=None,
cum_index=None,
strike=None,
contract_type=None,
premium_output=None,
position_multiplier=None,
depo_tenor_for_option=None,
freeze_implied_vol=None,
tot_label=None,
cal=None,
output_calculation_fields=None):
if market_data_generator is None:
market_data_generator = self._market_data_generator
if fx_options_trading_tenor is None:
fx_options_trading_tenor = self._fx_options_trading_tenor
if roll_days_before is None: roll_days_before = self._roll_days_before
if roll_event is None: roll_event = self._roll_event
if construct_via_currency is None:
construct_via_currency = self._construct_via_currency
if fx_options_tenor_for_interpolation is None:
fx_options_tenor_for_interpolation = self._fx_options_tenor_for_interpolation
if base_depos_tenor is None: base_depos_tenor = self._base_depos_tenor
if roll_months is None: roll_months = self._roll_months
if strike is None: strike = self._strike
if contract_type is None: contract_type = self._contact_type
if premium_output is None: premium_output = self._premium_output
if position_multiplier is None:
position_multiplier = self._position_multiplier
if depo_tenor_for_option is None:
depo_tenor_for_option = self._depo_tenor_for_option
if freeze_implied_vol is None:
freeze_implied_vol = self._freeze_implied_vol
if tot_label is None: tot_label = self._tot_label
if cal is None: cal = self._cal
if output_calculation_fields is None:
output_calculation_fields = self._output_calculation_fields
# Eg. we construct EURJPY via EURJPY directly (note: would need to have sufficient options/forward data for this)
if construct_via_currency == 'no':
# Download FX spot, FX forwards points and base depos etc.
market = Market(market_data_generator=market_data_generator)
md_request_download = MarketDataRequest(md_request=md_request)
fx_conv = FXConv()
# CAREFUL: convert the tickers to correct notation, eg. USDEUR => EURUSD, because our data
# should be fetched in correct convention
md_request_download.tickers = [
fx_conv.correct_notation(x) for x in md_request.tickers
]
md_request_download.category = 'fx-vol-market'
md_request_download.fields = 'close'
md_request_download.abstract_curve = None
md_request_download.fx_options_tenor = fx_options_tenor_for_interpolation
md_request_download.base_depos_tenor = base_depos_tenor
# md_request_download.base_depos_currencies = []
forwards_market_df = market.fetch_market(md_request_download)
# Now use the original tickers
return self.construct_total_return_index(
md_request.tickers,
forwards_market_df,
fx_options_trading_tenor=fx_options_trading_tenor,
roll_days_before=roll_days_before,
roll_event=roll_event,
fx_options_tenor_for_interpolation=
fx_options_tenor_for_interpolation,
roll_months=roll_months,
cum_index=cum_index,
strike=strike,
contract_type=contract_type,
premium_output=premium_output,
position_multiplier=position_multiplier,
freeze_implied_vol=freeze_implied_vol,
depo_tenor_for_option=depo_tenor_for_option,
tot_label=tot_label,
cal=cal,
output_calculation_fields=output_calculation_fields)
else:
# eg. we calculate via your domestic currency such as USD, so returns will be in your domestic currency
# Hence AUDJPY would be calculated via AUDUSD and JPYUSD (subtracting the difference in returns)
total_return_indices = []
for tick in md_request.tickers:
base = tick[0:3]
terms = tick[3:6]
md_request_base = MarketDataRequest(md_request=md_request)
md_request_base.tickers = base + construct_via_currency
md_request_terms = MarketDataRequest(md_request=md_request)
md_request_terms.tickers = terms + construct_via_currency
# Construct the base and terms separately (ie. AUDJPY => AUDUSD & JPYUSD)
base_vals = self.fetch_continuous_time_series(
md_request_base,
market_data_generator,
fx_options_trading_tenor=fx_options_trading_tenor,
roll_days_before=roll_days_before,
roll_event=roll_event,
fx_options_tenor_for_interpolation=
fx_options_tenor_for_interpolation,
base_depos_tenor=base_depos_tenor,
roll_months=roll_months,
cum_index=cum_index,
strike=strike,
contract_type=contract_type,
premium_output=premium_output,
position_multiplier=position_multiplier,
depo_tenor_for_option=depo_tenor_for_option,
freeze_implied_vol=freeze_implied_vol,
tot_label=tot_label,
cal=cal,
output_calculation_fields=output_calculation_fields,
construct_via_currency='no')
terms_vals = self.fetch_continuous_time_series(
md_request_terms,
market_data_generator,
fx_options_trading_tenor=fx_options_trading_tenor,
roll_days_before=roll_days_before,
roll_event=roll_event,
fx_options_tenor_for_interpolation=
fx_options_tenor_for_interpolation,
base_depos_tenor=base_depos_tenor,
roll_months=roll_months,
cum_index=cum_index,
strike=strike,
contract_type=contract_type,
position_multiplier=position_multiplier,
depo_tenor_for_option=depo_tenor_for_option,
freeze_implied_vol=freeze_implied_vol,
tot_label=tot_label,
cal=cal,
output_calculation_fields=output_calculation_fields,
construct_via_currency='no')
# Special case for USDUSD case (and if base or terms USD are USDUSD
if base + terms == construct_via_currency + construct_via_currency:
base_rets = self._calculations.calculate_returns(base_vals)
cross_rets = pd.DataFrame(0,
index=base_rets.index,
columns=base_rets.columns)
elif base + construct_via_currency == construct_via_currency + construct_via_currency:
cross_rets = -self._calculations.calculate_returns(
terms_vals)
elif terms + construct_via_currency == construct_via_currency + construct_via_currency:
cross_rets = self._calculations.calculate_returns(
base_vals)
else:
base_rets = self._calculations.calculate_returns(base_vals)
terms_rets = self._calculations.calculate_returns(
terms_vals)
cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0)
# First returns of a time series will by NaN, given we don't know previous point
cross_rets.iloc[0] = 0
cross_vals = self._calculations.create_mult_index(cross_rets)
cross_vals.columns = [tick + '-option-tot.close']
total_return_indices.append(cross_vals)
return self._calculations.pandas_outer_join(total_return_indices)
def unhedged_asset_fx(self,
assets_df,
asset_currency,
home_curr,
start_date,
finish_date,
spot_df=None):
pass
def hedged_asset_fx(self,
assets_df,
asset_currency,
home_curr,
start_date,
finish_date,
spot_df=None,
total_return_indices_df=None):
pass
def get_day_count_conv(self, currency):
if currency in market_constants.currencies_with_365_basis:
return 365.0
return 360.0
def construct_total_return_index(self,
cross_fx,
market_df,
fx_options_trading_tenor=None,
roll_days_before=None,
roll_event=None,
roll_months=None,
cum_index=None,
strike=None,
contract_type=None,
premium_output=None,
position_multiplier=None,
fx_options_tenor_for_interpolation=None,
freeze_implied_vol=None,
depo_tenor_for_option=None,
tot_label=None,
cal=None,
output_calculation_fields=None):
if fx_options_trading_tenor is None:
fx_options_trading_tenor = self._fx_options_trading_tenor
if roll_days_before is None: roll_days_before = self._roll_days_before
if roll_event is None: roll_event = self._roll_event
if roll_months is None: roll_months = self._roll_months
if cum_index is None: cum_index = self._cum_index
if strike is None: strike = self._strike
if contract_type is None: contract_type = self._contact_type
if premium_output is None: premium_output = self._premium_output
if position_multiplier is None:
position_multiplier = self._position_multiplier
if fx_options_tenor_for_interpolation is None:
fx_options_tenor_for_interpolation = self._fx_options_tenor_for_interpolation
if freeze_implied_vol is None:
freeze_implied_vol = self._freeze_implied_vol
if depo_tenor_for_option is None:
depo_tenor_for_option = self._depo_tenor_for_option
if tot_label is None: tot_label = self._tot_label
if cal is None: cal = self._cal
if output_calculation_fields is None:
output_calculation_fields = self._output_calculation_fields
if not (isinstance(cross_fx, list)):
cross_fx = [cross_fx]
total_return_index_df_agg = []
# Remove columns where there is no data (because these vols typically aren't quoted)
market_df = market_df.dropna(how='all', axis=1)
fx_options_pricer = FXOptionsPricer(premium_output=premium_output)
def get_roll_date(horizon_d, expiry_d, asset_hols, month_adj=0):
if roll_event == 'month-end':
roll_d = horizon_d + CustomBusinessMonthEnd(
roll_months + month_adj, holidays=asset_hols)
# Special case so always rolls on month end date, if specify 0 days
if roll_days_before > 0:
return (roll_d - CustomBusinessDay(n=roll_days_before,
holidays=asset_hols))
elif roll_event == 'expiry-date':
roll_d = expiry_d
# Special case so always rolls on expiry date, if specify 0 days
if roll_days_before > 0:
return (roll_d - CustomBusinessDay(n=roll_days_before,
holidays=asset_hols))
return roll_d
for cross in cross_fx:
if cal is None:
cal = cross
# Eg. if we specify USDUSD
if cross[0:3] == cross[3:6]:
total_return_index_df_agg.append(
pd.DataFrame(100,
index=market_df.index,
columns=[cross + "-option-tot.close"]))
else:
# Is the FX cross in the correct convention
old_cross = cross
cross = FXConv().correct_notation(cross)
# TODO also specification of non-standard crosses like USDGBP
if old_cross != cross:
pass
fx_vol_surface = FXVolSurface(
market_df=market_df,
asset=cross,
tenors=fx_options_tenor_for_interpolation,
depo_tenor=depo_tenor_for_option)
horizon_date = market_df.index
expiry_date = []
roll_date = []
new_trade = np.full(len(horizon_date), False, dtype=bool)
asset_holidays = self._calendar.get_holidays(cal=cross)
# Get first expiry date
expiry_date.append(
self._calendar.get_expiry_date_from_horizon_date(
pd.DatetimeIndex([horizon_date[0]]),
fx_options_trading_tenor,
cal=cal,
asset_class='fx-vol')[0])
# For first month want it to expire within that month (for consistency), hence month_adj=0 ONLY here
roll_date.append(
get_roll_date(horizon_date[0],
expiry_date[0],
asset_holidays,
month_adj=0))
# New trade => entry at beginning AND on every roll
new_trade[0] = True
# Get all the expiry dates and roll dates
# At each "roll/trade" day we need to reset them for the new contract
for i in range(1, len(horizon_date)):
# If the horizon date has reached the roll date (from yesterday), we're done, and we have a
# new roll/trade
if (horizon_date[i] - roll_date[i - 1]).days >= 0:
new_trade[i] = True
else:
new_trade[i] = False
# If we're entering a new trade/contract, we need to get new expiry and roll dates
if new_trade[i]:
exp = self._calendar.get_expiry_date_from_horizon_date(
pd.DatetimeIndex([horizon_date[i]]),
fx_options_trading_tenor,
cal=cal,
asset_class='fx-vol')[0]
# Make sure we don't expire on a date in the history where there isn't market data
# It is ok for future values to expire after market data (just not in the backtest!)
if exp not in market_df.index:
exp_index = market_df.index.searchsorted(exp)
if exp_index < len(market_df.index):
exp_index = min(exp_index,
len(market_df.index))
exp = market_df.index[exp_index]
expiry_date.append(exp)
roll_date.append(
get_roll_date(horizon_date[i], expiry_date[i],
asset_holidays))
else:
# Otherwise use previous expiry and roll dates, because we're still holding same contract
expiry_date.append(expiry_date[i - 1])
roll_date.append(roll_date[i - 1])
# Note: may need to add discount factor when marking to market option
mtm = np.zeros(len(horizon_date))
calculated_strike = np.zeros(len(horizon_date))
interpolated_option = np.zeros(len(horizon_date))
implied_vol = np.zeros(len(horizon_date))
delta = np.zeros(len(horizon_date))
# For debugging
df_temp = pd.DataFrame()
df_temp['expiry-date'] = expiry_date
df_temp['horizon-date'] = horizon_date
df_temp['roll-date'] = roll_date
# Special case: for first day of history (given have no previous positions)
option_values_, spot_, strike_, vol_, delta_, expiry_date_, intrinsic_values_ = \
fx_options_pricer.price_instrument(cross, horizon_date[0], strike, expiry_date[0],
contract_type=contract_type,
tenor=fx_options_trading_tenor,
fx_vol_surface=fx_vol_surface,
return_as_df=False)
interpolated_option[0] = option_values_
calculated_strike[0] = strike_
implied_vol[0] = vol_
mtm[0] = 0
# Now price options for rest of history
# On rolling dates: MTM will be the previous option contract (interpolated)
# On non-rolling dates: it will be the current option contract
for i in range(1, len(horizon_date)):
if new_trade[i]:
# Price option trade being exited
option_values_, spot_, strike_, vol_, delta_, expiry_date_, intrinsic_values_ = \
fx_options_pricer.price_instrument(cross, horizon_date[i], calculated_strike[i-1], expiry_date[i-1],
contract_type=contract_type,
tenor=fx_options_trading_tenor,
fx_vol_surface=fx_vol_surface,
return_as_df=False)
# Store as MTM
mtm[i] = option_values_ # option_output[cross + '-option-price.close'].values
# Price new option trade being entered
option_values_, spot_, strike_, vol_, delta_, expiry_date_, intrinsic_values_ = \
fx_options_pricer.price_instrument(cross, horizon_date[i], strike, expiry_date[i],
contract_type=contract_type,
tenor=fx_options_trading_tenor,
fx_vol_surface=fx_vol_surface,
return_as_df=False)
calculated_strike[
i] = strike_ # option_output[cross + '-strike.close'].values
implied_vol[i] = vol_
interpolated_option[
i] = option_values_ # option_output[cross + '-option-price.close'].values
else:
# Price current option trade
# - strike/expiry the same as yesterday
# - other market inputs taken live, closer to expiry
calculated_strike[i] = calculated_strike[i - 1]
if freeze_implied_vol:
frozen_vol = implied_vol[i - 1]
else:
frozen_vol = None
option_values_, spot_, strike_, vol_, delta_, expiry_date_, intrinsic_values_ = \
fx_options_pricer.price_instrument(cross, horizon_date[i], calculated_strike[i],
expiry_date[i],
vol=frozen_vol,
contract_type=contract_type,
tenor=fx_options_trading_tenor,
fx_vol_surface=fx_vol_surface,
return_as_df=False)
interpolated_option[
i] = option_values_ # option_output[cross + '-option-price.close'].values
implied_vol[i] = vol_
mtm[i] = interpolated_option[i]
delta[
i] = delta_ # option_output[cross + '-delta.close'].values
# Calculate delta hedging P&L
spot_rets = (market_df[cross + ".close"] /
market_df[cross + ".close"].shift(1) - 1).values
if tot_label == '':
tot_rets = spot_rets
else:
tot_rets = (
market_df[cross + "-" + tot_label + ".close"] /
market_df[cross + "-" + tot_label + ".close"].shift(1)
- 1).values
# Remember to take the inverted sign, eg. if call is +20%, we need to -20% of spot to flatten delta
# Also invest for whether we are long or short the option
delta_hedging_pnl = -np.roll(
delta, 1) * tot_rets * position_multiplier
delta_hedging_pnl[0] = 0
# Calculate options P&L (given option premium is already percentage, only need to subtract)
# Again need to invert if we are short option
option_rets = (mtm - np.roll(interpolated_option,
1)) * position_multiplier
option_rets[0] = 0
# Calculate option + delta hedging P&L
option_delta_rets = delta_hedging_pnl + option_rets
if cum_index == 'mult':
cum_rets = 100 * np.cumprod(1.0 + option_rets)
cum_delta_rets = 100 * np.cumprod(1.0 + delta_hedging_pnl)
cum_option_delta_rets = 100 * np.cumprod(1.0 +
option_delta_rets)
elif cum_index == 'add':
cum_rets = 100 + 100 * np.cumsum(option_rets)
cum_delta_rets = 100 + 100 * np.cumsum(delta_hedging_pnl)
cum_option_delta_rets = 100 + 100 * np.cumsum(
option_delta_rets)
total_return_index_df = pd.DataFrame(
index=horizon_date, columns=[cross + "-option-tot.close"])
total_return_index_df[cross + "-option-tot.close"] = cum_rets
if output_calculation_fields:
total_return_index_df[
cross +
'-interpolated-option.close'] = interpolated_option
total_return_index_df[cross + '-mtm.close'] = mtm
total_return_index_df[cross +
'-implied-vol.close'] = implied_vol
total_return_index_df[cross + '-roll.close'] = new_trade
total_return_index_df[cross + '.roll-date'] = roll_date
total_return_index_df[cross + '.expiry-date'] = expiry_date
total_return_index_df[
cross + '-calculated-strike.close'] = calculated_strike
total_return_index_df[cross +
'-option-return.close'] = option_rets
total_return_index_df[cross +
'-spot-return.close'] = spot_rets
total_return_index_df[cross +
'-tot-return.close'] = tot_rets
total_return_index_df[cross + '-delta.close'] = delta
total_return_index_df[
cross + '-delta-pnl-return.close'] = delta_hedging_pnl
total_return_index_df[
cross + '-delta-pnl-index.close'] = cum_delta_rets
total_return_index_df[
cross +
'-option-delta-return.close'] = option_delta_rets
total_return_index_df[
cross +
'-option-delta-tot.close'] = cum_option_delta_rets
total_return_index_df_agg.append(total_return_index_df)
return self._calculations.pandas_outer_join(total_return_index_df_agg)
def apply_tc_to_total_return_index(self,
cross_fx,
total_return_index_orig_df,
option_tc_bp,
spot_tc_bp,
cum_index=None):
if cum_index is None: cum_index = self._cum_index
total_return_index_df_agg = []
if not (isinstance(cross_fx, list)):
cross_fx = [cross_fx]
option_tc = option_tc_bp / (2 * 100 * 100)
spot_tc = spot_tc_bp / (2 * 100 * 100)
total_return_index_df = total_return_index_orig_df.copy()
for cross in cross_fx:
p = abs(total_return_index_df[cross +
'-roll.close'].shift(1)) * option_tc
q = abs(total_return_index_df[cross + '-delta.close'] -
total_return_index_df[cross +
'-delta.close'].shift(1)) * spot_tc
# Additional columns to include P&L with transaction costs
total_return_index_df[cross + '-option-return-with-tc.close'] = \
total_return_index_df[cross + '-option-return.close'] - abs(total_return_index_df[cross + '-roll.close'].shift(1)) * option_tc
total_return_index_df[cross + '-delta-pnl-return-with-tc.close'] = \
total_return_index_df[cross + '-delta-pnl-return.close'] \
- abs(total_return_index_df[cross + '-delta.close'] - total_return_index_df[cross + '-delta.close'].shift(1)) * spot_tc
total_return_index_df[cross +
'-option-return-with-tc.close'][0] = 0
total_return_index_df[cross +
'-delta-pnl-return-with-tc.close'][0] = 0
total_return_index_df[cross + '-option-delta-return-with-tc.close'] = \
total_return_index_df[cross + '-option-return-with-tc.close'] + total_return_index_df[cross + '-delta-pnl-return-with-tc.close']
if cum_index == 'mult':
cum_rets = 100 * np.cumprod(1.0 + total_return_index_df[
cross + '-option-return-with-tc.close'].values)
cum_delta_rets = 100 * np.cumprod(1.0 + total_return_index_df[
cross + '-delta-pnl-return-with-tc.close'].values)
cum_option_delta_rets = 100 * np.cumprod(
1.0 + total_return_index_df[
cross + '-option-delta-return-with-tc.close'].values)
elif cum_index == 'add':
cum_rets = 100 + 100 * np.cumsum(total_return_index_df[
cross + '-option-return-with-tc.close'].values)
cum_delta_rets = 100 + 100 * np.cumsum(total_return_index_df[
cross + '-delta-pnl-return-with-tc.close'].values)
cum_option_delta_rets = 100 + 100 * np.cumsum(
total_return_index_df[
cross + '-option-delta-return-with-tc.close'].values)
total_return_index_df[cross +
"-option-tot-with-tc.close"] = cum_rets
total_return_index_df[
cross + '-delta-pnl-index-with-tc.close'] = cum_delta_rets
total_return_index_df[
cross +
'-option-delta-tot-with-tc.close'] = cum_option_delta_rets
total_return_index_df_agg.append(total_return_index_df)
return self._calculations.pandas_outer_join(total_return_index_df_agg)
def fetch_market(self, md_request=None):
"""Fetches market data for specific tickers
The user does not need to know to the low level API for each data provider works. The MarketDataRequest
needs to supply parameters that define each data request. It has details which include:
ticker eg. EURUSD
field eg. close
category eg. fx
data_source eg. bloomberg
start_date eg. 01 Jan 2015
finish_date eg. 01 Jan 2017
It can also have many optional attributes, such as
vendor_ticker eg. EURUSD Curncy
vendor_field eg. PX_LAST
Parameters
----------
md_request : MarketDataRequest
Describing what market data to fetch
Returns
-------
pandas.DataFrame
Contains the requested market data
"""
if self.md_request is not None:
md_request = self.md_request
key = md_request.generate_key()
data_frame = None
# if internet_load has been specified don't bother going to cache (might end up calling lower level cache though
# through MarketDataGenerator
if 'cache_algo' in md_request.cache_algo:
data_frame = self.speed_cache.get_dataframe(key)
if data_frame is not None:
return data_frame
# special cases when a predefined category has been asked
if md_request.category is not None:
if (md_request.category == 'fx-spot-volume'
and md_request.data_source == 'quandl'):
# NOT CURRENTLY IMPLEMENTED FOR FUTURE USE
from findatapy.market.fxclsvolume import FXCLSVolume
fxcls = FXCLSVolume(
market_data_generator=self.market_data_generator)
data_frame = fxcls.get_fx_volume(
md_request.start_date,
md_request.finish_date,
md_request.tickers,
cut="LOC",
data_source="quandl",
cache_algo=md_request.cache_algo)
# for FX we have special methods for returning cross rates or total returns
if (md_request.category == 'fx' or md_request.category
== 'fx-tot') and md_request.tickers is not None:
fxcf = FXCrossFactory(
market_data_generator=self.market_data_generator)
if md_request.category == 'fx':
type = 'spot'
elif md_request.category == 'fx-tot':
type = 'tot'
if (md_request.freq != 'tick' and md_request.fields == ['close']) or \
(md_request.freq == 'tick' and md_request.data_source in ['dukascopy', 'fxcm']):
data_frame = fxcf.get_fx_cross(
md_request.start_date,
md_request.finish_date,
md_request.tickers,
cut=md_request.cut,
data_source=md_request.data_source,
freq=md_request.freq,
cache_algo=md_request.cache_algo,
type=type,
environment=md_request.environment,
fields=md_request.fields)
# for implied volatility we can return the full surface
if (md_request.category == 'fx-implied-vol'):
if md_request.tickers is not None and md_request.freq == 'daily':
df = []
fxvf = FXVolFactory(
market_data_generator=self.market_data_generator)
for t in md_request.tickers:
if len(t) == 6:
df.append(
fxvf.get_fx_implied_vol(
md_request.start_date,
md_request.finish_date,
t,
fxvf.tenor,
cut=md_request.cut,
data_source=md_request.data_source,
part=fxvf.part,
cache_algo=md_request.cache_algo))
if df != []:
data_frame = Calculations().pandas_outer_join(df)
# for FX vol market return all the market data necessarily for pricing options
# which includes FX spot, volatility surface, forward points, deposit rates
if (md_request.category == 'fx-vol-market'):
if md_request.tickers is not None:
df = []
fxcf = FXCrossFactory(
market_data_generator=self.market_data_generator)
fxvf = FXVolFactory(
market_data_generator=self.market_data_generator)
rates = RatesFactory(
market_data_generator=self.market_data_generator)
# for each FX cross fetch the spot, vol and forward points
for t in md_request.tickers:
if len(t) == 6:
df.append(
fxcf.get_fx_cross(
start=md_request.start_date,
end=md_request.finish_date,
cross=t,
cut=md_request.cut,
data_source=md_request.data_source,
freq=md_request.freq,
cache_algo=md_request.cache_algo,
type='spot',
environment=md_request.environment,
fields=['close']))
df.append(
fxvf.get_fx_implied_vol(
md_request.start_date,
md_request.finish_date,
t,
fxvf.tenor,
cut=md_request.cut,
data_source=md_request.data_source,
part=fxvf.part,
cache_algo=md_request.cache_algo))
df.append(
rates.get_fx_forward_points(
md_request.start_date,
md_request.finish_date,
t,
fxvf.tenor,
cut=md_request.cut,
data_source=md_request.data_source,
cache_algo=md_request.cache_algo))
# lastly fetch the base depos
df.append(
rates.get_base_depos(
md_request.start_date,
md_request.finish_date,
["USD", "EUR", "CHF", "GBP"],
fxvf.tenor,
cut=md_request.cut,
data_source=md_request.data_source,
cache_algo=md_request.cache_algo))
if df != []:
data_frame = Calculations().pandas_outer_join(df)
if md_request.abstract_curve is not None:
data_frame = md_request.abstract_curve.fetch_continuous_time_series\
(md_request, self.market_data_generator)
if (md_request.category == 'crypto'):
# add more features later
data_frame = self.market_data_generator.fetch_market_data(
md_request)
# TODO add more special examples here for different asset classes
# the idea is that we do all the market data downloading here, rather than elsewhere
# by default: pass the market data request to MarketDataGenerator
if data_frame is None:
data_frame = self.market_data_generator.fetch_market_data(
md_request)
# special case where we can sometimes have duplicated data times
if md_request.freq == 'intraday' and md_request.cut == 'BSTP':
data_frame = self.filter.remove_duplicate_indices(data_frame)
# push into cache
self.speed_cache.put_dataframe(key, data_frame)
return data_frame
def run_strategy_returns_stats(self, trading_model, index=None, engine='finmarketpy'):
"""Plots useful statistics for the trading strategy using various backends
Parameters
----------
trading_model : TradingModel
defining trading strategy
engine : str
'pyfolio' - use PyFolio as a backend
'finmarketpy' - use finmarketpy as a backend
index: DataFrame
define strategy by a time series
"""
if index is None:
pnl = trading_model.strategy_pnl()
else:
pnl = index
tz = Timezone()
calculations = Calculations()
if engine == 'pyfolio':
# PyFolio assumes UTC time based DataFrames (so force this localisation)
try:
pnl = tz.localise_index_as_UTC(pnl)
except:
pass
# set the matplotlib style sheet & defaults
# at present this only works in Matplotlib engine
try:
import matplotlib
import matplotlib.pyplot as plt
matplotlib.rcdefaults()
plt.style.use(ChartConstants().chartfactory_style_sheet['chartpy-pyfolio'])
except:
pass
# TODO for intraday strategies, make daily
# convert DataFrame (assumed to have only one column) to Series
pnl = calculations.calculate_returns(pnl)
pnl = pnl.dropna()
pnl = pnl[pnl.columns[0]]
fig = pf.create_returns_tear_sheet(pnl, return_fig=True)
try:
plt.savefig(trading_model.DUMP_PATH + "stats.png")
except:
pass
plt.show()
elif engine == 'finmarketpy':
# assume we have TradingModel
# to do to take in a time series
from chartpy import Canvas, Chart
# temporarily make scale factor smaller so fits the window
old_scale_factor = trading_model.SCALE_FACTOR
trading_model.SCALE_FACTOR = 0.75
pnl = trading_model.plot_strategy_pnl(silent_plot=True) # plot the final strategy
individual = trading_model.plot_strategy_group_pnl_trades(
silent_plot=True) # plot the individual trade P&Ls
pnl_comp = trading_model.plot_strategy_group_benchmark_pnl(
silent_plot=True) # plot all the cumulative P&Ls of each component
ir_comp = trading_model.plot_strategy_group_benchmark_pnl_ir(
silent_plot=True) # plot all the IR of each component
leverage = trading_model.plot_strategy_leverage(silent_plot=True) # plot the leverage of the portfolio
ind_lev = trading_model.plot_strategy_group_leverage(silent_plot=True) # plot all the individual leverages
canvas = Canvas([[pnl, individual],
[pnl_comp, ir_comp],
[leverage, ind_lev]]
)
canvas.generate_canvas(page_title=trading_model.FINAL_STRATEGY + ' Return Statistics',
silent_display=False, canvas_plotter='plain',
output_filename=trading_model.FINAL_STRATEGY + ".html", render_pdf=False)
trading_model.SCALE_FACTOR = old_scale_factor
import pandas as pd
# For plotting
from chartpy import Chart, Style
# For loading market data
from findatapy.market import Market, MarketDataGenerator, MarketDataRequest
from findatapy.timeseries import Calculations, Calendar
from findatapy.util.loggermanager import LoggerManager
logger = LoggerManager().getLogger(__name__)
chart = Chart(engine='plotly')
market = Market(market_data_generator=MarketDataGenerator())
calculations = Calculations()
# Choose run_example = 0 for everything
# run_example = 1 - get forwards prices for AUDUSD interpolated for an odd date/broken date
# run_example = 2 - get implied deposit rate
run_example = 2
from finmarketpy.curve.rates.fxforwardspricer import FXForwardsPricer
###### Value forwards for AUDUSD for odd days
if run_example == 1 or run_example == 0:
cross = 'AUDUSD'
fx_forwards_tenors = ['1W', '2W', '3W', '1M']
def get_fx_volume(self, start, end, currency_pairs, cut="LOC", source="quandl",
cache_algo="internet_load_return"):
"""Gets forward points for specified cross, tenor and part of surface
Parameters
----------
start_date : str
start date of download
end_date : str
end data of download
cross : str
asset to be calculated
tenor : str
tenor to calculate
cut : str
closing time of data
source : str
source of data eg. bloomberg
Returns
-------
pandas.DataFrame
"""
market_data_generator = self.market_data_generator
if isinstance(currency_pairs, str): currency_pairs = [currency_pairs]
tickers = []
market_data_request = MarketDataRequest(
start_date=start, finish_date=end,
data_source=source,
category='fx-spot-volume',
freq='daily',
cut=cut,
tickers=currency_pairs,
fields = ['0h','1h','2h','3h','4h','5h','6h','7h','8h','9h','10h','11h','12h','13h','14h','15h','16h','17h','18h','19h','20h',
'21h','22h','23h'],
cache_algo=cache_algo,
environment='backtest'
)
data_frame = market_data_generator.fetch_market_data(market_data_request)
data_frame.index.name = 'Date'
data_frame.index = pandas.DatetimeIndex(data_frame.index)
df_list = []
for t in currency_pairs:
df = None
for i in range(0, 24):
txt = str(i)
df1 = pandas.DataFrame(data_frame[t + "." + txt + 'h'].copy())
df1.columns = [t + '.volume']
df1.index = df1.index + pandas.DateOffset(hours=i)
if df is None:
df = df1
else:
df = df.append(df1)
df = df.sort_index()
df_list.append(df)
data_frame_new = Calculations().pandas_outer_join(df_list)
import pytz
data_frame_new = data_frame_new.tz_localize(pytz.utc)
return data_frame_new
def run_strategy_returns_stats(self,
trading_model,
index=None,
engine='pyfolio'):
"""
run_strategy_returns_stats - Plots useful statistics for the trading strategy (using PyFolio)
Parameters
----------
trading_model : TradingModel
defining trading strategy
index: DataFrame
define strategy by a time series
"""
if index is None:
pnl = trading_model.get_strategy_pnl()
else:
pnl = index
tz = Timezone()
calculations = Calculations()
if engine == 'pyfolio':
# PyFolio assumes UTC time based DataFrames (so force this localisation)
try:
pnl = tz.localise_index_as_UTC(pnl)
except:
pass
# set the matplotlib style sheet & defaults
# at present this only works in Matplotlib engine
try:
matplotlib.rcdefaults()
plt.style.use(ChartConstants().
chartfactory_style_sheet['chartpy-pyfolio'])
except:
pass
# TODO for intraday strategies, make daily
# convert DataFrame (assumed to have only one column) to Series
pnl = calculations.calculate_returns(pnl)
pnl = pnl.dropna()
pnl = pnl[pnl.columns[0]]
fig = pf.create_returns_tear_sheet(pnl, return_fig=True)
try:
plt.savefig(trading_model.DUMP_PATH + "stats.png")
except:
pass
plt.show()
elif engine == 'finmarketpy':
# assume we have TradingModel
# to do to take in a time series
from chartpy import Canvas, Chart
pnl = trading_model.plot_strategy_pnl(
silent_plot=True) # plot the final strategy
individual = trading_model.plot_strategy_group_pnl_trades(
silent_plot=True) # plot the individual trade P&Ls
pnl_comp = trading_model.plot_strategy_group_benchmark_pnl(
silent_plot=True
) # plot all the cumulative P&Ls of each component
ir_comp = trading_model.plot_strategy_group_benchmark_pnl_ir(
silent_plot=True) # plot all the IR of each component
leverage = trading_model.plot_strategy_leverage(
silent_plot=True) # plot the leverage of the portfolio
ind_lev = trading_model.plot_strategy_group_leverage(
silent_plot=True) # plot all the individual leverages
canvas = Canvas([[pnl, individual], [pnl_comp, ir_comp],
[leverage, ind_lev]])
canvas.generate_canvas(silent_display=False,
canvas_plotter='plain')
def plot_chart(self,
tickers=None,
tickers_rhs=None,
start_date=None,
finish_date=None,
chart_file=None,
chart_type='line',
title='',
fields={'close': 'PX_LAST'},
freq='daily',
source='Web',
brand_label='Cuemacro',
display_brand_label=True,
reindex=False,
additive_index=False,
yoy=False,
plotly_plot_mode='offline_png',
quandl_api_key=dataconstants.quandl_api_key,
fred_api_key=dataconstants.fred_api_key,
alpha_vantage_api_key=dataconstants.alpha_vantage_api_key,
df=None):
if start_date is None:
start_date = datetime.datetime.utcnow().date() - timedelta(days=60)
if finish_date is None:
finish_date = datetime.datetime.utcnow()
if isinstance(tickers, str):
tickers = {tickers: tickers}
elif isinstance(tickers, list):
tickers_dict = {}
for t in tickers:
tickers_dict[t] = t
tickers = tickers_dict
if tickers_rhs is not None:
if isinstance(tickers_rhs, str):
tickers_rhs = {tickers_rhs: tickers_rhs}
elif isinstance(tickers, list):
tickers_rhs_dict = {}
for t in tickers_rhs:
tickers_rhs_dict[t] = t
tickers_rhs = tickers_rhs_dict
tickers.update(tickers_rhs)
else:
tickers_rhs = {}
if df is None:
md_request = MarketDataRequest(
start_date=start_date,
finish_date=finish_date,
freq=freq,
data_source=self._data_source,
tickers=list(tickers.keys()),
vendor_tickers=list(tickers.values()),
fields=list(fields.keys()),
vendor_fields=list(fields.values()),
quandl_api_key=quandl_api_key,
fred_api_key=fred_api_key,
alpha_vantage_api_key=alpha_vantage_api_key)
df = self._market.fetch_market(md_request=md_request)
df = df.fillna(method='ffill')
df.columns = [x.split('.')[0] for x in df.columns]
style = Style(title=title,
chart_type=chart_type,
html_file_output=chart_file,
scale_factor=-1,
height=400,
width=600,
file_output=datetime.date.today().strftime("%Y%m%d") +
" " + title + ".png",
plotly_plot_mode=plotly_plot_mode,
source=source,
brand_label=brand_label,
display_brand_label=display_brand_label)
if reindex:
df = Calculations().create_mult_index_from_prices(df)
style.y_title = 'Reindexed from 100'
if additive_index:
df = (df - df.shift(1)).cumsum()
style.y_title = 'Additive changes from 0'
if yoy:
if freq == 'daily':
obs_in_year = 252
elif freq == 'intraday':
obs_in_year = 1440
df_rets = Calculations().calculate_returns(df)
df = Calculations().average_by_annualised_year(
df_rets, obs_in_year=obs_in_year) * 100
style.y_title = 'Annualized % YoY'
if list(tickers_rhs.keys()) != []:
style.y_axis_2_series = list(tickers_rhs.keys())
style.y_axis_2_showgrid = False
style.y_axis_showgrid = False
return self._chart.plot(df, style=style), df
class VolStats(object):
"""Arranging underlying volatility market in easier to read format. Also provides methods for calculating various
volatility metrics, such as realized_vol volatility and volatility risk premium. Has extensive support for estimating
implied_vol volatility addons.
"""
def __init__(self, market_df=None, intraday_spot_df=None):
self._market_df = market_df
self._intraday_spot_df = intraday_spot_df
self._calculations = Calculations()
self._timezone = Timezone()
self._filter = Filter()
def calculate_realized_vol(self,
asset,
spot_df=None,
returns_df=None,
tenor_label="ON",
freq='daily',
freq_min_mult=1,
hour_of_day=10,
minute_of_day=0,
field='close',
returns_calc='simple',
timezone_hour_minute='America/New_York'):
"""Calculates rolling realized vol with daily cutoffs either using daily spot data or intraday spot data
(which is assumed to be in UTC timezone)
Parameters
----------
asset : str
asset to be calculated
spot_df : pd.DataFrame
minute spot returns (freq_min_mult should be the same as the frequency and should have timezone set)
tenor_label : str
tenor to calculate
freq_min_mult : int
frequency multiply for data (1 = 1 min)
hour_of_day : closing time of data in the timezone specified
eg. 10 which is 1000 time (default = 10)
minute_of_day : closing time of data in the timezone specified
eg. 0 which is 0 time (default = 0)
field : str
By default 'close'
returns_calc : str
'simple' calculate simple returns
'log' calculate log returns
timezone_hour_minute : str
The timezone for the closing hour/minute (default: 'America/New_York')
Returns
-------
pd.DataFrame of realized volatility
"""
if returns_df is None:
if spot_df is None:
if freq == 'daily':
spot_df = self._market_df[asset + "." + field]
else:
spot_df = self._intraday_spot_df[asset + "." + field]
if returns_calc == 'simple':
returns_df = self._calculations.calculate_returns(spot_df)
else:
returns_df = self._calculations.calculate_log_returns(spot_df)
cal = Calendar()
tenor_days = cal.get_business_days_tenor(tenor_label)
if freq == 'intraday':
# Annualization factor (1440 is number of minutes in the day)
mult = int(1440.0 / float(freq_min_mult))
realized_rolling = self._calculations.rolling_volatility(
returns_df, tenor_days * mult, obs_in_year=252 * mult)
# Convert to NYC time (or whatever timezone hour is specified in)
realized_rolling = self._timezone.convert_index_aware_to_alt(
realized_rolling, timezone_hour_minute)
realized_vol = self._filter.filter_time_series_by_time_of_day(
hour_of_day, minute_of_day, realized_rolling)
realized_vol = self._timezone.convert_index_aware_to_UTC_time(
realized_vol)
realized_vol = self._timezone.set_as_no_timezone(realized_vol)
elif freq == 'daily':
realized_vol = self._calculations.rolling_volatility(
spot_df, tenor_days, obs_in_year=252)
# Strip the time off the date
realized_vol.index = realized_vol.index.date
realized_vol = pd.DataFrame(realized_vol)
realized_vol.columns = [asset + 'H' + tenor_label + '.close']
return realized_vol
def calculate_vol_risk_premium(self,
asset,
tenor_label="ON",
implied_vol=None,
realized_vol=None,
field='close',
adj_ON_friday=False):
"""Calculates volatility risk premium given implied and realized quotes (ie. implied - realized) and tenor
Calculates both a version which is aligned (VRP), where the implied and realized volatilities cover
the same period (note: you will have a gap for recent points, where you can't grab future implied volatilities),
and an unaligned version (VRPV), which is the typical one used in the market
Parameters
----------
asset : str
asset to calculate value for
tenor_label : str
tenor to calculate
implied_vol : pd.DataFrame
implied vol quotes where columns are of the form eg. EURUSDV1M.close
realized_vol : pd.DataFrame
realized vol eg. EURUSDH1M.close
field : str
the field of the data to use (default: 'close')
Returns
-------
pd.DataFrame of vrp (both lagged - VRPV & contemporanous - VRP)
"""
cal = Calendar()
tenor_days = cal.get_business_days_tenor(tenor_label)
if tenor_label == 'ON' and adj_ON_friday:
implied_vol = self.adjust_implied_ON_fri_vol(implied_vol)
# Add x business days to implied_vol to make it equivalent to realized_vol (better than "shift")
# approximation for options which are not ON or 1W
# bday = CustomBusinessDay(weekmask='Mon Tue Wed Thu Fri')
implied_vol = implied_vol.copy(deep=True)
implied_unaligned = implied_vol.copy(deep=True)
cols_to_change = implied_vol.columns.values
new_cols = []
for i in range(0, len(cols_to_change)):
temp_col = list(cols_to_change[i])
temp_col[6] = 'U'
new_cols.append(''.join(temp_col))
implied_vol.columns = new_cols
## Construct volatility risk premium such that implied covers the same period as realized
# Add by number of days (note: for overnight tenors/1 week in FX we can add business days like this)
# For because they are always +1 business days, +5 business days (exc. national holidays and only including
# weekend). For longer dates like 1 month this is an approximation
implied_vol.index = [
pd.Timestamp(x) + pd.tseries.offsets.BDay(tenor_days)
for x in implied_vol.index
]
vrp = implied_vol.join(realized_vol, how='outer')
vrp[asset + "VRP" + tenor_label + ".close"] = vrp[asset + "U" + tenor_label + "." + field] \
- vrp[asset + "H" + tenor_label + "." + field]
## Construct "traditional" volatility risk premium,
# so implied does not cover the same period as realized volatility
vrp = vrp.join(implied_unaligned, how='outer')
vrp[asset + "VRPV" + tenor_label + ".close"] = \
vrp[asset + "V" + tenor_label + "." + field] - vrp[asset + "H" + tenor_label + "." + field]
return vrp
def calculate_implied_vol_addon(self,
asset,
implied_vol=None,
tenor_label='ON',
model_window=20,
model_algo='weighted-median-model',
field='close',
adj_ON_friday=True):
"""Calculates the implied volatility add on for specific tenors. The implied volatility addon can be seen as
a proxy for the event weights of large scheduled events for that day, such as the US employment report.
If there are multiple large events in the same period covered by the option, then this approach is not going
to be able to disentangle this.
Parameters
----------
asset : str
Asset to be traded (eg. EURUSD)
tenor: str
eg. ON
Returns
------
Implied volatility addon
"""
part = 'V'
if implied_vol is None:
implied_vol = self._market_df[asset + "V" + tenor_label + "." +
field]
implied_vol = implied_vol.copy(deep=True)
implied_vol = pd.DataFrame(implied_vol)
# So we eliminate impact of holidays on addons
if tenor_label == 'ON' and adj_ON_friday:
implied_vol = self.adjust_implied_ON_fri_vol(implied_vol)
implied_vol = implied_vol.dropna(
) # otherwise the moving averages get corrupted
# vol_data_avg_by_weekday = vol_data.groupby(vol_data.index.weekday).transform(lambda x: pandas.rolling_mean(x, window=10))
# Create a simple estimate for recent implied_vol volatility using multiple tenors
# vol_data_20D_avg = time_series_calcs.rolling_average(vol_data,window1)
# vol_data_10D_avg = time_series_calcs.rolling_average(vol_data,window1)
# vol_data_5D_avg = time_series_calcs.rolling_average(vol_data, window1)
if model_algo == 'weighted-median-model':
vol_data_20D_avg = self._calculations.rolling_median(
implied_vol, model_window)
vol_data_10D_avg = self._calculations.rolling_median(
implied_vol, model_window)
vol_data_5D_avg = self._calculations.rolling_median(
implied_vol, model_window)
vol_data_avg = (vol_data_20D_avg + vol_data_10D_avg +
vol_data_5D_avg) / 3
vol_data_addon = implied_vol - vol_data_avg
elif model_algo == 'weighted-mean-model':
vol_data_20D_avg = self._calculations.rolling_average(
implied_vol, model_window)
vol_data_10D_avg = self._calculations.rolling_average(
implied_vol, model_window)
vol_data_5D_avg = self._calculations.rolling_average(
implied_vol, model_window)
vol_data_avg = (vol_data_20D_avg + vol_data_10D_avg +
vol_data_5D_avg) / 3
vol_data_addon = implied_vol - vol_data_avg
# TODO add other implied vol addon models
vol_data_addon = pd.DataFrame(vol_data_addon)
implied_vol = pd.DataFrame(implied_vol)
new_cols = implied_vol.columns.values
new_cols = [
w.replace(part + tenor_label, 'ADD' + tenor_label)
for w in new_cols
]
vol_data_addon.columns = new_cols
return vol_data_addon
def adjust_implied_ON_fri_vol(self, data_frame):
cols_ON = [x for x in data_frame.columns if 'VON' in x]
for c in cols_ON:
data_frame[c][data_frame.index.dayofweek == 4] = data_frame[c][
data_frame.index.dayofweek == 4] * math.sqrt(3)
# data_frame[data_frame.index.dayofweek == 4] = data_frame[data_frame.index.dayofweek == 4] * math.sqrt(3)
return data_frame
# loading data import datetime import pandas from chartpy import Chart, Style from finmarketpy.economics import Seasonality from findatapy.market import Market, MarketDataGenerator, MarketDataRequest from chartpy.style import Style from findatapy.timeseries import Calculations from findatapy.util.loggermanager import LoggerManager seasonality = Seasonality() calc = Calculations() logger = LoggerManager().getLogger(__name__) chart = Chart(engine='matplotlib') market = Market(market_data_generator=MarketDataGenerator()) # choose run_example = 0 for everything # run_example = 1 - seasonality of gold # run_example = 2 - seasonality of FX vol # run_example = 3 - seasonality of gasoline # run_example = 4 - seasonality in NFP # run_example = 5 - seasonal adjustment in NFP run_example = 0
import pandas as pd
# For plotting
from chartpy import Chart, Style
# For loading market data
from findatapy.market import Market, MarketDataGenerator, MarketDataRequest
from findatapy.timeseries import Calculations
from findatapy.util.loggermanager import LoggerManager
logger = LoggerManager().getLogger(__name__)
chart = Chart(engine='plotly')
market = Market(market_data_generator=MarketDataGenerator())
calculations = Calculations()
# Choose run_example = 0 for everything
# run_example = 1 - creating USDTRY total return index rolling forwards and compare with BBG indices
# run_example = 2 - creating AUDJPY (via AUDUSD and JPYUSD) total return index rolling forwards & compare with BBG indices
run_example = 0
from finmarketpy.curve.fxforwardscurve import FXForwardsCurve
###### Create total return indices plot for USDBRL using forwards
# We shall be using USDBRL 1M forward contracts and rolling them 5 business days before month end
if run_example == 1 or run_example == 0:
cross = 'USDBRL'
# Download more tenors
def get_intraday_moves_over_custom_event(self, data_frame_rets, ef_time_frame, vol=False,
minute_start = 5, mins = 3 * 60, min_offset = 0 , create_index = False,
resample = False, freq = 'minutes'):
filter = Filter()
ef_time_frame = filter.filter_time_series_by_date(data_frame_rets.index[0], data_frame_rets.index[-1], ef_time_frame)
ef_time = ef_time_frame.index
if freq == 'minutes':
ef_time_start = ef_time - timedelta(minutes = minute_start)
ef_time_end = ef_time + timedelta(minutes = mins)
ann_factor = 252 * 1440
elif freq == 'days':
ef_time = ef_time_frame.index.normalize()
ef_time_start = ef_time - timedelta(days = minute_start)
ef_time_end = ef_time + timedelta(days = mins)
ann_factor = 252
ords = range(-minute_start + min_offset, mins + min_offset)
# all data needs to be equally spaced
if resample:
# make sure time series is properly sampled at 1 min intervals
data_frame_rets = data_frame_rets.resample('1min')
data_frame_rets = data_frame_rets.fillna(value = 0)
data_frame_rets = filter.remove_out_FX_out_of_hours(data_frame_rets)
data_frame_rets['Ind'] = numpy.nan
start_index = data_frame_rets.index.searchsorted(ef_time_start)
finish_index = data_frame_rets.index.searchsorted(ef_time_end)
# not all observation windows will be same length (eg. last one?)
# fill the indices which represent minutes
# TODO vectorise this!
for i in range(0, len(ef_time_frame.index)):
try:
data_frame_rets.ix[start_index[i]:finish_index[i], 'Ind'] = ords
except:
data_frame_rets.ix[start_index[i]:finish_index[i], 'Ind'] = ords[0:(finish_index[i] - start_index[i])]
# set the release dates
data_frame_rets.ix[start_index,'Rel'] = ef_time # set entry points
data_frame_rets.ix[finish_index + 1,'Rel'] = numpy.zeros(len(start_index)) # set exit points
data_frame_rets['Rel'] = data_frame_rets['Rel'].fillna(method = 'pad') # fill down signals
data_frame_rets = data_frame_rets[pandas.notnull(data_frame_rets['Ind'])] # get rid of other
data_frame = data_frame_rets.pivot(index='Ind',
columns='Rel', values=data_frame_rets.columns[0])
data_frame.index.names = [None]
if create_index:
calculations = Calculations()
data_frame.ix[-minute_start + min_offset,:] = numpy.nan
data_frame = calculations.create_mult_index(data_frame)
else:
if vol is True:
# annualise (if vol)
data_frame = data_frame.rolling(center=False,window=5).std() * math.sqrt(ann_factor)
else:
data_frame = data_frame.cumsum()
return data_frame
def run_strategy_returns_stats(self, trading_model, index = None, engine = 'pyfolio'):
"""Plots useful statistics for the trading strategy (using PyFolio)
Parameters
----------
trading_model : TradingModel
defining trading strategy
index: DataFrame
define strategy by a time series
"""
if index is None:
pnl = trading_model.get_strategy_pnl()
else:
pnl = index
tz = Timezone()
calculations = Calculations()
if engine == 'pyfolio':
# PyFolio assumes UTC time based DataFrames (so force this localisation)
try:
pnl = tz.localise_index_as_UTC(pnl)
except: pass
# set the matplotlib style sheet & defaults
# at present this only works in Matplotlib engine
try:
matplotlib.rcdefaults()
plt.style.use(ChartConstants().chartfactory_style_sheet['chartpy-pyfolio'])
except: pass
# TODO for intraday strategies, make daily
# convert DataFrame (assumed to have only one column) to Series
pnl = calculations.calculate_returns(pnl)
pnl = pnl.dropna()
pnl = pnl[pnl.columns[0]]
fig = pf.create_returns_tear_sheet(pnl, return_fig=True)
try:
plt.savefig (trading_model.DUMP_PATH + "stats.png")
except: pass
plt.show()
elif engine == 'finmarketpy':
# assume we have TradingModel
# to do to take in a time series
from chartpy import Canvas, Chart
pnl = trading_model.plot_strategy_pnl(silent_plot=True) # plot the final strategy
individual = trading_model.plot_strategy_group_pnl_trades(silent_plot=True) # plot the individual trade P&Ls
pnl_comp = trading_model.plot_strategy_group_benchmark_pnl(silent_plot=True) # plot all the cumulative P&Ls of each component
ir_comp = trading_model.plot_strategy_group_benchmark_pnl_ir(silent_plot=True) # plot all the IR of each component
leverage = trading_model.plot_strategy_leverage(silent_plot=True) # plot the leverage of the portfolio
ind_lev = trading_model.plot_strategy_group_leverage(silent_plot=True) # plot all the individual leverages
canvas = Canvas([[pnl, individual],
[pnl_comp, ir_comp],
[leverage, ind_lev]]
)
canvas.generate_canvas(silent_display=False, canvas_plotter='plain')
md_request = MarketDataRequest(
start_date=start_date, # start date
finish_date=finish_date, # finish date
category='fx',
freq='intraday', # intraday
data_source='bloomberg', # use Bloomberg as data source
tickers=['USDJPY'], # ticker (finmarketpy)
fields=['close'], # which fields to download
cache_algo='internet_load_return') # how to return data
market = Market(market_data_generator=MarketDataGenerator())
df = None
df = market.fetch_market(md_request)
calc = Calculations()
df = calc.calculate_returns(df)
# fetch NFP times from Bloomberg
md_request = MarketDataRequest(
start_date=start_date, # start date
finish_date=finish_date, # finish date
category="events",
freq='daily', # daily data
data_source='bloomberg', # use Bloomberg as data source
tickers=['NFP'],
fields=['release-date-time-full'], # which fields to download
vendor_tickers=['NFP TCH Index'], # ticker (Bloomberg)
cache_algo='internet_load_return') # how to return data
df_event_times = market.fetch_market(md_request)
def construct_strategy(self, br = None):
"""Constructs the returns for all the strategies which have been specified.
It gets backtesting parameters from fill_backtest_request (although these can be overwritten
and then market data from fill_assets
Parameters
----------
br : BacktestRequest
Parameters which define the backtest (for example start date, end date, transaction costs etc.
"""
calculations = Calculations()
# get the parameters for backtesting
if hasattr(self, 'br'):
br = self.br
elif br is None:
br = self.load_parameters()
# get market data for backtest
market_data = self.load_assets()
asset_df = market_data[0]
spot_df = market_data[1]
spot_df2 = market_data[2]
basket_dict = market_data[3]
# optional database output
contract_value_df = None
if len(market_data) == 5:
contract_value_df = market_data[4]
if hasattr(br, 'tech_params'):
tech_params = br.tech_params
else:
tech_params = TechParams()
cumresults = pandas.DataFrame(index = asset_df.index)
portleverage = pandas.DataFrame(index = asset_df.index)
from collections import OrderedDict
ret_statsresults = OrderedDict()
# each portfolio key calculate returns - can put parts of the portfolio in the key
for key in basket_dict.keys():
asset_cut_df = asset_df[[x +'.close' for x in basket_dict[key]]]
spot_cut_df = spot_df[[x +'.close' for x in basket_dict[key]]]
self.logger.info("Calculating " + key)
results, backtest = self.construct_individual_strategy(br, spot_cut_df, spot_df2, asset_cut_df, tech_params, key,
contract_value_df = contract_value_df)
cumresults[results.columns[0]] = results
portleverage[results.columns[0]] = backtest.get_portfolio_leverage()
ret_statsresults[key] = backtest.get_portfolio_pnl_ret_stats()
# for a key, designated as the final strategy save that as the "strategy"
if key == self.FINAL_STRATEGY:
self._strategy_pnl = results
self._strategy_pnl_ret_stats = backtest.get_portfolio_pnl_ret_stats()
self._strategy_leverage = backtest.get_portfolio_leverage()
# collect the position sizes and trade sizes (in several different formats)
self._strategy_signal = backtest.get_portfolio_signal()
self._strategy_trade = backtest.get_portfolio_trade()
# scaled by notional
self._strategy_signal_notional = backtest.get_portfolio_signal_notional()
self._strategy_trade_notional = backtest.get_portfolio_trade_notional()
# scaled by notional and adjusted into contract sizes
self._strategy_signal_contracts = backtest.get_portfolio_signal_contracts()
self._strategy_trade_contracts = backtest.get_portfolio_trade_contracts()
self._strategy_pnl_trades = backtest.get_pnl_trades()
# get benchmark for comparison
benchmark = self.construct_strategy_benchmark()
cumresults_benchmark = self.compare_strategy_vs_benchmark(br, cumresults, benchmark)
self._strategy_group_benchmark_ret_stats = ret_statsresults
if hasattr(self, '_benchmark_ret_stats'):
ret_statslist = ret_statsresults
ret_statslist['Benchmark'] = (self._benchmark_ret_stats)
self._strategy_group_benchmark_ret_stats = ret_statslist
# calculate annualised returns
years = calculations.average_by_annualised_year(calculations.calculate_returns(cumresults_benchmark))
self._strategy_group_pnl = cumresults
self._strategy_group_pnl_ret_stats = ret_statsresults
self._strategy_group_benchmark_pnl = cumresults_benchmark
self._strategy_group_leverage = portleverage
self._strategy_group_benchmark_annualised_pnl = years
def fetch_market(self, md_request = None):
if self.md_request is not None:
md_request = self.md_request
# special cases when a predefined category has been asked
if md_request.category is not None:
if (md_request.category == 'fx-spot-volume' and md_request.data_source == 'quandl'):
# NOT CURRENTLY IMPLEMENTED FOR FUTURE USE
from findatapy.market.fxclsvolume import FXCLSVolume
fxcls = FXCLSVolume(market_data_generator=self.market_data_generator)
return fxcls.get_fx_volume(md_request.start_date, md_request.finish_date, md_request.tickers, cut="LOC", source="quandl",
cache_algo=md_request.cache_algo)
if (md_request.category == 'fx' or md_request.category == 'fx-tot') and md_request.tickers is not None:
fxcf = FXCrossFactory(market_data_generator=self.market_data_generator)
if md_request.category == 'fx':
type = 'spot'
elif md_request.category == 'fx-tot':
type = 'tot'
if (md_request.freq != 'tick' and md_request.fields == ['close']) or (md_request.freq == 'tick' and md_request.data_source == 'dukascopy'):
return fxcf.get_fx_cross(md_request.start_date, md_request.finish_date,
md_request.tickers,
cut = md_request.cut, source = md_request.data_source, freq = md_request.freq, cache_algo=md_request.cache_algo, type = type,
environment = md_request.environment)
if (md_request.category == 'fx-implied-vol'):
if md_request.tickers is not None and md_request.freq == 'daily':
df = []
fxvf = FXVolFactory(market_data_generator=self.market_data_generator)
for t in md_request.tickers:
if len(t) == 6:
df.append(fxvf.get_fx_implied_vol(md_request.start_date, md_request.finish_date, t, fxvf.tenor,
cut=md_request.cut, source=md_request.data_source, part=fxvf.part,
cache_algo_return=md_request.cache_algo))
if df != []:
return Calculations().pandas_outer_join(df)
if(md_request.category == 'fx-vol-market'):
if md_request.tickers is not None:
df = []
fxcf = FXCrossFactory(market_data_generator=self.market_data_generator)
fxvf = FXVolFactory(market_data_generator=self.market_data_generator)
rates = RatesFactory(market_data_generator=self.market_data_generator)
for t in md_request.tickers:
if len(t) == 6:
df.append(fxcf.get_fx_cross(start=md_request.start_date, end=md_request.finish_date, cross=t,
cut=md_request.cut, source=md_request.data_source, freq=md_request.freq,
cache_algo=md_request.cache_algo, type='spot', environment=md_request.environment,
fields=['close']))
df.append(fxvf.get_fx_implied_vol(md_request.start_date, md_request.finish_date, t, fxvf.tenor,
cut=md_request.cut, source=md_request.data_source,
part=fxvf.part,
cache_algo=md_request.cache_algo))
df.append(rates.get_fx_forward_points(md_request.start_date, md_request.finish_date, t, fxvf.tenor,
cut=md_request.cut, source=md_request.data_source,
cache_algo=md_request.cache_algo))
df.append(rates.get_base_depos(md_request.start_date, md_request.finish_date, ["USD", "EUR", "CHF", "GBP"], fxvf.tenor,
cut=md_request.cut, source=md_request.data_source,
cache_algo=md_request.cache_algo
))
if df != []:
return Calculations().pandas_outer_join(df)
# TODO add more special examples here for different asset classes
# the idea is that we do all the market data downloading here, rather than elsewhere
# by default: pass the market data request to MarketDataGenerator
return self.market_data_generator.fetch_market_data(md_request)
"""
# For plotting
from chartpy import Chart, Style
# For loading market data
from findatapy.market import Market, MarketDataGenerator, MarketDataRequest
from findatapy.timeseries import Calculations
from findatapy.util.loggermanager import LoggerManager
logger = LoggerManager().getLogger(__name__)
chart = Chart(engine='plotly')
market = Market(market_data_generator=MarketDataGenerator())
calculations = Calculations()
# Choose run_example = 0 for everything
# run_example = 1 - create total return indices from FX spot data + deposit for AUDJPY, and compare
run_example = 0
from finmarketpy.curve.fxspotcurve import FXSpotCurve
###### Create total return indices plot for AUDJPY (from perspective of a USD investor)
###### Compare with AUDJPY FX spot and BBG constructed AUDJPY total return indices
if run_example == 1 or run_example == 0:
# Get AUDJPY total returns from perspective of USD investor (via AUDUSD & JPYUSD and AUD, USD & JPY overnight deposit rates)
md_request = MarketDataRequest(start_date='01 Jan 1999',
finish_date='01 Dec 2020',
def calculate_leverage_factor(self, returns_df, vol_target, vol_max_leverage, vol_periods=60, vol_obs_in_year=252,
vol_rebalance_freq='BM', data_resample_freq=None, data_resample_type='mean',
returns=True, period_shift=0):
"""Calculates the time series of leverage for a specified vol target
Parameters
----------
returns_df : DataFrame
Asset returns
vol_target : float
vol target for assets
vol_max_leverage : float
maximum leverage allowed
vol_periods : int
number of periods to calculate volatility
vol_obs_in_year : int
number of observations in the year
vol_rebalance_freq : str
how often to rebalance
vol_resample_type : str
do we need to resample the underlying data first? (eg. have we got intraday data?)
returns : boolean
is this returns time series or prices?
period_shift : int
should we delay the signal by a number of periods?
Returns
-------
pandas.Dataframe
"""
calculations = Calculations()
filter = Filter()
if data_resample_freq is not None:
return
# TODO not implemented yet
if not returns: returns_df = calculations.calculate_returns(returns_df)
roll_vol_df = calculations.rolling_volatility(returns_df,
periods=vol_periods, obs_in_year=vol_obs_in_year).shift(
period_shift)
# calculate the leverage as function of vol target (with max lev constraint)
lev_df = vol_target / roll_vol_df
lev_df[lev_df > vol_max_leverage] = vol_max_leverage
lev_df = filter.resample_time_series_frequency(lev_df, vol_rebalance_freq, data_resample_type)
returns_df, lev_df = returns_df.align(lev_df, join='left', axis=0)
lev_df = lev_df.fillna(method='ffill')
lev_df.ix[0:vol_periods] = numpy.nan # ignore the first elements before the vol window kicks in
return lev_df
def calculate_trading_PnL(self, br, asset_a_df, signal_df, contract_value_df = None):
"""Calculates P&L of a trading strategy and statistics to be retrieved later
Calculates the P&L for each asset/signal combination and also for the finally strategy applying appropriate
weighting in the portfolio, depending on predefined parameters, for example:
static weighting for each asset
static weighting for each asset + vol weighting for each asset
static weighting for each asset + vol weighting for each asset + vol weighting for the portfolio
Parameters
----------
br : BacktestRequest
Parameters for the backtest specifying start date, finish data, transaction costs etc.
asset_a_df : pandas.DataFrame
Asset prices to be traded
signal_df : pandas.DataFrame
Signals for the trading strategy
contract_value_df : pandas.DataFrame
Daily size of contracts
"""
calculations = Calculations()
# make sure the dates of both traded asset and signal are aligned properly
asset_df, signal_df = asset_a_df.align(signal_df, join='left', axis = 'index')
if (contract_value_df is not None):
asset_df, contract_value_df = asset_df.align(contract_value_df, join='left', axis='index')
contract_value_df = contract_value_df.fillna(method='ffill') # fill down asset holidays (we won't trade on these days)
# non-trading days
non_trading_days = numpy.isnan(asset_df.values)
# only allow signals to change on the days when we can trade assets
signal_df = signal_df.mask(non_trading_days) # fill asset holidays with NaN signals
signal_df = signal_df.fillna(method='ffill') # fill these down
tc = br.spot_tc_bp
signal_cols = signal_df.columns.values
asset_df_cols = asset_df.columns.values
pnl_cols = []
for i in range(0, len(asset_df_cols)):
pnl_cols.append(asset_df_cols[i] + " / " + signal_cols[i])
asset_df_copy = asset_df.copy()
asset_df = asset_df.fillna(method='ffill') # fill down asset holidays (we won't trade on these days)
returns_df = calculations.calculate_returns(asset_df)
# apply a stop loss/take profit to every trade if this has been specified
# do this before we start to do vol weighting etc.
if hasattr(br, 'take_profit') and hasattr(br, 'stop_loss'):
returns_df = calculations.calculate_returns(asset_df)
temp_strategy_rets_df = calculations.calculate_signal_returns(signal_df, returns_df)
trade_rets_df = calculations.calculate_cum_rets_trades(signal_df, temp_strategy_rets_df)
# pre_signal_df = signal_df.copy()
signal_df = calculations.calculate_risk_stop_signals(signal_df, trade_rets_df, br.stop_loss, br.take_profit)
# make sure we can't trade where asset price is undefined and carry over signal
signal_df = signal_df.mask(non_trading_days) # fill asset holidays with NaN signals
signal_df = signal_df.fillna(method='ffill') # fill these down (when asset is not trading
# signal_df.columns = [x + '_final_signal' for x in signal_df.columns]
# for debugging purposes
# if False:
# signal_df_copy = signal_df.copy()
# trade_rets_df_copy = trade_rets_df.copy()
#
# asset_df_copy.columns = [x + '_asset' for x in temp_strategy_rets_df.columns]
# temp_strategy_rets_df.columns = [x + '_strategy_rets' for x in temp_strategy_rets_df.columns]
# signal_df_copy.columns = [x + '_final_signal' for x in signal_df_copy.columns]
# trade_rets_df_copy.columns = [x + '_cum_trade' for x in trade_rets_df_copy.columns]
#
# to_plot = calculations.pandas_outer_join([asset_df_copy, pre_signal_df, signal_df_copy, trade_rets_df_copy, temp_strategy_rets_df])
# to_plot.to_csv('test.csv')
# do we have a vol target for individual signals?
if hasattr(br, 'signal_vol_adjust'):
if br.signal_vol_adjust is True:
risk_engine = RiskEngine()
if not(hasattr(br, 'signal_vol_resample_type')):
br.signal_vol_resample_type = 'mean'
if not(hasattr(br, 'signal_vol_resample_freq')):
br.signal_vol_resample_freq = None
if not(hasattr(br, 'signal_vol_period_shift')):
br.signal_vol_period_shift = 0
leverage_df = risk_engine.calculate_leverage_factor(returns_df, br.signal_vol_target, br.signal_vol_max_leverage,
br.signal_vol_periods, br.signal_vol_obs_in_year,
br.signal_vol_rebalance_freq, br.signal_vol_resample_freq,
br.signal_vol_resample_type, period_shift=br.signal_vol_period_shift)
signal_df = pandas.DataFrame(
signal_df.values * leverage_df.values, index = signal_df.index, columns = signal_df.columns)
self._individual_leverage = leverage_df # contains leverage of individual signal (before portfolio vol target)
_pnl = calculations.calculate_signal_returns_with_tc_matrix(signal_df, returns_df, tc = tc)
_pnl.columns = pnl_cols
adjusted_weights_matrix = None
# portfolio is average of the underlying signals: should we sum them or average them?
if hasattr(br, 'portfolio_combination'):
if br.portfolio_combination == 'sum':
portfolio = pandas.DataFrame(data = _pnl.sum(axis = 1), index = _pnl.index, columns = ['Portfolio'])
elif br.portfolio_combination == 'mean':
portfolio = pandas.DataFrame(data = _pnl.mean(axis = 1), index = _pnl.index, columns = ['Portfolio'])
adjusted_weights_matrix = self.create_portfolio_weights(br, _pnl, method='mean')
elif isinstance(br.portfolio_combination, dict):
# get the weights for each asset
adjusted_weights_matrix = self.create_portfolio_weights(br, _pnl, method='weighted')
portfolio = pandas.DataFrame(data=(_pnl.values * adjusted_weights_matrix), index=_pnl.index)
is_all_na = pandas.isnull(portfolio).all(axis=1)
portfolio = pandas.DataFrame(portfolio.sum(axis = 1), columns = ['Portfolio'])
# overwrite days when every asset PnL was null is NaN with nan
portfolio[is_all_na] = numpy.nan
else:
portfolio = pandas.DataFrame(data = _pnl.mean(axis = 1), index = _pnl.index, columns = ['Portfolio'])
adjusted_weights_matrix = self.create_portfolio_weights(br, _pnl, method='mean')
portfolio_leverage_df = pandas.DataFrame(data = numpy.ones(len(_pnl.index)), index = _pnl.index, columns = ['Portfolio'])
# should we apply vol target on a portfolio level basis?
if hasattr(br, 'portfolio_vol_adjust'):
if br.portfolio_vol_adjust is True:
risk_engine = RiskEngine()
portfolio, portfolio_leverage_df = risk_engine.calculate_vol_adjusted_returns(portfolio, br = br)
self._portfolio = portfolio
self._signal = signal_df # individual signals (before portfolio leverage)
self._portfolio_leverage = portfolio_leverage_df # leverage on portfolio
# multiply portfolio leverage * individual signals to get final position signals
length_cols = len(signal_df.columns)
leverage_matrix = numpy.repeat(portfolio_leverage_df.values.flatten()[numpy.newaxis,:], length_cols, 0)
# final portfolio signals (including signal & portfolio leverage)
self._portfolio_signal = pandas.DataFrame(
data = numpy.multiply(numpy.transpose(leverage_matrix), signal_df.values),
index = signal_df.index, columns = signal_df.columns)
if hasattr(br, 'portfolio_combination'):
if br.portfolio_combination == 'sum':
pass
elif br.portfolio_combination == 'mean' or isinstance(br.portfolio_combination, dict):
self._portfolio_signal = pandas.DataFrame(data=(self._portfolio_signal.values * adjusted_weights_matrix),
index=self._portfolio_signal.index,
columns=self._portfolio_signal.columns)
else:
self._portfolio_signal = pandas.DataFrame(data=(self._portfolio_signal.values * adjusted_weights_matrix),
index=self._portfolio_signal.index,
columns=self._portfolio_signal.columns)
# calculate each period of trades
self._portfolio_trade = self._portfolio_signal - self._portfolio_signal.shift(1)
self._portfolio_signal_notional = None
self._portfolio_signal_trade_notional = None
self._portfolio_signal_contracts = None
self._portfolio_signal_trade_contracts = None
# also create other measures of portfolio
# portfolio & trades in terms of a predefined notional (in USD)
# portfolio & trades in terms of contract sizes (particularly useful for futures)
if hasattr(br, 'portfolio_notional_size'):
# express positions in terms of the notional size specified
self._portfolio_signal_notional = self._portfolio_signal * br.portfolio_notional_size
self._portfolio_signal_trade_notional = self._portfolio_signal_notional - self._portfolio_signal_notional.shift(1)
# get the positions in terms of the contract sizes
notional_copy = self._portfolio_signal_notional.copy(deep=True)
notional_copy_cols = [x.split('.')[0] for x in notional_copy.columns]
notional_copy_cols = [x + '.contract-value' for x in notional_copy_cols]
notional_copy.columns = notional_copy_cols
contract_value_df = contract_value_df[notional_copy_cols]
notional_df, contract_value_df = notional_copy.align(contract_value_df, join='left', axis='index')
# careful make sure orders of magnitude are same for the notional and the contract value
self._portfolio_signal_contracts = notional_df / contract_value_df
self._portfolio_signal_contracts.columns = self._portfolio_signal_notional.columns
self._portfolio_signal_trade_contracts = self._portfolio_signal_contracts - self._portfolio_signal_contracts.shift(1)
self._pnl = _pnl # individual signals P&L
# TODO FIX very slow - hence only calculate on demand
_pnl_trades = None
# _pnl_trades = calculations.calculate_individual_trade_gains(signal_df, _pnl)
self._pnl_trades = _pnl_trades
self._ret_stats_pnl = RetStats()
self._ret_stats_pnl.calculate_ret_stats(self._pnl, br.ann_factor)
self._portfolio.columns = ['Port']
self._ret_stats_portfolio = RetStats()
self._ret_stats_portfolio.calculate_ret_stats(self._portfolio, br.ann_factor)
self._cumpnl = calculations.create_mult_index(self._pnl) # individual signals cumulative P&L
self._cumpnl.columns = pnl_cols
self._cumportfolio = calculations.create_mult_index(self._portfolio) # portfolio cumulative P&L
self._cumportfolio.columns = ['Port']
def download_intraday_tick(self, market_data_request):
"""Loads intraday time series from specified data provider
Parameters
----------
market_data_request : MarketDataRequest
contains various properties describing time series to fetched, including ticker, start & finish date etc.
Returns
-------
pandas.DataFrame
"""
data_frame_agg = None
calcuations = Calculations()
ticker_cycle = 0
data_frame_group = []
# single threaded version
# handle intraday ticker calls separately one by one
if len(market_data_request.tickers) == 1 or DataConstants().market_thread_no['other'] == 1:
for ticker in market_data_request.tickers:
market_data_request_single = copy.copy(market_data_request)
market_data_request_single.tickers = ticker
if market_data_request.vendor_tickers is not None:
market_data_request_single.vendor_tickers = [market_data_request.vendor_tickers[ticker_cycle]]
ticker_cycle = ticker_cycle + 1
# we downscale into float32, to avoid memory problems in Python (32 bit)
# data is stored on disk as float32 anyway
# old_finish_date = market_data_request_single.finish_date
#
# market_data_request_single.finish_date = self.refine_expiry_date(market_data_request)
#
# if market_data_request_single.finish_date >= market_data_request_single.start_date:
# data_frame_single = data_vendor.load_ticker(market_data_request_single)
# else:
# data_frame_single = None
#
# market_data_request_single.finish_date = old_finish_date
#
# data_frame_single = data_vendor.load_ticker(market_data_request_single)
data_frame_single = self.fetch_single_time_series(market_data_request)
# if the vendor doesn't provide any data, don't attempt to append
if data_frame_single is not None:
if data_frame_single.empty == False:
data_frame_single.index.name = 'Date'
data_frame_single = data_frame_single.astype('float32')
data_frame_group.append(data_frame_single)
# # if you call for returning multiple tickers, be careful with memory considerations!
# if data_frame_agg is not None:
# data_frame_agg = data_frame_agg.join(data_frame_single, how='outer')
# else:
# data_frame_agg = data_frame_single
# key = self.create_category_key(market_data_request, ticker)
# fname = self.create_cache_file_name(key)
# self._time_series_cache[fname] = data_frame_agg # cache in memory (disable for intraday)
# if you call for returning multiple tickers, be careful with memory considerations!
if data_frame_group is not None:
data_frame_agg = calcuations.pandas_outer_join(data_frame_group)
return data_frame_agg
else:
market_data_request_list = []
# create a list of MarketDataRequests
for ticker in market_data_request.tickers:
market_data_request_single = copy.copy(market_data_request)
market_data_request_single.tickers = ticker
if market_data_request.vendor_tickers is not None:
market_data_request_single.vendor_tickers = [market_data_request.vendor_tickers[ticker_cycle]]
ticker_cycle = ticker_cycle + 1
market_data_request_list.append(market_data_request_single)
return self.fetch_group_time_series(market_data_request_list)
tech_ind = TechIndicator()
tech_ind.create_tech_ind(spot_df, indicator, tech_params)
signal_df = tech_ind.get_signal()
# use the same data for generating signals
backtest.calculate_trading_PnL(br, asset_df, signal_df)
port = backtest.get_cumportfolio()
port.columns = [
indicator + ' = ' + str(tech_params.sma_period) + ' ' +
str(backtest.get_portfolio_pnl_desc()[0])
]
signals = backtest.get_porfolio_signal(
) # get final signals for each series
returns = backtest.get_pnl() # get P&L for each series
calculations = Calculations()
trade_returns = calculations.calculate_individual_trade_gains(
signals, returns)
print(trade_returns)
# print the last positions (we could also save as CSV etc.)
print(signals.tail(1))
style = Style()
style.title = "EUR/USD trend model"
style.source = 'Quandl'
style.scale_factor = 1
style.file_output = 'eurusd-trend-example.png'
Chart(port, style=style).plot()
class FXSpotCurve(object):
"""Construct total return (spot) indices for FX. In future will also convert assets from local currency to foreign currency
denomination and construct indices from forwards series.
"""
def __init__(self,
market_data_generator=None,
depo_tenor='ON',
construct_via_currency='no'):
self._market_data_generator = market_data_generator
self._calculations = Calculations()
self._depo_tenor = depo_tenor
self._construct_via_currency = construct_via_currency
def generate_key(self):
from findatapy.market.ioengine import SpeedCache
# Don't include any "large" objects in the key
return SpeedCache().generate_key(
self, ['_market_data_generator', '_calculations'])
def fetch_continuous_time_series(self,
md_request,
market_data_generator,
construct_via_currency=None):
if market_data_generator is None:
market_data_generator = self._market_data_generator
if construct_via_currency is None:
construct_via_currency = self._construct_via_currency
# Eg. we construct AUDJPY via AUDJPY directly
if construct_via_currency == 'no':
base_depo_tickers = [
x[0:3] + self._depo_tenor for x in md_request.tickers
]
terms_depo_tickers = [
x[3:6] + self._depo_tenor for x in md_request.tickers
]
depo_tickers = list(set(base_depo_tickers + terms_depo_tickers))
market = Market(market_data_generator=market_data_generator)
# Deposit data for base and terms currency
md_request_download = MarketDataRequest(md_request=md_request)
md_request_download.tickers = depo_tickers
md_request_download.category = 'base-depos'
md_request_download.fields = 'close'
md_request_download.abstract_curve = None
depo_df = market.fetch_market(md_request_download)
# Spot data
md_request_download.tickers = md_request.tickers
md_request_download.category = 'fx'
spot_df = market.fetch_market(md_request_download)
return self.construct_total_return_index(md_request.tickers,
self._depo_tenor, spot_df,
depo_df)
else:
# eg. we calculate via your domestic currency such as USD, so returns will be in your domestic currency
# Hence AUDJPY would be calculated via AUDUSD and JPYUSD (subtracting the difference in returns)
total_return_indices = []
for tick in md_request.tickers:
base = tick[0:3]
terms = tick[3:6]
md_request_base = MarketDataRequest(md_request=md_request)
md_request_base.tickers = base + construct_via_currency
md_request_terms = MarketDataRequest(md_request=md_request)
md_request_terms.tickers = terms + construct_via_currency
base_vals = self.fetch_continuous_time_series(
md_request_base,
market_data_generator,
construct_via_currency='no')
terms_vals = self.fetch_continuous_time_series(
md_request_terms,
market_data_generator,
construct_via_currency='no')
# Special case for USDUSD case (and if base or terms USD are USDUSD
if base + terms == 'USDUSD':
base_rets = self._calculations.calculate_returns(base_vals)
cross_rets = pd.DataFrame(0,
index=base_rets.index,
columns=base_rets.columns)
elif base + 'USD' == 'USDUSD':
cross_rets = -self._calculations.calculate_returns(
terms_vals)
elif terms + 'USD' == 'USDUSD':
cross_rets = self._calculations.calculate_returns(
base_vals)
else:
base_rets = self._calculations.calculate_returns(base_vals)
terms_rets = self._calculations.calculate_returns(
terms_vals)
cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0)
# First returns of a time series will by NaN, given we don't know previous point
cross_rets.iloc[0] = 0
cross_vals = self._calculations.create_mult_index(cross_rets)
cross_vals.columns = [tick + '-tot.close']
total_return_indices.append(cross_vals)
return self._calculations.pandas_outer_join(total_return_indices)
def unhedged_asset_fx(self,
assets_df,
asset_currency,
home_curr,
start_date,
finish_date,
spot_df=None):
pass
def hedged_asset_fx(self,
assets_df,
asset_currency,
home_curr,
start_date,
finish_date,
spot_df=None,
total_return_indices_df=None):
pass
def get_day_count_conv(self, currency):
if currency in ['AUD', 'CAD', 'GBP', 'NZD']:
return 365.0
return 360.0
def construct_total_return_index(self, cross_fx, tenor, spot_df,
deposit_df):
"""Creates total return index for selected FX crosses from spot and deposit data
Parameters
----------
cross_fx : String
Crosses to construct total return indices (can be a list)
tenor : String
Tenor of deposit rates to use to compute carry (typically ON for spot)
spot_df : pd.DataFrame
Spot data (must include crosses we select)
deposit_df : pd.DataFrame
Deposit data
Returns
-------
pd.DataFrame
"""
if not (isinstance(cross_fx, list)):
cross_fx = [cross_fx]
total_return_index_agg = []
for cross in cross_fx:
# Get the spot series, base deposit
base_deposit = deposit_df[cross[0:3] + tenor + ".close"].to_frame()
terms_deposit = deposit_df[cross[3:6] + tenor +
".close"].to_frame()
# Eg. if we specify USDUSD
if cross[0:3] == cross[3:6]:
total_return_index_agg.append(
pd.DataFrame(100,
index=base_deposit.index,
columns=[cross + "-tot.close"]))
else:
carry = base_deposit.join(terms_deposit, how='inner')
spot = spot_df[cross + ".close"].to_frame()
base_daycount = self.get_day_count_conv(cross[0:3])
terms_daycount = self.get_day_count_conv(cross[4:6])
# Align the base & terms deposits series to spot
spot, carry = spot.align(carry, join='left', axis=0)
# Sometimes depo data can be patchy, ok to fill down, given not very volatile (don't do this with spot!)
carry = carry.fillna(method='ffill') / 100.0
# In case there are values missing at start of list (fudge for old data!)
carry = carry.fillna(method='bfill')
spot = spot[cross + ".close"].to_frame()
base_deposit = carry[base_deposit.columns]
terms_deposit = carry[terms_deposit.columns]
# Calculate the time difference between each data point
spot['index_col'] = spot.index
time = spot['index_col'].diff()
spot = spot.drop('index_col', 1)
total_return_index = pd.DataFrame(
index=spot.index, columns=[cross + "-tot.close"])
total_return_index.iloc[0] = 100
time_diff = time.values.astype(
float) / 86400000000000.0 # get time difference in days
for i in range(1, len(total_return_index.index)):
# TODO vectorise this formulae or use Numba
# Calculate total return index as product of yesterday, changes in spot and carry accrued
total_return_index.values[i] = total_return_index.values[i - 1] * \
(1 + (1 + base_deposit.values[i] * time_diff[i] / base_daycount) *
(spot.values[i] / spot.values[i - 1]) \
- (1 + terms_deposit.values[i] * time_diff[i] / terms_daycount))
total_return_index_agg.append(total_return_index)
return self._calculations.pandas_outer_join(total_return_index_agg)
# Fetch USD/JPY spot
md_request = MarketDataRequest(
start_date=start_date, # start date
finish_date=finish_date, # finish date
category='fx',
freq='intraday', # intraday
data_source='bloomberg', # use Bloomberg as data source
tickers=['USDJPY'], # ticker (finmarketpy)
fields=['close'], # which fields to download
cache_algo='cache_algo_return') # how to return data
df = None
df = market.fetch_market(md_request)
calc = Calculations()
df = calc.calculate_returns(df)
es = EventStudy()
# Work out cumulative asset price moves moves over the event
df_event = es.get_intraday_moves_over_custom_event(df, df_event_times)
# Create an average move
df_event['Avg'] = df_event.mean(axis=1)
# Plotting spot over economic data event
style = Style()
style.scale_factor = 3
style.file_output = 'usdjpy-nfp.png'
# See the License for the specific language governing permissions and
# limitations under the License.
#
if __name__ == "__main__":
###### below line CRUCIAL when running Windows, otherwise multiprocessing
# doesn"t work! (not necessary on Linux)
from findatapy.util import SwimPool;
SwimPool()
from findatapy.timeseries import Filter, Calendar, Calculations
import pandas as pd
calculations = Calculations()
calendar = Calendar()
filter = Filter()
# choose run_example = 0 for everything
# run_example = 1 - combine intraday dataframe with daily data dataframe
run_example = 0
if run_example == 1 or run_example == 0:
df_intraday = pd.DataFrame(
index=pd.date_range(start="01 Jan 2020", end="10 Jan 2020",
freq="1min"),
columns=["ones"])
df_intraday["ones"] = 1
def calculate_leverage_factor(self,
returns_df,
vol_target,
vol_max_leverage,
vol_periods=60,
vol_obs_in_year=252,
vol_rebalance_freq='BM',
data_resample_freq=None,
data_resample_type='mean',
returns=True,
period_shift=0):
"""
calculate_leverage_factor - Calculates the time series of leverage for a specified vol target
Parameters
----------
returns_df : DataFrame
Asset returns
vol_target : float
vol target for assets
vol_max_leverage : float
maximum leverage allowed
vol_periods : int
number of periods to calculate volatility
vol_obs_in_year : int
number of observations in the year
vol_rebalance_freq : str
how often to rebalance
vol_resample_type : str
do we need to resample the underlying data first? (eg. have we got intraday data?)
returns : boolean
is this returns time series or prices?
period_shift : int
should we delay the signal by a number of periods?
Returns
-------
pandas.Dataframe
"""
calculations = Calculations()
filter = Filter()
if data_resample_freq is not None:
return
# TODO not implemented yet
if not returns: returns_df = calculations.calculate_returns(returns_df)
roll_vol_df = calculations.rolling_volatility(
returns_df, periods=vol_periods,
obs_in_year=vol_obs_in_year).shift(period_shift)
# calculate the leverage as function of vol target (with max lev constraint)
lev_df = vol_target / roll_vol_df
lev_df[lev_df > vol_max_leverage] = vol_max_leverage
lev_df = filter.resample_time_series_frequency(lev_df,
vol_rebalance_freq,
data_resample_type)
returns_df, lev_df = returns_df.align(lev_df, join='left', axis=0)
lev_df = lev_df.fillna(method='ffill')
lev_df.ix[
0:
vol_periods] = numpy.nan # ignore the first elements before the vol window kicks in
return lev_df
class MarketDataGenerator(object):
"""Returns market data time series by directly calling market data sources.
At present it supports Bloomberg (bloomberg), Yahoo (yahoo), Quandl (quandl), FRED (fred) etc. which are implemented
in subclasses of DataVendor class. This provides a common wrapper for all these data sources.
"""
def __init__(self):
self.config = ConfigManager().get_instance()
self.logger = LoggerManager().getLogger(__name__)
self.filter = Filter()
self.calculations = Calculations()
self.io_engine = IOEngine()
self._intraday_code = -1
self.days_expired_intraday_contract_download = -1
return
def set_intraday_code(self, code):
self._intraday_code = code
def get_data_vendor(self, source):
"""Loads appropriate data service class
Parameters
----------
source : str
the data service to use "bloomberg", "quandl", "yahoo", "google", "fred" etc.
we can also have forms like "bloomberg-boe" separated by hyphens
Returns
-------
DataVendor
"""
data_vendor = None
try:
source = source.split("-")[0]
except:
self.logger.error("Was data source specified?")
return None
if source == 'bloomberg':
try:
from findatapy.market.datavendorbbg import DataVendorBBGOpen
data_vendor = DataVendorBBGOpen()
except:
self.logger.warn("Bloomberg needs to be installed")
elif source == 'quandl':
from findatapy.market.datavendorweb import DataVendorQuandl
data_vendor = DataVendorQuandl()
elif source == 'ons':
from findatapy.market.datavendorweb import DataVendorONS
data_vendor = DataVendorONS()
elif source == 'boe':
from findatapy.market.datavendorweb import DataVendorBOE
data_vendor = DataVendorBOE()
elif source == 'dukascopy':
from findatapy.market.datavendorweb import DataVendorDukasCopy
data_vendor = DataVendorDukasCopy()
elif source == 'fxcm':
from findatapy.market.datavendorweb import DataVendorFXCM
data_vendor = DataVendorFXCM()
elif source == 'alfred':
from findatapy.market.datavendorweb import DataVendorALFRED
data_vendor = DataVendorALFRED()
elif source in ['yahoo', 'google', 'fred', 'oecd', 'eurostat', 'edgar-index']:
from findatapy.market.datavendorweb import DataVendorPandasWeb
data_vendor = DataVendorPandasWeb()
elif source == 'bitcoincharts':
from findatapy.market.datavendorweb import DataVendorBitcoincharts
data_vendor = DataVendorBitcoincharts()
elif source == 'poloniex':
from findatapy.market.datavendorweb import DataVendorPoloniex
data_vendor = DataVendorPoloniex()
elif source == 'binance':
from findatapy.market.datavendorweb import DataVendorBinance
data_vendor = DataVendorBinance()
elif source == 'bitfinex':
from findatapy.market.datavendorweb import DataVendorBitfinex
data_vendor = DataVendorBitfinex()
elif source == 'gdax':
from findatapy.market.datavendorweb import DataVendorGdax
data_vendor = DataVendorGdax()
elif source == 'kraken':
from findatapy.market.datavendorweb import DataVendorKraken
data_vendor = DataVendorKraken()
# TODO add support for other data sources (like Reuters)
return data_vendor
def fetch_market_data(self, market_data_request, kill_session = True):
"""Loads time series from specified data provider
Parameters
----------
market_data_request : MarketDataRequest
contains various properties describing time series to fetched, including ticker, start & finish date etc.
Returns
-------
pandas.DataFrame
"""
# data_vendor = self.get_data_vendor(market_data_request.data_source)
# check if tickers have been specified (if not load all of them for a category)
# also handle single tickers/list tickers
create_tickers = False
if market_data_request.vendor_tickers is not None and market_data_request.tickers is None:
market_data_request.tickers = market_data_request.vendor_tickers
tickers = market_data_request.tickers
if tickers is None :
create_tickers = True
elif isinstance(tickers, str):
if tickers == '': create_tickers = True
elif isinstance(tickers, list):
if tickers == []: create_tickers = True
if create_tickers:
market_data_request.tickers = ConfigManager().get_instance().get_tickers_list_for_category(
market_data_request.category, market_data_request.data_source, market_data_request.freq, market_data_request.cut)
# intraday or tick: only one ticker per cache file
if (market_data_request.freq in ['intraday', 'tick', 'second', 'hour', 'minute']):
data_frame_agg = self.download_intraday_tick(market_data_request)
# return data_frame_agg
# daily: multiple tickers per cache file - assume we make one API call to vendor library
else:
data_frame_agg = self.download_daily(market_data_request)
if('internet_load' in market_data_request.cache_algo):
self.logger.debug("Internet loading.. ")
# signal to data_vendor template to exit session
# if data_vendor is not None and kill_session == True: data_vendor.kill_session()
if(market_data_request.cache_algo == 'cache_algo'):
self.logger.debug("Only caching data in memory, do not return any time series."); return
# only return time series if specified in the algo
if 'return' in market_data_request.cache_algo:
# special case for events/events-dt which is not indexed like other tables (also same for downloading futures
# contracts dates)
if market_data_request.category is not None:
if 'events' in market_data_request.category:
return data_frame_agg
# pad columns a second time (is this necessary to do here again?)
# TODO only do this for not daily data?
try:
return self.filter.filter_time_series(market_data_request, data_frame_agg, pad_columns=True)
except:
if data_frame_agg is not None:
return data_frame_agg
import traceback
self.logger.warn("No data returned for " + str(market_data_request.tickers))
return None
def create_time_series_hash_key(self, market_data_request, ticker = None):
"""Creates a hash key for retrieving the time series
Parameters
----------
market_data_request : MarketDataRequest
contains various properties describing time series to fetched, including ticker, start & finish date etc.
Returns
-------
str
"""
if(isinstance(ticker, list)):
ticker = ticker[0]
return self.create_cache_file_name(MarketDataRequest().create_category_key(market_data_request, ticker))
def download_intraday_tick(self, market_data_request):
"""Loads intraday time series from specified data provider
Parameters
----------
market_data_request : MarketDataRequest
contains various properties describing time series to fetched, including ticker, start & finish date etc.
Returns
-------
pandas.DataFrame
"""
data_frame_agg = None
calcuations = Calculations()
ticker_cycle = 0
data_frame_group = []
# single threaded version
# handle intraday ticker calls separately one by one
if len(market_data_request.tickers) == 1 or DataConstants().market_thread_no['other'] == 1:
for ticker in market_data_request.tickers:
market_data_request_single = copy.copy(market_data_request)
market_data_request_single.tickers = ticker
if market_data_request.vendor_tickers is not None:
market_data_request_single.vendor_tickers = [market_data_request.vendor_tickers[ticker_cycle]]
ticker_cycle = ticker_cycle + 1
# we downscale into float32, to avoid memory problems in Python (32 bit)
# data is stored on disk as float32 anyway
# old_finish_date = market_data_request_single.finish_date
#
# market_data_request_single.finish_date = self.refine_expiry_date(market_data_request)
#
# if market_data_request_single.finish_date >= market_data_request_single.start_date:
# data_frame_single = data_vendor.load_ticker(market_data_request_single)
# else:
# data_frame_single = None
#
# market_data_request_single.finish_date = old_finish_date
#
# data_frame_single = data_vendor.load_ticker(market_data_request_single)
data_frame_single = self.fetch_single_time_series(market_data_request)
# if the vendor doesn't provide any data, don't attempt to append
if data_frame_single is not None:
if data_frame_single.empty == False:
data_frame_single.index.name = 'Date'
data_frame_single = data_frame_single.astype('float32')
data_frame_group.append(data_frame_single)
# # if you call for returning multiple tickers, be careful with memory considerations!
# if data_frame_agg is not None:
# data_frame_agg = data_frame_agg.join(data_frame_single, how='outer')
# else:
# data_frame_agg = data_frame_single
# key = self.create_category_key(market_data_request, ticker)
# fname = self.create_cache_file_name(key)
# self._time_series_cache[fname] = data_frame_agg # cache in memory (disable for intraday)
# if you call for returning multiple tickers, be careful with memory considerations!
if data_frame_group is not None:
data_frame_agg = calcuations.pandas_outer_join(data_frame_group)
return data_frame_agg
else:
market_data_request_list = []
# create a list of MarketDataRequests
for ticker in market_data_request.tickers:
market_data_request_single = copy.copy(market_data_request)
market_data_request_single.tickers = ticker
if market_data_request.vendor_tickers is not None:
market_data_request_single.vendor_tickers = [market_data_request.vendor_tickers[ticker_cycle]]
ticker_cycle = ticker_cycle + 1
market_data_request_list.append(market_data_request_single)
return self.fetch_group_time_series(market_data_request_list)
def fetch_single_time_series(self, market_data_request):
market_data_request = MarketDataRequest(md_request=market_data_request)
# only includes those tickers have not expired yet!
start_date = pandas.Timestamp(market_data_request.start_date).date()
import datetime
current_date = datetime.datetime.utcnow().date()
from datetime import timedelta
tickers = market_data_request.tickers
vendor_tickers = market_data_request.vendor_tickers
expiry_date = market_data_request.expiry_date
config = ConfigManager().get_instance()
# in many cases no expiry is defined so skip them
for i in range(0, len(tickers)):
try:
expiry_date = config.get_expiry_for_ticker(market_data_request.data_source, tickers[i])
except:
pass
if expiry_date is not None:
expiry_date = pandas.Timestamp(expiry_date).date()
# use pandas Timestamp, a bit more robust with weird dates (can fail if comparing date vs datetime)
# if the expiry is before the start date of our download don't bother downloading this ticker
if expiry_date < start_date:
tickers[i] = None
# special case for futures-contracts which are intraday
# avoid downloading if the expiry date is very far in the past
# (we need this before there might be odd situations where we run on an expiry date, but still want to get
# data right till expiry time)
if market_data_request.category == 'futures-contracts' and market_data_request.freq == 'intraday' \
and self.days_expired_intraday_contract_download > 0:
if expiry_date + timedelta(days=self.days_expired_intraday_contract_download) < current_date:
tickers[i] = None
if vendor_tickers is not None and tickers[i] is None:
vendor_tickers[i] = None
market_data_request.tickers = [e for e in tickers if e != None]
if vendor_tickers is not None:
market_data_request.vendor_tickers = [e for e in vendor_tickers if e != None]
data_frame_single = None
if len(market_data_request.tickers) > 0:
data_frame_single = self.get_data_vendor(market_data_request.data_source).load_ticker(market_data_request)
#print(data_frame_single.head(n=10))
if data_frame_single is not None:
if data_frame_single.empty == False:
data_frame_single.index.name = 'Date'
# will fail for dataframes which includes dates/strings (eg. futures contract names)
try:
data_frame_single = data_frame_single.astype('float32')
except:
self.logger.warning('Could not convert to float')
if market_data_request.freq == "second":
data_frame_single = data_frame_single.resample("1s")
return data_frame_single
def fetch_group_time_series(self, market_data_request_list):
data_frame_agg = None
thread_no = DataConstants().market_thread_no['other']
if market_data_request_list[0].data_source in DataConstants().market_thread_no:
thread_no = DataConstants().market_thread_no[market_data_request_list[0].data_source]
if thread_no > 0:
pool = SwimPool().create_pool(thread_technique = DataConstants().market_thread_technique, thread_no=thread_no)
# open the market data downloads in their own threads and return the results
result = pool.map_async(self.fetch_single_time_series, market_data_request_list)
data_frame_group = result.get()
pool.close()
pool.join()
else:
data_frame_group = []
for md_request in market_data_request_list:
data_frame_group.append(self.fetch_single_time_series(md_request))
# collect together all the time series
if data_frame_group is not None:
data_frame_group = [i for i in data_frame_group if i is not None]
# for debugging!
# import pickle
# import datetime
# pickle.dump(data_frame_group, open(str(datetime.datetime.now()).replace(':', '-').replace(' ', '-').replace(".", "-") + ".p", "wb"))
if data_frame_group is not None:
try:
data_frame_agg = self.calculations.pandas_outer_join(data_frame_group)
except Exception as e:
self.logger.warning('Possible overlap of columns? Have you specifed same ticker several times: ' + str(e))
return data_frame_agg
def download_daily(self, market_data_request):
"""Loads daily time series from specified data provider
Parameters
----------
market_data_request : MarketDataRequest
contains various properties describing time series to fetched, including ticker, start & finish date etc.
Returns
-------
pandas.DataFrame
"""
key = MarketDataRequest().create_category_key(market_data_request)
is_key_overriden = False
for k in DataConstants().override_multi_threading_for_categories:
if k in key:
is_key_overriden = True
break
# by default use other
thread_no = DataConstants().market_thread_no['other']
if market_data_request.data_source in DataConstants().market_thread_no:
thread_no = DataConstants().market_thread_no[market_data_request.data_source]
# daily data does not include ticker in the key, as multiple tickers in the same file
if thread_no == 1:
# data_frame_agg = data_vendor.load_ticker(market_data_request)
data_frame_agg = self.fetch_single_time_series(market_data_request)
else:
market_data_request_list = []
# when trying your example 'equitiesdata_example' I had a -1 result so it went out of the comming loop and I had errors in execution
group_size = max(int(len(market_data_request.tickers) / thread_no - 1),0)
if group_size == 0: group_size = 1
# split up tickers into groups related to number of threads to call
for i in range(0, len(market_data_request.tickers), group_size):
market_data_request_single = copy.copy(market_data_request)
market_data_request_single.tickers = market_data_request.tickers[i:i + group_size]
if market_data_request.vendor_tickers is not None:
market_data_request_single.vendor_tickers = \
market_data_request.vendor_tickers[i:i + group_size]
market_data_request_list.append(market_data_request_single)
# special case where we make smaller calls one after the other
if is_key_overriden:
data_frame_list = []
for md in market_data_request_list:
data_frame_list.append(self.fetch_single_time_series(md))
data_frame_agg = self.calculations.pandas_outer_join(data_frame_list)
else:
data_frame_agg = self.fetch_group_time_series(market_data_request_list)
# fname = self.create_cache_file_name(key)
# self._time_series_cache[fname] = data_frame_agg # cache in memory (ok for daily data)
return data_frame_agg
def refine_expiry_date(self, market_data_request):
# expiry date
if market_data_request.expiry_date is None:
ConfigManager().get_instance().get_expiry_for_ticker(market_data_request.data_source, market_data_request.ticker)
return market_data_request
def create_cache_file_name(self, filename):
return DataConstants().folder_time_series_data + "/" + filename
def calculate_trading_PnL(self, br, asset_a_df, signal_df):
"""
calculate_trading_PnL - Calculates P&L of a trading strategy and statistics to be retrieved later
Parameters
----------
br : BacktestRequest
Parameters for the backtest specifying start date, finish data, transaction costs etc.
asset_a_df : pandas.DataFrame
Asset prices to be traded
signal_df : pandas.DataFrame
Signals for the trading strategy
"""
calculations = Calculations()
# make sure the dates of both traded asset and signal are aligned properly
asset_df, signal_df = asset_a_df.align(signal_df,
join='left',
axis='index')
# only allow signals to change on the days when we can trade assets
signal_df = signal_df.mask(numpy.isnan(
asset_df.values)) # fill asset holidays with NaN signals
signal_df = signal_df.fillna(method='ffill') # fill these down
asset_df = asset_df.fillna(method='ffill') # fill down asset holidays
returns_df = calculations.calculate_returns(asset_df)
tc = br.spot_tc_bp
signal_cols = signal_df.columns.values
returns_cols = returns_df.columns.values
pnl_cols = []
for i in range(0, len(returns_cols)):
pnl_cols.append(returns_cols[i] + " / " + signal_cols[i])
# do we have a vol target for individual signals?
if hasattr(br, 'signal_vol_adjust'):
if br.signal_vol_adjust is True:
risk_engine = RiskEngine()
if not (hasattr(br, 'signal_vol_resample_type')):
br.signal_vol_resample_type = 'mean'
if not (hasattr(br, 'signal_vol_resample_freq')):
br.signal_vol_resample_freq = None
leverage_df = risk_engine.calculate_leverage_factor(
returns_df, br.signal_vol_target,
br.signal_vol_max_leverage, br.signal_vol_periods,
br.signal_vol_obs_in_year, br.signal_vol_rebalance_freq,
br.signal_vol_resample_freq, br.signal_vol_resample_type)
signal_df = pandas.DataFrame(signal_df.values *
leverage_df.values,
index=signal_df.index,
columns=signal_df.columns)
self._individual_leverage = leverage_df # contains leverage of individual signal (before portfolio vol target)
_pnl = calculations.calculate_signal_returns_with_tc_matrix(signal_df,
returns_df,
tc=tc)
_pnl.columns = pnl_cols
# portfolio is average of the underlying signals: should we sum them or average them?
if hasattr(br, 'portfolio_combination'):
if br.portfolio_combination == 'sum':
portfolio = pandas.DataFrame(data=_pnl.sum(axis=1),
index=_pnl.index,
columns=['Portfolio'])
elif br.portfolio_combination == 'mean':
portfolio = pandas.DataFrame(data=_pnl.mean(axis=1),
index=_pnl.index,
columns=['Portfolio'])
else:
portfolio = pandas.DataFrame(data=_pnl.mean(axis=1),
index=_pnl.index,
columns=['Portfolio'])
portfolio_leverage_df = pandas.DataFrame(data=numpy.ones(
len(_pnl.index)),
index=_pnl.index,
columns=['Portfolio'])
# should we apply vol target on a portfolio level basis?
if hasattr(br, 'portfolio_vol_adjust'):
if br.portfolio_vol_adjust is True:
risk_engine = RiskEngine()
portfolio, portfolio_leverage_df = risk_engine.calculate_vol_adjusted_returns(
portfolio, br=br)
self._portfolio = portfolio
self._signal = signal_df # individual signals (before portfolio leverage)
self._portfolio_leverage = portfolio_leverage_df # leverage on portfolio
# multiply portfolio leverage * individual signals to get final position signals
length_cols = len(signal_df.columns)
leverage_matrix = numpy.repeat(
portfolio_leverage_df.values.flatten()[numpy.newaxis, :],
length_cols, 0)
# final portfolio signals (including signal & portfolio leverage)
self._portfolio_signal = pandas.DataFrame(data=numpy.multiply(
numpy.transpose(leverage_matrix), signal_df.values),
index=signal_df.index,
columns=signal_df.columns)
if hasattr(br, 'portfolio_combination'):
if br.portfolio_combination == 'sum':
pass
elif br.portfolio_combination == 'mean':
self._portfolio_signal = self._portfolio_signal / float(
length_cols)
else:
self._portfolio_signal = self._portfolio_signal / float(
length_cols)
self._pnl = _pnl # individual signals P&L
# TODO FIX very slow - hence only calculate on demand
_pnl_trades = None
# _pnl_trades = calculations.calculate_individual_trade_gains(signal_df, _pnl)
self._pnl_trades = _pnl_trades
self._ret_stats_pnl = RetStats()
self._ret_stats_pnl.calculate_ret_stats(self._pnl, br.ann_factor)
self._portfolio.columns = ['Port']
self._ret_stats_portfolio = RetStats()
self._ret_stats_portfolio.calculate_ret_stats(self._portfolio,
br.ann_factor)
self._cumpnl = calculations.create_mult_index(
self._pnl) # individual signals cumulative P&L
self._cumpnl.columns = pnl_cols
self._cumportfolio = calculations.create_mult_index(
self._portfolio) # portfolio cumulative P&L
self._cumportfolio.columns = ['Port']
def run_arbitrary_sensitivity(self, trading_model, parameter_list=None,
pretty_portfolio_names=None, parameter_type=None, run_in_parallel=False,
reload_market_data=True):
if not(reload_market_data):
asset_df, spot_df, spot_df2, basket_dict, contract_value_df = self._load_assets(trading_model)
port_list = []
ret_stats_list = []
if market_constants.backtest_thread_no[market_constants.generic_plat] > 1 and run_in_parallel:
swim_pool = SwimPool(multiprocessing_library=market_constants.multiprocessing_library)
pool = swim_pool.create_pool(thread_technique=market_constants.backtest_thread_technique,
thread_no=market_constants.backtest_thread_no[market_constants.generic_plat])
mult_results = []
for i in range(0, len(parameter_list)):
# br = copy.copy(trading_model.load_parameters())
# reset all parameters
br = copy.copy(trading_model.load_parameters())
current_parameter = parameter_list[i]
# for calculating P&L, change the assets
for k in current_parameter.keys():
setattr(br, k, current_parameter[k])
setattr(br.tech_params, k, current_parameter[k])
# should specify reloading the data, if our parameters impact which assets we are fetching
if reload_market_data:
asset_df, spot_df, spot_df2, basket_dict, contract_value_df = self._load_assets(trading_model, br = br)
mult_results.append(
pool.apply_async(self._run_strategy, args=(trading_model, asset_df, spot_df, spot_df2, br,
contract_value_df,
pretty_portfolio_names[i],)))
for p in mult_results:
port, ret_stats = p.get()
port_list.append(port)
ret_stats_list.append(ret_stats)
try:
swim_pool.close_pool(pool)
except:
pass
else:
for i in range(0, len(parameter_list)):
# reset all parameters
br = copy.copy(trading_model.load_parameters())
current_parameter = parameter_list[i]
# for calculating P&L
for k in current_parameter.keys():
setattr(br, k, current_parameter[k])
setattr(br.tech_params, k, current_parameter[k])
# should specify reloading the data, if our parameters impact which assets we are fetching
if reload_market_data:
asset_df, spot_df, spot_df2, basket_dict, contract_value_df = self._load_assets(trading_model, br = br)
br = copy.copy(trading_model.br)
port, ret_stats = self._run_strategy(trading_model, asset_df, spot_df, spot_df2, br, contract_value_df,
pretty_portfolio_names[i])
port_list.append(port)
ret_stats_list.append(ret_stats)
port_list = Calculations().pandas_outer_join(port_list)
# reset the parameters of the strategy
trading_model.br = trading_model.load_parameters()
style = Style()
ir = [t.inforatio()[0] for t in ret_stats_list]
rets = [t.ann_returns()[0] for t in ret_stats_list]
# if we have too many combinations remove legend and use scaled shaded colour
# if len(port_list) > 10:
# style.color = 'Blues'
# style.display_legend = False
# careful with plotting labels, may need to convert to strings
pretty_portfolio_names = [str(p) for p in pretty_portfolio_names]
# plot all the variations
style.resample = 'B'
style.file_output = self.DUMP_PATH + trading_model.FINAL_STRATEGY + ' ' + parameter_type + '.png'
style.html_file_output = self.DUMP_PATH + trading_model.FINAL_STRATEGY + ' ' + parameter_type + '.html'
style.scale_factor = trading_model.SCALE_FACTOR
style.title = trading_model.FINAL_STRATEGY + ' ' + parameter_type
self.chart.plot(port_list, chart_type='line', style=style)
# plot all the IR in a bar chart form (can be easier to read!)
style = Style()
style.file_output = self.DUMP_PATH + trading_model.FINAL_STRATEGY + ' ' + parameter_type + ' IR.png'
style.html_file_output = self.DUMP_PATH + trading_model.FINAL_STRATEGY + ' ' + parameter_type + ' IR.html'
style.scale_factor = trading_model.SCALE_FACTOR
style.title = trading_model.FINAL_STRATEGY + ' ' + parameter_type
summary_ir = pandas.DataFrame(index=pretty_portfolio_names, data=ir, columns=['IR'])
self.chart.plot(summary_ir, chart_type='bar', style=style)
# plot all the rets
style.file_output = self.DUMP_PATH + trading_model.FINAL_STRATEGY + ' ' + parameter_type + ' Rets.png'
style.html_file_output = self.DUMP_PATH + trading_model.FINAL_STRATEGY + ' ' + parameter_type + ' Rets.html'
summary_rets = pandas.DataFrame(index=pretty_portfolio_names, data=rets, columns=['Rets (%)']) * 100
self.chart.plot(summary_rets, chart_type='bar', style=style)
return port_list, summary_ir, summary_rets
def construct_strategy(self, br=None):
"""
construct_strategy - Constructs the returns for all the strategies which have been specified.
- gets parameters form fill_backtest_request
- market data from fill_assets
"""
calculations = Calculations()
# get the parameters for backtesting
if hasattr(self, 'br'):
br = self.br
elif br is None:
br = self.load_parameters()
# get market data for backtest
asset_df, spot_df, spot_df2, basket_dict = self.load_assets()
if hasattr(br, 'tech_params'):
tech_params = br.tech_params
else:
tech_params = TechParams()
cumresults = pandas.DataFrame(index=asset_df.index)
portleverage = pandas.DataFrame(index=asset_df.index)
from collections import OrderedDict
ret_statsresults = OrderedDict()
# each portfolio key calculate returns - can put parts of the portfolio in the key
for key in basket_dict.keys():
asset_cut_df = asset_df[[x + '.close' for x in basket_dict[key]]]
spot_cut_df = spot_df[[x + '.close' for x in basket_dict[key]]]
self.logger.info("Calculating " + key)
results, backtest = self.construct_individual_strategy(
br, spot_cut_df, spot_df2, asset_cut_df, tech_params, key)
cumresults[results.columns[0]] = results
portleverage[results.columns[0]] = backtest.get_porfolio_leverage()
ret_statsresults[key] = backtest.get_portfolio_pnl_ret_stats()
# for a key, designated as the final strategy save that as the "strategy"
if key == self.FINAL_STRATEGY:
self._strategy_pnl = results
self._strategy_pnl_ret_stats = backtest.get_portfolio_pnl_ret_stats(
)
self._strategy_leverage = backtest.get_porfolio_leverage()
self._strategy_signal = backtest.get_porfolio_signal()
self._strategy_pnl_trades = backtest.get_pnl_trades()
# get benchmark for comparison
benchmark = self.construct_strategy_benchmark()
cumresults_benchmark = self.compare_strategy_vs_benchmark(
br, cumresults, benchmark)
self._strategy_group_benchmark_ret_stats = ret_statsresults
if hasattr(self, '_benchmark_ret_stats'):
ret_statslist = ret_statsresults
ret_statslist['Benchmark'] = (self._benchmark_ret_stats)
self._strategy_group_benchmark_ret_stats = ret_statslist
# calculate annualised returns
years = calculations.average_by_annualised_year(
calculations.calculate_returns(cumresults_benchmark))
self._strategy_group_pnl = cumresults
self._strategy_group_pnl_ret_stats = ret_statsresults
self._strategy_group_benchmark_pnl = cumresults_benchmark
self._strategy_group_leverage = portleverage
self._strategy_group_benchmark_annualised_pnl = years
class FXCrossFactory(object):
"""Generates FX spot time series and FX total return time series (assuming we already have
total return indices available from xxxUSD form) from underlying series. Can also produce cross rates from the USD
crosses.
"""
def __init__(self, market_data_generator=None):
self.logger = LoggerManager().getLogger(__name__)
self.fxconv = FXConv()
self.cache = {}
self.calculations = Calculations()
self.market_data_generator = market_data_generator
return
def get_fx_cross_tick(self,
start,
end,
cross,
cut="NYC",
data_source="dukascopy",
cache_algo='internet_load_return',
type='spot',
environment='backtest',
fields=['bid', 'ask']):
if isinstance(cross, str):
cross = [cross]
market_data_request = MarketDataRequest(
gran_freq="tick",
freq_mult=1,
freq='tick',
cut=cut,
fields=['bid', 'ask', 'bidv', 'askv'],
cache_algo=cache_algo,
environment=environment,
start_date=start,
finish_date=end,
data_source=data_source,
category='fx')
market_data_generator = self.market_data_generator
data_frame_agg = None
for cr in cross:
if (type == 'spot'):
market_data_request.tickers = cr
cross_vals = market_data_generator.fetch_market_data(
market_data_request)
# if user only wants 'close' calculate that from the bid/ask fields
if fields == ['close']:
cross_vals = cross_vals[[cr + '.bid',
cr + '.ask']].mean(axis=1)
cross_vals.columns = [cr + '.close']
else:
filter = Filter()
filter_columns = [cr + '.' + f for f in fields]
cross_vals = filter.filter_time_series_by_columns(
filter_columns, cross_vals)
if data_frame_agg is None:
data_frame_agg = cross_vals
else:
data_frame_agg = data_frame_agg.join(cross_vals, how='outer')
# strip the nan elements
data_frame_agg = data_frame_agg.dropna()
return data_frame_agg
def get_fx_cross(self,
start,
end,
cross,
cut="NYC",
data_source="bloomberg",
freq="intraday",
cache_algo='internet_load_return',
type='spot',
environment='backtest',
fields=['close']):
if data_source == "gain" or data_source == 'dukascopy' or freq == 'tick':
return self.get_fx_cross_tick(start,
end,
cross,
cut=cut,
data_source=data_source,
cache_algo=cache_algo,
type='spot',
fields=fields)
if isinstance(cross, str):
cross = [cross]
market_data_request_list = []
freq_list = []
type_list = []
for cr in cross:
market_data_request = MarketDataRequest(freq_mult=1,
cut=cut,
fields=['close'],
freq=freq,
cache_algo=cache_algo,
start_date=start,
finish_date=end,
data_source=data_source,
environment=environment)
market_data_request.type = type
market_data_request.cross = cr
if freq == 'intraday':
market_data_request.gran_freq = "minute" # intraday
elif freq == 'daily':
market_data_request.gran_freq = "daily" # daily
market_data_request_list.append(market_data_request)
data_frame_agg = []
# depends on the nature of operation as to whether we should use threading or multiprocessing library
if DataConstants().market_thread_technique is "thread":
from multiprocessing.dummy import Pool
else:
# most of the time is spend waiting for Bloomberg to return, so can use threads rather than multiprocessing
# must use the multiprocessing_on_dill library otherwise can't pickle objects correctly
# note: currently not very stable
from multiprocessing_on_dill import Pool
thread_no = DataConstants().market_thread_no['other']
if market_data_request_list[0].data_source in DataConstants(
).market_thread_no:
thread_no = DataConstants().market_thread_no[
market_data_request_list[0].data_source]
# fudge, issue with multithreading and accessing HDF5 files
# if self.market_data_generator.__class__.__name__ == 'CachedMarketDataGenerator':
# thread_no = 0
thread_no = 0
if (thread_no > 0):
pool = Pool(thread_no)
# open the market data downloads in their own threads and return the results
df_list = pool.map_async(self._get_individual_fx_cross,
market_data_request_list).get()
data_frame_agg = self.calculations.iterative_outer_join(df_list)
# data_frame_agg = self.calculations.pandas_outer_join(result.get())
try:
pool.close()
pool.join()
except:
pass
else:
for md_request in market_data_request_list:
data_frame_agg.append(
self._get_individual_fx_cross(md_request))
data_frame_agg = self.calculations.pandas_outer_join(
data_frame_agg)
# strip the nan elements
data_frame_agg = data_frame_agg.dropna(how='all')
# self.speed_cache.put_dataframe(key, data_frame_agg)
return data_frame_agg
def _get_individual_fx_cross(self, market_data_request):
cr = market_data_request.cross
type = market_data_request.type
freq = market_data_request.freq
base = cr[0:3]
terms = cr[3:6]
if (type == 'spot'):
# non-USD crosses
if base != 'USD' and terms != 'USD':
base_USD = self.fxconv.correct_notation('USD' + base)
terms_USD = self.fxconv.correct_notation('USD' + terms)
# TODO check if the cross exists in the database
# download base USD cross
market_data_request.tickers = base_USD
market_data_request.category = 'fx'
base_vals = self.market_data_generator.fetch_market_data(
market_data_request)
# download terms USD cross
market_data_request.tickers = terms_USD
market_data_request.category = 'fx'
terms_vals = self.market_data_generator.fetch_market_data(
market_data_request)
# if quoted USD/base flip to get USD terms
if (base_USD[0:3] == 'USD'):
base_vals = 1 / base_vals
# if quoted USD/terms flip to get USD terms
if (terms_USD[0:3] == 'USD'):
terms_vals = 1 / terms_vals
base_vals.columns = ['temp']
terms_vals.columns = ['temp']
cross_vals = base_vals.div(terms_vals, axis='index')
cross_vals.columns = [cr + '.close']
base_vals.columns = [base_USD + '.close']
terms_vals.columns = [terms_USD + '.close']
else:
# if base == 'USD': non_USD = terms
# if terms == 'USD': non_USD = base
correct_cr = self.fxconv.correct_notation(cr)
market_data_request.tickers = correct_cr
market_data_request.category = 'fx'
cross_vals = self.market_data_generator.fetch_market_data(
market_data_request)
# special case for USDUSD!
if base + terms == 'USDUSD':
if freq == 'daily':
cross_vals = pandas.DataFrame(
1,
index=cross_vals.index,
columns=cross_vals.columns)
filter = Filter()
cross_vals = filter.filter_time_series_by_holidays(
cross_vals, cal='WEEKDAY')
else:
# flip if not convention (eg. JPYUSD)
if (correct_cr != cr):
cross_vals = 1 / cross_vals
# cross_vals = self.market_data_generator.harvest_time_series(market_data_request)
cross_vals.columns = [cr + '.close']
elif type[0:3] == "tot":
if freq == 'daily':
# download base USD cross
market_data_request.tickers = base + 'USD'
market_data_request.category = 'fx-tot'
if type == "tot":
base_vals = self.market_data_generator.fetch_market_data(
market_data_request)
else:
x = 0
# download terms USD cross
market_data_request.tickers = terms + 'USD'
market_data_request.category = 'fx-tot'
if type == "tot":
terms_vals = self.market_data_generator.fetch_market_data(
market_data_request)
else:
pass
# base_rets = self.calculations.calculate_returns(base_vals)
# terms_rets = self.calculations.calculate_returns(terms_vals)
# special case for USDUSD case (and if base or terms USD are USDUSD
if base + terms == 'USDUSD':
base_rets = self.calculations.calculate_returns(base_vals)
cross_rets = pandas.DataFrame(0,
index=base_rets.index,
columns=base_rets.columns)
elif base + 'USD' == 'USDUSD':
cross_rets = -self.calculations.calculate_returns(
terms_vals)
elif terms + 'USD' == 'USDUSD':
cross_rets = self.calculations.calculate_returns(base_vals)
else:
base_rets = self.calculations.calculate_returns(base_vals)
terms_rets = self.calculations.calculate_returns(
terms_vals)
cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0)
# first returns of a time series will by NaN, given we don't know previous point
cross_rets.iloc[0] = 0
cross_vals = self.calculations.create_mult_index(cross_rets)
cross_vals.columns = [cr + '-tot.close']
elif freq == 'intraday':
self.logger.info(
'Total calculated returns for intraday not implemented yet'
)
return None
return cross_vals
def compare_strategy_vs_benchmark(self, br, strategy_df, benchmark_df):
"""
compare_strategy_vs_benchmark - Compares the trading strategy we are backtesting against a benchmark
Parameters
----------
br : BacktestRequest
Parameters for backtest such as start and finish dates
strategy_df : pandas.DataFrame
Strategy time series
benchmark_df : pandas.DataFrame
Benchmark time series
"""
include_benchmark = False
calc_stats = False
if hasattr(br, 'include_benchmark'):
include_benchmark = br.include_benchmark
if hasattr(br, 'calc_stats'): calc_stats = br.calc_stats
if include_benchmark:
ret_stats = RetStats()
risk_engine = RiskEngine()
filter = Filter()
calculations = Calculations()
# align strategy time series with that of benchmark
strategy_df, benchmark_df = strategy_df.align(benchmark_df,
join='left',
axis=0)
# if necessary apply vol target to benchmark (to make it comparable with strategy)
if hasattr(br, 'portfolio_vol_adjust'):
if br.portfolio_vol_adjust is True:
benchmark_df = risk_engine.calculate_vol_adjusted_index_from_prices(
benchmark_df, br=br)
# only calculate return statistics if this has been specified (note when different frequencies of data
# might underrepresent vol
# if calc_stats:
benchmark_df = benchmark_df.fillna(method='ffill')
ret_stats.calculate_ret_stats_from_prices(benchmark_df,
br.ann_factor)
if calc_stats:
benchmark_df.columns = ret_stats.summary()
# realign strategy & benchmark
strategy_benchmark_df = strategy_df.join(benchmark_df, how='inner')
strategy_benchmark_df = strategy_benchmark_df.fillna(
method='ffill')
strategy_benchmark_df = filter.filter_time_series_by_date(
br.plot_start, br.finish_date, strategy_benchmark_df)
strategy_benchmark_df = calculations.create_mult_index_from_prices(
strategy_benchmark_df)
self._benchmark_pnl = benchmark_df
self._benchmark_ret_stats = ret_stats
return strategy_benchmark_df
return strategy_df
class FXForwardsCurve(object):
"""Constructs continuous forwards time series total return indices from underlying forwards contracts.
"""
def __init__(self, market_data_generator=None, fx_forwards_trading_tenor=market_constants.fx_forwards_trading_tenor,
roll_days_before=market_constants.fx_forwards_roll_days_before,
roll_event=market_constants.fx_forwards_roll_event, construct_via_currency='no',
fx_forwards_tenor_for_interpolation=market_constants.fx_forwards_tenor_for_interpolation,
base_depos_tenor=data_constants.base_depos_tenor,
roll_months=market_constants.fx_forwards_roll_months,
cum_index=market_constants.fx_forwards_cum_index,
output_calculation_fields=market_constants.output_calculation_fields,
field='close'):
"""Initializes FXForwardsCurve
Parameters
----------
market_data_generator : MarketDataGenerator
Used for downloading market data
fx_forwards_trading_tenor : str
What is primary forward contract being used to trade (default - '1M')
roll_days_before : int
Number of days before roll event to enter into a new forwards contract
roll_event : str
What constitutes a roll event? ('month-end', 'quarter-end', 'year-end', 'expiry')
construct_via_currency : str
What currency should we construct the forward via? Eg. if we asked for AUDJPY we can construct it via
AUDUSD & JPYUSD forwards, as opposed to AUDJPY forwards (default - 'no')
fx_forwards_tenor_for_interpolation : str(list)
Which forwards should we use for interpolation
base_depos_tenor : str(list)
Which base deposits tenors do we need (this is only necessary if we want to start inferring depos)
roll_months : int
After how many months should we initiate a roll. Typically for trading 1M this should 1, 3M this should be 3
etc.
cum_index : str
In total return index, do we compute in additive or multiplicative way ('add' or 'mult')
output_calculation_fields : bool
Also output additional data should forward expiries etc. alongside total returns indices
"""
self._market_data_generator = market_data_generator
self._calculations = Calculations()
self._calendar = Calendar()
self._filter = Filter()
self._fx_forwards_trading_tenor = fx_forwards_trading_tenor
self._roll_days_before = roll_days_before
self._roll_event = roll_event
self._construct_via_currency = construct_via_currency
self._fx_forwards_tenor_for_interpolation = fx_forwards_tenor_for_interpolation
self._base_depos_tenor = base_depos_tenor
self._roll_months = roll_months
self._cum_index = cum_index
self._output_calcultion_fields = output_calculation_fields
self._field = field
def generate_key(self):
from findatapy.market.ioengine import SpeedCache
# Don't include any "large" objects in the key
return SpeedCache().generate_key(self, ['_market_data_generator', '_calculations', '_calendar', '_filter'])
def fetch_continuous_time_series(self, md_request, market_data_generator, fx_forwards_trading_tenor=None,
roll_days_before=None, roll_event=None,
construct_via_currency=None, fx_forwards_tenor_for_interpolation=None, base_depos_tenor=None,
roll_months=None, cum_index=None, output_calculation_fields=False, field=None):
if market_data_generator is None: market_data_generator = self._market_data_generator
if fx_forwards_trading_tenor is None: fx_forwards_trading_tenor = self._fx_forwards_trading_tenor
if roll_days_before is None: roll_days_before = self._roll_days_before
if roll_event is None: roll_event = self._roll_event
if construct_via_currency is None: construct_via_currency = self._construct_via_currency
if fx_forwards_tenor_for_interpolation is None: fx_forwards_tenor_for_interpolation = self._fx_forwards_tenor_for_interpolation
if base_depos_tenor is None: base_depos_tenor = self._base_depos_tenor
if roll_months is None: roll_months = self._roll_months
if cum_index is None: cum_index = self._cum_index
if output_calculation_fields is None: output_calculation_fields = self._output_calcultion_fields
if field is None: field = self._field
# Eg. we construct EURJPY via EURJPY directly (note: would need to have sufficient forward data for this)
if construct_via_currency == 'no':
# Download FX spot, FX forwards points and base depos etc.
market = Market(market_data_generator=market_data_generator)
md_request_download = MarketDataRequest(md_request=md_request)
fx_conv = FXConv()
# CAREFUL: convert the tickers to correct notation, eg. USDEUR => EURUSD, because our data
# should be fetched in correct convention
md_request_download.tickers = [fx_conv.correct_notation(x) for x in md_request.tickers]
md_request_download.category = 'fx-forwards-market'
md_request_download.fields = field
md_request_download.abstract_curve = None
md_request_download.fx_forwards_tenor = fx_forwards_tenor_for_interpolation
md_request_download.base_depos_tenor = base_depos_tenor
forwards_market_df = market.fetch_market(md_request_download)
# Now use the original tickers
return self.construct_total_return_index(md_request.tickers, forwards_market_df,
fx_forwards_trading_tenor=fx_forwards_trading_tenor,
roll_days_before=roll_days_before, roll_event=roll_event,
fx_forwards_tenor_for_interpolation=fx_forwards_tenor_for_interpolation,
roll_months=roll_months,
cum_index=cum_index,
output_calculation_fields=output_calculation_fields,
field=field)
else:
# eg. we calculate via your domestic currency such as USD, so returns will be in your domestic currency
# Hence AUDJPY would be calculated via AUDUSD and JPYUSD (subtracting the difference in returns)
total_return_indices = []
for tick in md_request.tickers:
base = tick[0:3]
terms = tick[3:6]
md_request_base = MarketDataRequest(md_request=md_request)
md_request_base.tickers = base + construct_via_currency
md_request_terms = MarketDataRequest(md_request=md_request)
md_request_terms.tickers = terms + construct_via_currency
# Construct the base and terms separately (ie. AUDJPY => AUDUSD & JPYUSD)
base_vals = self.fetch_continuous_time_series(md_request_base, market_data_generator,
fx_forwards_trading_tenor=fx_forwards_trading_tenor,
roll_days_before=roll_days_before, roll_event=roll_event,
fx_forwards_tenor_for_interpolation=fx_forwards_tenor_for_interpolation,
base_depos_tenor=base_depos_tenor,
roll_months=roll_months, output_calculation_fields=False,
cum_index=cum_index,
construct_via_currency='no',
field=field)
terms_vals = self.fetch_continuous_time_series(md_request_terms, market_data_generator,
fx_forwards_trading_tenor=fx_forwards_trading_tenor,
roll_days_before=roll_days_before, roll_event=roll_event,
fx_forwards_tenor_for_interpolation=fx_forwards_tenor_for_interpolation,
base_depos_tenor=base_depos_tenor,
roll_months=roll_months,
cum_index=cum_index,
output_calculation_fields=False,
construct_via_currency='no',
field=field)
# Special case for USDUSD case (and if base or terms USD are USDUSD
if base + terms == construct_via_currency + construct_via_currency:
base_rets = self._calculations.calculate_returns(base_vals)
cross_rets = pd.DataFrame(0, index=base_rets.index, columns=base_rets.columns)
elif base + construct_via_currency == construct_via_currency + construct_via_currency:
cross_rets = -self._calculations.calculate_returns(terms_vals)
elif terms + construct_via_currency == construct_via_currency + construct_via_currency:
cross_rets = self._calculations.calculate_returns(base_vals)
else:
base_rets = self._calculations.calculate_returns(base_vals)
terms_rets = self._calculations.calculate_returns(terms_vals)
cross_rets = base_rets.sub(terms_rets.iloc[:, 0], axis=0)
# First returns of a time series will by NaN, given we don't know previous point
cross_rets.iloc[0] = 0
cross_vals = self._calculations.create_mult_index(cross_rets)
cross_vals.columns = [tick + '-forward-tot.' + field]
total_return_indices.append(cross_vals)
return self._calculations.join(total_return_indices, how='outer')
def unhedged_asset_fx(self, assets_df, asset_currency, home_curr, start_date, finish_date, spot_df=None):
pass
def hedged_asset_fx(self, assets_df, asset_currency, home_curr, start_date, finish_date, spot_df=None,
total_return_indices_df=None):
pass
def get_day_count_conv(self, currency):
if currency in market_constants.currencies_with_365_basis:
return 365.0
return 360.0
def construct_total_return_index(self, cross_fx, forwards_market_df,
fx_forwards_trading_tenor=None,
roll_days_before=None,
roll_event=None,
roll_months=None,
fx_forwards_tenor_for_interpolation=None,
cum_index=None,
output_calculation_fields=None,
field=None):
if not (isinstance(cross_fx, list)):
cross_fx = [cross_fx]
if fx_forwards_trading_tenor is None: fx_forwards_trading_tenor = self._fx_forwards_trading_tenor
if roll_days_before is None: roll_days_before = self._roll_days_before
if roll_event is None: roll_event = self._roll_event
if roll_months is None: roll_months = self._roll_months
if fx_forwards_tenor_for_interpolation is None: fx_forwards_tenor_for_interpolation = self._fx_forwards_tenor_for_interpolation
if cum_index is None: cum_index = self._cum_index
if field is None: field = self._field
total_return_index_df_agg = []
# Remove columns where there is no data (because these points typically aren't quoted)
forwards_market_df = forwards_market_df.dropna(how='all', axis=1)
fx_forwards_pricer = FXForwardsPricer()
def get_roll_date(horizon_d, delivery_d, asset_hols, month_adj=1):
if roll_event == 'month-end':
roll_d = horizon_d + CustomBusinessMonthEnd(roll_months + month_adj, holidays=asset_hols)
elif roll_event == 'delivery-date':
roll_d = delivery_d
return (roll_d - CustomBusinessDay(n=roll_days_before, holidays=asset_hols))
for cross in cross_fx:
# Eg. if we specify USDUSD
if cross[0:3] == cross[3:6]:
total_return_index_df_agg.append(
pd.DataFrame(100, index=forwards_market_df.index, columns=[cross + "-forward-tot.close"]))
else:
# Is the FX cross in the correct convention
old_cross = cross
cross = FXConv().correct_notation(cross)
horizon_date = forwards_market_df.index
delivery_date = []
roll_date = []
new_trade = np.full(len(horizon_date), False, dtype=bool)
asset_holidays = self._calendar.get_holidays(cal=cross)
# Get first delivery date
delivery_date.append(
self._calendar.get_delivery_date_from_horizon_date(horizon_date[0],
fx_forwards_trading_tenor, cal=cross, asset_class='fx')[0])
# For first month want it to expire within that month (for consistency), hence month_adj=0 ONLY here
roll_date.append(get_roll_date(horizon_date[0], delivery_date[0], asset_holidays, month_adj=0))
# New trade => entry at beginning AND on every roll
new_trade[0] = True
# Get all the delivery dates and roll dates
# At each "roll/trade" day we need to reset them for the new contract
for i in range(1, len(horizon_date)):
# If the horizon date has reached the roll date (from yesterday), we're done, and we have a
# new roll/trade
if (horizon_date[i] - roll_date[i-1]).days == 0:
new_trade[i] = True
# else:
# new_trade[i] = False
# If we're entering a new trade/contract, we need to get new delivery and roll dates
if new_trade[i]:
delivery_date.append(self._calendar.get_delivery_date_from_horizon_date(horizon_date[i],
fx_forwards_trading_tenor, cal=cross, asset_class='fx')[0])
roll_date.append(get_roll_date(horizon_date[i], delivery_date[i], asset_holidays))
else:
# Otherwise use previous delivery and roll dates, because we're still holding same contract
delivery_date.append(delivery_date[i-1])
roll_date.append(roll_date[i-1])
interpolated_forward = fx_forwards_pricer.price_instrument(cross, horizon_date, delivery_date, market_df=forwards_market_df,
fx_forwards_tenor_for_interpolation=fx_forwards_tenor_for_interpolation)[cross + '-interpolated-outright-forward.' + field].values
# To record MTM prices
mtm = np.copy(interpolated_forward)
# Note: may need to add discount factor when marking to market forwards?
# Special case: for very first trading day
# mtm[0] = interpolated_forward[0]
# On rolling dates, MTM will be the previous forward contract (interpolated)
# otherwise it will be the current forward contract
for i in range(1, len(horizon_date)):
if new_trade[i]:
mtm[i] = fx_forwards_pricer.price_instrument(cross, horizon_date[i], delivery_date[i-1],
market_df=forwards_market_df,
fx_forwards_tenor_for_interpolation=fx_forwards_tenor_for_interpolation) \
[cross + '-interpolated-outright-forward.' + field].values
# else:
# mtm[i] = interpolated_forward[i]
# Eg. if we asked for USDEUR, we first constructed spot/forwards for EURUSD
# and then need to invert it
if old_cross != cross:
mtm = 1.0 / mtm
interpolated_forward = 1.0 / interpolated_forward
forward_rets = mtm / np.roll(interpolated_forward, 1) - 1.0
forward_rets[0] = 0
if cum_index == 'mult':
cum_rets = 100 * np.cumprod(1.0 + forward_rets)
elif cum_index == 'add':
cum_rets = 100 + 100 * np.cumsum(forward_rets)
total_return_index_df = pd.DataFrame(index=horizon_date, columns=[cross + "-forward-tot." + field])
total_return_index_df[cross + "-forward-tot." + field] = cum_rets
if output_calculation_fields:
total_return_index_df[cross + '-interpolated-outright-forward.' + field] = interpolated_forward
total_return_index_df[cross + '-mtm.close'] = mtm
total_return_index_df[cross + '-roll.close'] = new_trade
total_return_index_df[cross + '.roll-date'] = roll_date
total_return_index_df[cross + '.delivery-date'] = delivery_date
total_return_index_df[cross + '-forward-return.' + field] = forward_rets
total_return_index_df_agg.append(total_return_index_df)
return self._calculations.join(total_return_index_df_agg, how='outer')
Shows how to calculate returns of an asset
"""
# Loading data
import datetime
from chartpy import Chart, Style
from finmarketpy.backtest import TradeAnalysis
from findatapy.market import Market, MarketDataGenerator, MarketDataRequest
from chartpy.style import Style
from findatapy.timeseries import Calculations
from findatapy.util.loggermanager import LoggerManager
ta = TradeAnalysis()
calc = Calculations()
logger = LoggerManager().getLogger(__name__)
chart = Chart(engine='matplotlib')
market = Market(market_data_generator=MarketDataGenerator())
# Choose run_example = 0 for everything
# run_example = 1 - use PyFolio to analyse gold's return properties
run_example = 0
###### Use PyFolio to analyse gold's return properties
if run_example == 1 or run_example == 0:
md_request = MarketDataRequest(
start_date="01 Jan 1996", # start date
def __init__(
self,
market_data_generator=None,
fx_options_trading_tenor=market_constants.fx_options_trading_tenor,
roll_days_before=market_constants.fx_options_roll_days_before,
roll_event=market_constants.fx_options_roll_event,
construct_via_currency='no',
fx_options_tenor_for_interpolation=market_constants.
fx_options_tenor_for_interpolation,
base_depos_tenor=data_constants.base_depos_tenor,
roll_months=market_constants.fx_options_roll_months,
cum_index=market_constants.fx_options_cum_index,
strike=market_constants.fx_options_index_strike,
contract_type=market_constants.fx_options_index_contract_type,
premium_output=market_constants.fx_options_index_premium_output,
position_multiplier=1,
depo_tenor_for_option=market_constants.fx_options_depo_tenor,
freeze_implied_vol=market_constants.fx_options_freeze_implied_vol,
tot_label='',
cal=None,
output_calculation_fields=market_constants.
output_calculation_fields):
"""Initializes FXForwardsCurve
Parameters
----------
market_data_generator : MarketDataGenerator
Used for downloading market data
fx_options_trading_tenor : str
What is primary forward contract being used to trade (default - '1M')
roll_days_before : int
Number of days before roll event to enter into a new forwards contract
roll_event : str
What constitutes a roll event? ('month-end', 'quarter-end', 'year-end', 'expiry')
cum_index : str
In total return index, do we compute in additive or multiplicative way ('add' or 'mult')
construct_via_currency : str
What currency should we construct the forward via? Eg. if we asked for AUDJPY we can construct it via
AUDUSD & JPYUSD forwards, as opposed to AUDJPY forwards (default - 'no')
fx_options_tenor_for_interpolation : str(list)
Which forwards should we use for interpolation
base_depos_tenor : str(list)
Which base deposits tenors do we need (this is only necessary if we want to start inferring depos)
roll_months : int
After how many months should we initiate a roll. Typically for trading 1M this should 1, 3M this should be 3
etc.
tot_label : str
Postfix for the total returns field
cal : str
Calendar to use for expiry (if None, uses that of FX pair)
output_calculation_fields : bool
Also output additional data should forward expiries etc. alongside total returns indices
"""
self._market_data_generator = market_data_generator
self._calculations = Calculations()
self._calendar = Calendar()
self._filter = Filter()
self._fx_options_trading_tenor = fx_options_trading_tenor
self._roll_days_before = roll_days_before
self._roll_event = roll_event
self._construct_via_currency = construct_via_currency
self._fx_options_tenor_for_interpolation = fx_options_tenor_for_interpolation
self._base_depos_tenor = base_depos_tenor
self._roll_months = roll_months
self._cum_index = cum_index
self._contact_type = contract_type
self._strike = strike
self._premium_output = premium_output
self._position_multiplier = position_multiplier
self._depo_tenor_for_option = depo_tenor_for_option
self._freeze_implied_vol = freeze_implied_vol
self._tot_label = tot_label
self._cal = cal
self._output_calculation_fields = output_calculation_fields
# Loading data import datetime import pandas from chartpy import Chart, Style from finmarketpy.economics import Seasonality from findatapy.market import Market, MarketDataGenerator, MarketDataRequest from chartpy.style import Style from findatapy.timeseries import Calculations from findatapy.util.loggermanager import LoggerManager seasonality = Seasonality() calc = Calculations() logger = LoggerManager().getLogger(__name__) chart = Chart(engine='matplotlib') market = Market(market_data_generator=MarketDataGenerator()) # choose run_example = 0 for everything # run_example = 1 - seasonality of gold # run_example = 2 - seasonality of FX vol # run_example = 3 - seasonality of gasoline # run_example = 4 - seasonality in NFP # run_example = 5 - seasonal adjustment in NFP run_example = 0
spot_df = asset_df
logger.info("Running backtest...")
# use technical indicator to create signals
# (we could obviously create whatever function we wanted for generating the signal dataframe)
tech_ind = TechIndicator()
tech_ind.create_tech_ind(spot_df, indicator, tech_params); signal_df = tech_ind.get_signal()
# use the same data for generating signals
backtest.calculate_trading_PnL(br, asset_df, signal_df)
port = backtest.get_cumportfolio()
port.columns = [indicator + ' = ' + str(tech_params.sma_period) + ' ' + str(backtest.get_portfolio_pnl_desc()[0])]
signals = backtest.get_porfolio_signal() # get final signals for each series
returns = backtest.get_pnl() # get P&L for each series
calculations = Calculations()
trade_returns = calculations.calculate_individual_trade_gains(signals, returns)
print(trade_returns)
# print the last positions (we could also save as CSV etc.)
print(signals.tail(1))
style = Style()
style.title = "EUR/USD trend model"
style.source = 'Quandl'
style.scale_factor = 1
style.file_output = 'eurusd-trend-example.png'
Chart(port, style = style).plot()
cross,
df_vol_market,
contract_type='european-put',
strike='10d-otm',
position_multiplier=1.0)
# Add transaction costs to the option index (bid/ask bp for the option premium and spot FX)
df_cuemacro_option_put_tc = fx_options_curve.apply_tc_signals_to_total_return_index(
cross, df_cuemacro_option_put_tot, option_tc_bp=5, spot_tc_bp=2)
# Get total returns for spot
df_bbg_tot = df_tot # from earlier!
df_bbg_tot.columns = [x + '-bbg' for x in df_bbg_tot.columns]
# Calculate a hedged portfolio of spot + 2*options (can we reduce drawdowns?)
calculations = Calculations()
ret_stats = RetStats()
df_hedged = calculations.join([
df_bbg_tot[cross + '-tot.close-bbg'].to_frame(),
df_cuemacro_option_put_tc[cross +
'-option-tot-with-tc.close'].to_frame()
],
how='outer')
df_hedged = df_hedged.fillna(method='ffill')
df_hedged = df_hedged.pct_change()
df_hedged['Spot + 2*option put hedge'] = df_hedged[
cross + '-tot.close-bbg'] + df_hedged[cross +
'-option-tot-with-tc.close']
def create_tech_ind(
self,
data_frame_non_nan,
name,
tech_params,
data_frame_non_nan_early=None):
self._signal = None
self._techind = None
if tech_params.fillna:
data_frame = data_frame_non_nan.fillna(method="ffill")
else:
data_frame = data_frame_non_nan
if data_frame_non_nan_early is not None:
data_frame_early = data_frame_non_nan_early.fillna(method="ffill")
if name == "SMA":
if (data_frame_non_nan_early is not None):
# calculate the lagged sum of the n-1 point
if pd.__version__ < '0.17':
rolling_sum = pd.rolling_sum(
data_frame.shift(1).rolling,
window=tech_params.sma_period - 1)
else:
rolling_sum = data_frame.shift(1).rolling(
center=False, window=tech_params.sma_period - 1).sum()
# add non-nan one for today
rolling_sum = rolling_sum + data_frame_early
# calculate average = sum / n
self._techind = rolling_sum / tech_params.sma_period
narray = np.where(data_frame_early > self._techind, 1, -1)
else:
if pd.__version__ < '0.17':
self._techind = pd.rolling_sum(
data_frame, window=tech_params.sma_period)
else:
self._techind = data_frame.rolling(
window=tech_params.sma_period, center=False).mean()
narray = np.where(data_frame > self._techind, 1, -1)
self._signal = pd.DataFrame(index=data_frame.index, data=narray)
self._signal.loc[0:tech_params.sma_period] = np.nan
self._signal.columns = [
x + " SMA Signal" for x in data_frame.columns.values]
self._techind.columns = [
x + " SMA" for x in data_frame.columns.values]
elif name == "EMA":
# self._techind = pd.ewma(data_frame, span = tech_params.ema_period)
self._techind = data_frame.ewm(
ignore_na=False,
span=tech_params.ema_period,
min_periods=0,
adjust=True).mean()
narray = np.where(data_frame > self._techind, 1, -1)
self._signal = pd.DataFrame(index=data_frame.index, data=narray)
self._signal.loc[0:tech_params.ema_period] = np.nan
self._signal.columns = [
x + " EMA Signal" for x in data_frame.columns.values]
self._techind.columns = [
x + " EMA" for x in data_frame.columns.values]
elif name == "ROC":
if (data_frame_non_nan_early is not None):
self._techind = data_frame_early / \
data_frame.shift(tech_params.roc_period) - 1
else:
self._techind = data_frame / \
data_frame.shift(tech_params.roc_period) - 1
narray = np.where(self._techind > 0, 1, -1)
self._signal = pd.DataFrame(index=data_frame.index, data=narray)
self._signal.loc[0:tech_params.roc_period] = np.nan
self._signal.columns = [
x + " ROC Signal" for x in data_frame.columns.values]
self._techind.columns = [
x + " ROC" for x in data_frame.columns.values]
elif name == "polarity":
self._techind = data_frame
narray = np.where(self._techind > 0, 1, -1)
self._signal = pd.DataFrame(index=data_frame.index, data=narray)
self._signal.columns = [
x + " Polarity Signal" for x in data_frame.columns.values]
self._techind.columns = [
x + " Polarity" for x in data_frame.columns.values]
elif name == "SMA2":
sma = data_frame.rolling(
window=tech_params.sma_period,
center=False).mean()
sma2 = data_frame.rolling(
window=tech_params.sma2_period,
center=False).mean()
narray = np.where(sma > sma2, 1, -1)
self._signal = pd.DataFrame(index=data_frame.index, data=narray)
self._signal.columns = [
x + " SMA2 Signal" for x in data_frame.columns.values]
sma.columns = [x + " SMA" for x in data_frame.columns.values]
sma2.columns = [x + " SMA2" for x in data_frame.columns.values]
most = max(tech_params.sma_period, tech_params.sma2_period)
self._signal.loc[0:most] = np.nan
self._techind = pd.concat([sma, sma2], axis=1)
elif name in ['RSI']:
# delta = data_frame.diff()
#
# dUp, dDown = delta.copy(), delta.copy()
# dUp[dUp < 0] = 0
# dDown[dDown > 0] = 0
#
# rolUp = pd.rolling_mean(dUp, tech_params.rsi_period)
# rolDown = pd.rolling_mean(dDown, tech_params.rsi_period).abs()
#
# rsi = rolUp / rolDown
# Get the difference in price from previous step
delta = data_frame.diff()
# Get rid of the first row, which is NaN since it did not have a previous
# row to calculate the differences
delta = delta[1:]
# Make the positive gains (up) and negative gains (down) Series
up, down = delta.copy(), delta.copy()
up[up < 0] = 0
down[down > 0] = 0
# Calculate the EWMA
roll_up1 = pd.stats.moments.ewma(up, tech_params.rsi_period)
roll_down1 = pd.stats.moments.ewma(
down.abs(), tech_params.rsi_period)
# Calculate the RSI based on EWMA
RS1 = roll_up1 / roll_down1
RSI1 = 100.0 - (100.0 / (1.0 + RS1))
# Calculate the SMA
roll_up2 = up.rolling(
window=tech_params.rsi_period,
center=False).mean()
roll_down2 = down.abs().rolling(
window=tech_params.rsi_period, center=False).mean()
# Calculate the RSI based on SMA
RS2 = roll_up2 / roll_down2
RSI2 = 100.0 - (100.0 / (1.0 + RS2))
self._techind = RSI2
self._techind.columns = [
x + " RSI" for x in data_frame.columns.values]
signal = data_frame.copy()
sells = (signal.shift(-1) <
tech_params.rsi_lower) & (signal > tech_params.rsi_lower)
buys = (signal.shift(-1) >
tech_params.rsi_upper) & (signal < tech_params.rsi_upper)
# print (buys[buys == True])
# buys
signal[buys] = 1
signal[sells] = -1
signal[~(buys | sells)] = np.nan
signal = signal.fillna(method='ffill')
self._signal = signal
self._signal.loc[0:tech_params.rsi_period] = np.nan
self._signal.columns = [
x + " RSI Signal" for x in data_frame.columns.values]
elif name in ["BB"]:
# calcuate Bollinger bands
mid = data_frame.rolling(
center=False, window=tech_params.bb_period).mean()
mid.columns = [x + " BB Mid" for x in data_frame.columns.values]
std_dev = data_frame.rolling(
center=False, window=tech_params.bb_period).std()
BB_std = tech_params.bb_mult * std_dev
lower = pd.DataFrame(
data=mid.values - BB_std.values,
index=mid.index,
columns=data_frame.columns)
upper = pd.DataFrame(
data=mid.values + BB_std.values,
index=mid.index,
columns=data_frame.columns)
# calculate signals
signal = data_frame.copy()
buys = signal > upper
sells = signal < lower
signal[buys] = 1
signal[sells] = -1
signal[~(buys | sells)] = np.nan
signal = signal.fillna(method='ffill')
self._signal = signal
self._signal.loc[0:tech_params.bb_period] = np.nan
self._signal.columns = [
x + " " + name + " Signal" for x in data_frame.columns.values]
lower.columns = [
x + " BB Lower" for x in data_frame.columns.values]
upper.columns = [x + " BB Mid" for x in data_frame.columns.values]
upper.columns = [
x + " BB Lower" for x in data_frame.columns.values]
self._techind = pd.concat([lower, mid, upper], axis=1)
elif name == "long-only":
# have +1 signals only
self._techind = data_frame # the technical indicator is just "prices"
narray = np.ones((len(data_frame.index), len(data_frame.columns)))
self._signal = pd.DataFrame(index=data_frame.index, data=narray)
self._signal.columns = [
x + " Long Only Signal" for x in data_frame.columns.values]
self._techind.columns = [
x + " Long Only" for x in data_frame.columns.values]
elif name == "ATR":
# get all the asset names (assume we have names 'close', 'low', 'high' in the Data)
# keep ordering of assets
asset_name = list(OrderedDict.fromkeys(
[x.split('.')[0] for x in data_frame.columns]))
df = []
# can improve the performance of this if vectorise more!
for a in asset_name:
close = [a + '.close']
low = [a + '.low']
high = [a + '.high']
# if we don't fill NaNs, we need to remove those rows and then
# calculate the ATR
if not(tech_params.fillna):
data_frame_short = data_frame[[close[0], low[0], high[0]]]
data_frame_short = data_frame_short.dropna()
else:
data_frame_short = data_frame
prev_close = data_frame_short[close].shift(1)
c1 = data_frame_short[high].values - \
data_frame_short[low].values
c2 = np.abs(data_frame_short[high].values - prev_close.values)
c3 = np.abs(data_frame_short[low].values - prev_close.values)
true_range = np.max((c1, c2, c3), axis=0)
true_range = pd.DataFrame(
index=data_frame_short.index, data=true_range, columns=[
close[0] + ' True Range'])
# put back NaNs into ATR if necessary
if (not(tech_params.fillna)):
true_range = true_range.reindex(
data_frame.index, fill_value=np.nan)
df.append(true_range)
calc = Calculations()
true_range = calc.pandas_outer_join(df)
self._techind = true_range.rolling(
window=tech_params.atr_period, center=False).mean()
# self._techind = true_range.ewm(ignore_na=False, span=tech_params.atr_period, min_periods=0, adjust=True).mean()
self._techind.columns = [x + ".close ATR" for x in asset_name]
elif name in ["VWAP"]:
asset_name = list(OrderedDict.fromkeys(
[x.split('.')[0] for x in data_frame.columns]))
df = []
for a in asset_name:
high = [a + '.high']
low = [a + '.low']
close = [a + '.close']
volume = [a + '.volume']
if not tech_params.fillna:
df_mod = data_frame[[high[0], low[0], close[0], volume[0]]]
df_mod.dropna(inplace=True)
else:
df_mod = data_frame
l = df_mod[low].values
h = df_mod[high].values
c = df_mod[close].values
v = df_mod[volume].values
vwap = np.cumsum(((h + l + c) / 3) * v) / np.cumsum(v)
vwap = pd.DataFrame(index=df_mod.index, data=vwap,
columns=[close[0] + ' VWAP'])
print(vwap.columns)
if not tech_params.fillna:
vwap = vwap.reindex(data_frame.index, fill_value=np.nan)
df.append(vwap)
calc = Calculations()
vwap = calc.pandas_outer_join(df)
self._techind = vwap
self._techind.columns = [x + ".close VWAP" for x in asset_name]
self.create_custom_tech_ind(
data_frame_non_nan,
name,
tech_params,
data_frame_non_nan_early)
# TODO create other indicators
if hasattr(tech_params, 'only_allow_longs'):
self._signal[self._signal < 0] = 0
# TODO create other indicators
if hasattr(tech_params, 'only_allow_shorts'):
self._signal[self._signal > 0] = 0
# apply signal multiplier (typically to flip signals)
if hasattr(tech_params, 'signal_mult'):
self._signal = self._signal * tech_params.signal_mult
if hasattr(tech_params, 'strip_signal_name'):
if tech_params.strip_signal_name:
self._signal.columns = data_frame.columns
return self._techind, self._signal
def get_intraday_moves_over_custom_event(self, data_frame_rets, ef_time_frame, vol=False,
minute_start=5, mins=3 * 60, min_offset=0, create_index=False,
resample=False, freq='minutes', cumsum=True):
filter = Filter()
ef_time_frame = filter.filter_time_series_by_date(data_frame_rets.index[0], data_frame_rets.index[-1],
ef_time_frame)
ef_time = ef_time_frame.index
if freq == 'minutes':
ef_time_start = ef_time - timedelta(minutes=minute_start)
ef_time_end = ef_time + timedelta(minutes=mins)
ann_factor = 252 * 1440
elif freq == 'days':
ef_time = ef_time_frame.index.normalize()
ef_time_start = ef_time - timedelta(days=minute_start)
ef_time_end = ef_time + timedelta(days=mins)
ann_factor = 252
ords = range(-minute_start + min_offset, mins + min_offset)
# all data needs to be equally spaced
if resample:
# make sure time series is properly sampled at 1 min intervals
data_frame_rets = data_frame_rets.resample('1min')
data_frame_rets = data_frame_rets.fillna(value=0)
data_frame_rets = filter.remove_out_FX_out_of_hours(data_frame_rets)
data_frame_rets['Ind'] = numpy.nan
start_index = data_frame_rets.index.searchsorted(ef_time_start)
finish_index = data_frame_rets.index.searchsorted(ef_time_end)
# not all observation windows will be same length (eg. last one?)
# fill the indices which represent minutes
# TODO vectorise this!
for i in range(0, len(ef_time_frame.index)):
try:
data_frame_rets['Ind'][start_index[i]:finish_index[i]] = ords
except:
data_frame_rets['Ind'][start_index[i]:finish_index[i]] = ords[0:(finish_index[i] - start_index[i])]
data_frame_rets['Rel'] = numpy.nan
# Set the release dates
data_frame_rets['Rel'][start_index] = ef_time # set entry points
data_frame_rets['Rel'][finish_index + 1] = numpy.zeros(len(start_index)) # set exit points
data_frame_rets['Rel'] = data_frame_rets['Rel'].fillna(method='pad') # fill down signals
data_frame_rets = data_frame_rets[pandas.notnull(data_frame_rets['Ind'])] # get rid of other
data_frame = data_frame_rets.pivot(index='Ind',
columns='Rel', values=data_frame_rets.columns[0])
data_frame.index.names = [None]
if create_index:
calculations = Calculations()
data_frame.iloc[-minute_start + min_offset] = numpy.nan
data_frame = calculations.create_mult_index(data_frame)
else:
if vol is True:
# annualise (if vol)
data_frame = data_frame.rolling(center=False, window=5).std() * math.sqrt(ann_factor)
elif cumsum:
data_frame = data_frame.cumsum()
return data_frame
