def get_liquid_hours(self, contract: IBContract) -> QFDataFrame:
""" Returns a QFDataFrame containing information about liquid hours of the given contract. """
with self.lock:
self._reset_action_lock()
request_id = 3
self.client.reqContractDetails(request_id, contract)
if self._wait_for_results():
contract_details = self.wrapper.contract_details
liquid_hours = contract_details.tradingHours.split(";")
liquid_hours_df = QFDataFrame.from_records([
hours.split("-")
for hours in liquid_hours if not hours.endswith("CLOSED")
],
columns=[
"FROM", "TO"
])
for col in liquid_hours_df.columns:
liquid_hours_df[col] = to_datetime(liquid_hours_df[col],
format="%Y%m%d:%H%M")
liquid_hours_df.name = contract_details.contract.symbol
return liquid_hours_df
else:
error_msg = 'Time out while getting contract details'
self.logger.error(error_msg)
raise BrokerException(error_msg)
def get_signals(self) -> QFDataFrame:
df = QFDataFrame.from_records(self._signals_data,
columns=["Date", "Ticker", "Signal"])
# Modify the dataframe to move all signals for certain tickers to separate columns and set the index to date
df = df.pivot_table(index='Date',
columns='Ticker',
values='Signal',
aggfunc='first')
return QFDataFrame(df)
def _read_csv(self,
content: str,
column_names: List[str],
replace_header: bool,
delimiter: str = "|"):
# Remove trailing delimiters
content = "\n".join(line.rstrip("|") for line in content.split("\n"))
lines = csv.reader(StringIO(content), delimiter=delimiter)
records = list(lines) if not replace_header else list(lines)[1:]
records = [line for line in records if len(line) == len(column_names)]
df = QFDataFrame.from_records(records, columns=column_names)
return df
def _add_performance_statistics(self):
"""
For each ticker computes its overall performance (PnL of short positions, PnL of long positions, total PnL).
It generates a table containing final PnL values for each of the ticker nad optionally plots the performance
throughout the backtest.
"""
closed_positions = self.backtest_result.portfolio.closed_positions()
closed_positions_pnl = QFDataFrame.from_records(
data=[(self._ticker_name(p.contract()), p.end_time, p.direction(), p.total_pnl) for p in closed_positions],
columns=["Tickers name", "Time", "Direction", "Realised PnL"]
)
closed_positions_pnl = closed_positions_pnl.sort_values(by="Time")
# Get all open positions history
open_positions_history = self.backtest_result.portfolio.positions_history()
open_positions_history = open_positions_history.reset_index().melt(
id_vars='index', value_vars=open_positions_history.columns, var_name='Contract',
value_name='Position summary')
open_positions_pnl = QFDataFrame(data={
"Tickers name": open_positions_history["Contract"].apply(lambda contract: self._ticker_name(contract)),
"Time": open_positions_history["index"],
"Direction": open_positions_history["Position summary"].apply(
lambda p: p.direction if isinstance(p, BacktestPositionSummary) else 0),
"Total PnL of open position": open_positions_history["Position summary"].apply(
lambda p: p.total_pnl if isinstance(p, BacktestPositionSummary) else 0)
})
all_positions_pnl = pd.concat([closed_positions_pnl, open_positions_pnl], sort=False)
performance_dicts_series = all_positions_pnl.groupby(by=["Tickers name"]).apply(
self._performance_series_for_ticker)
performance_df = QFDataFrame(performance_dicts_series.tolist(), index=performance_dicts_series.index)
self.document.add_element(NewPageElement())
self.document.add_element(HeadingElement(level=2, text="Performance of each asset"))
final_performance = performance_df. \
applymap(lambda pnl_series: pnl_series.iloc[-1] if not pnl_series.empty else 0.0). \
sort_values(by="Overall performance", ascending=False). \
applymap(lambda p: '{:,.2f}'.format(p)). \
reset_index()
table = DFTable(final_performance, css_classes=['table', 'left-align'])
table.add_columns_classes(["Tickers name"], 'wide-column')
self.document.add_element(table)
# Add performance plots
if self.generate_pnl_chart_per_ticker:
self.document.add_element(NewPageElement())
self.document.add_element(
HeadingElement(level=2, text="Performance of each asset during the whole backtest"))
for ticker_name, performance in performance_df.iterrows():
self._plot_ticker_performance(ticker_name, performance)
def _filter_transactions(self, ticker: Ticker, transactions: List[Transaction], start_date: datetime,
end_date: datetime) -> QFSeries:
""" Filters out transactions, which do not correspond to the given ticker and returns a QFSeries of remaining
transactions. Only transactions between start_date market open and end_date market close are considered. """
transactions = [t for t in transactions if start_date + MarketOpenEvent.trigger_time()
<= t.time <= end_date + MarketCloseEvent.trigger_time()]
if isinstance(ticker, FutureTicker):
transactions_for_tickers = [t for t in transactions if ticker.belongs_to_family(t.ticker)]
else:
transactions_for_tickers = [t for t in transactions if ticker == t.ticker]
transactions_records = [(t, t.time) for t in transactions_for_tickers]
transactions_series = QFDataFrame.from_records(transactions_records, columns=["Transaction", "Index"]) \
.set_index("Index").iloc[:, 0]
return transactions_series
def create_trades_from_transactions(self, transactions: Sequence, portfolio_values: Optional[QFSeries] = None) \
-> Sequence[Trade]:
"""
Generates trades based on a series of Transactions.
Parameters
-----------
transactions: Sequence
Sequence of transactions, which should be parsed
portfolio_values: Optional[QFSeries]
Series containing portfolio values at different point in time. It is optional and if provided, the
percentage pnl value is set in the Trade.
Returns
--------
trades: Sequence[Trade]
List containing trades information, sorted by the time of their creation
"""
transactions_df = QFDataFrame.from_records(
[(t, t.time, t.contract.symbol, t.quantity) for t in transactions],
columns=["transaction", "time", "contract", "quantity"])
# Position size after transacting the transaction, where position is identified by "contract" variable
transactions_df.sort_values(by="time", inplace=True)
transactions_df["position size"] = transactions_df.groupby(
by="contract")["quantity"].cumsum()
# Assign position start values - a position was opened when the position size was equal to the quantity of
# the transaction (the quantity of the contract in the portfolio before transaction was = 0)
new_positions_beginning = transactions_df[
"position size"] - transactions_df["quantity"] == 0
transactions_df["position start"] = None
transactions_df["position start"].loc[
new_positions_beginning] = transactions_df.loc[
new_positions_beginning].index
transactions_df["position start"] = transactions_df.groupby(
by="contract")["position start"].apply(
lambda tms: tms.fillna(method="ffill"))
trades_series = transactions_df.groupby(
by=["position start"])["transaction"].apply(
lambda t: self._parse_position(t, portfolio_values))
trades = trades_series.sort_index(level=1).tolist()
return trades
def get_signals_for_ticker(self,
ticker: Ticker,
alpha_model=None) -> QFSeries:
def signal_to_return(signal: Signal):
if alpha_model is None:
return signal.ticker == ticker
else:
return signal.ticker == ticker and str(
signal.alpha_model) == str(alpha_model)
signals_data_for_ticker = [(d, s) for (d, _, s) in self._signals_data
if signal_to_return(s)]
df = QFDataFrame.from_records(signals_data_for_ticker,
columns=["Date", "Signal"])
df = df.set_index("Date").sort_index()
series = df.iloc[:, 0]
return series
def _get_distribution_summary_table(self, scenarios_results: SimpleReturnsSeries) -> DFTable:
rows = []
percentage_list = [0.05, 0.1, 0.2, 0.3]
for percentage in percentage_list:
rows.append(("{:.0%} Tail".format(percentage),
"{:.2%}".format(np.quantile(scenarios_results, percentage))))
rows.append(("50%", "{:.2%}".format(np.quantile(scenarios_results, 0.5))))
for percentage in reversed(percentage_list):
rows.append(("{:.0%} Top".format(percentage),
"{:.2%}".format(np.quantile(scenarios_results, (1.0 - percentage)))))
table = DFTable(data=QFDataFrame.from_records(rows, columns=["Measure", "Value"]),
css_classes=['table', 'left-align'])
table.add_columns_classes(["Measure"], 'wide-column')
return table
def _get_monte_carlos_simulator_outputs(self, scenarios_df: PricesDataFrame, total_returns: SimpleReturnsSeries) \
-> DFTable:
_, all_scenarios_number = scenarios_df.shape
rows = []
# Add the Median Return value
median_return = np.median(total_returns)
rows.append(("Median Return", "{:.2%}".format(median_return)))
# Add the Mean Return value
mean_return = total_returns.mean()
rows.append(("Mean Return", "{:.2%}".format(mean_return)))
trade_returns = QFSeries(data=[trade.percentage_pnl for trade in self.trades])
sample_len = int(self._average_number_of_trades_per_year())
std = trade_returns.std()
expectation_adj_series = np.ones(sample_len) * (trade_returns.mean() - 0.5 * std * std)
expectation_adj_series = SimpleReturnsSeries(data=expectation_adj_series)
expectation_adj_series = expectation_adj_series.to_prices(suggested_initial_date=0)
mean_volatility_adjusted_return = expectation_adj_series.iloc[-1] / expectation_adj_series.iloc[0] - 1.0
rows.append(("Mean Volatility Adjusted Return", "{:.2%}".format(mean_volatility_adjusted_return)))
# Add the Median Drawdown
max_drawdowns = max_drawdown(scenarios_df)
median_drawdown = np.median(max_drawdowns)
rows.append(("Median Maximum Drawdown", "{:.2%}".format(median_drawdown)))
# Add the Median Return / Median Drawdown
rows.append(("Return / Drawdown", "{:.2f}".format(median_return / median_drawdown)))
# Probability, that the return will be > 0
scenarios_with_positive_result = total_returns[total_returns > 0.0].count()
probability = scenarios_with_positive_result / all_scenarios_number
rows.append(("Probability of positive return", "{:.2%}".format(probability)))
table = DFTable(data=QFDataFrame.from_records(rows, columns=["Measure", "Value"]),
css_classes=['table', 'left-align'])
table.add_columns_classes(["Measure"], 'wide-column')
return table
def _get_chances_of_dropping_below_table(self, scenarios_df: PricesDataFrame) -> DFTable:
_, all_scenarios_number = scenarios_df.shape
rows = []
crop_table = False
for percentage in np.linspace(0.1, 0.9, 9):
# Count number of scenarios, whose returns at some point of time dropped below the percentage * initial
# value
_, scenarios_above_percentage = scenarios_df.where(scenarios_df > (1.0 - percentage)).dropna(axis=1).shape
probability = (all_scenarios_number - scenarios_above_percentage) / all_scenarios_number
rows.append(("{:.0%}".format(percentage), "{:.2%}".format(probability)))
if crop_table is True:
break
elif probability < 0.1:
crop_table = True
table = DFTable(QFDataFrame.from_records(rows, columns=["Chances of dropping below", "Probability"]),
css_classes=['table', 'left-align'])
table.add_columns_classes(["Chances of dropping below"], 'wide-column')
return table
def _add_stats_table(self):
statistics = [] # type: List[Tuple]
def append_to_statistics(measure_description: str, function: Callable, trades_containers,
percentage_style: bool = False):
style_format = "{:.2%}" if percentage_style else "{:.2f}"
returned_values = (function(tc) for tc in trades_containers)
returned_values = (value if is_finite_number(value) else 0.0 for value in returned_values)
statistics.append((measure_description, *(style_format.format(val) for val in returned_values)))
# Prepare trades data frame, used to generate all statistics
trades_df = QFDataFrame.from_records(
data=[(t.start_time, t.end_time, t.percentage_pnl, t.direction) for t in self.trades],
columns=["start time", "end time", "percentage pnl", "direction"]
)
# In case if the initial risk is not set all the return statistic will be computed using the percentage pnl,
# otherwise the r_multiply = percentage pnl / initial risk is used
unit = "%" if self.initial_risk is None else "R"
trades_df["returns"] = trades_df["percentage pnl"] if self.initial_risk is None \
else trades_df["percentage pnl"] / self.initial_risk
# Filter out only long and only
long_trades_df = trades_df[trades_df["direction"] > 0]
short_trades_df = trades_df[trades_df["direction"] < 0]
all_dfs = [trades_df, long_trades_df, short_trades_df]
append_to_statistics("Number of trades", len, all_dfs)
append_to_statistics("% of trades number", lambda df: len(df) / len(trades_df) if len(trades_df) > 0 else 0,
all_dfs, percentage_style=True)
period_length_in_years = Timedelta(self.end_date - self.start_date) / Timedelta(days=1) / DAYS_PER_YEAR_AVG
append_to_statistics("Avg number of trades per year", lambda df: len(df) / period_length_in_years, all_dfs)
append_to_statistics("Avg number of trades per year per asset",
lambda df: len(df) / period_length_in_years / self.nr_of_assets_traded, all_dfs)
def percentage_of_positive_trades(df: QFDataFrame):
return len(df[df["returns"] > 0]) / len(df) if len(df) > 0 else 0.0
append_to_statistics("% of positive trades", percentage_of_positive_trades, all_dfs, percentage_style=True)
def percentage_of_negative_trades(df: QFDataFrame):
return len(df[df["returns"] < 0]) / len(df) if len(df) > 0 else 0.0
append_to_statistics("% of negative trades", percentage_of_negative_trades, all_dfs, percentage_style=True)
def avg_trade_duration(df: QFDataFrame):
trades_duration = (df["end time"] - df["start time"]) / Timedelta(days=1)
return trades_duration.mean()
append_to_statistics("Average trade duration [days]", avg_trade_duration, all_dfs)
append_to_statistics("Average trade return [{}]".format(unit), lambda df: df["returns"].mean(), all_dfs,
percentage_style=(self.initial_risk is None))
append_to_statistics("Std trade return [{}]".format(unit), lambda df: df["returns"].std(), all_dfs,
percentage_style=(self.initial_risk is None))
def avg_positive_trade_return(df: QFDataFrame):
positive_trades = df[df["returns"] > 0]
return positive_trades["returns"].mean()
append_to_statistics("Average positive return [{}]".format(unit), avg_positive_trade_return, all_dfs,
percentage_style=(self.initial_risk is None))
def avg_negative_trade_return(df: QFDataFrame):
negative_trades = df[df["returns"] < 0]
return negative_trades["returns"].mean()
append_to_statistics("Average negative return [{}]".format(unit), avg_negative_trade_return, all_dfs,
percentage_style=(self.initial_risk is None))
append_to_statistics("Best trade return [{}]".format(unit), lambda df: df["returns"].max(), all_dfs,
percentage_style=(self.initial_risk is None))
append_to_statistics("Worst trade return [{}]".format(unit), lambda df: df["returns"].min(), all_dfs,
percentage_style=(self.initial_risk is None))
append_to_statistics("SQN (per trade) [{}]".format(unit), lambda df: sqn(df["returns"]), all_dfs,
percentage_style=(self.initial_risk is None))
append_to_statistics("SQN (per 100 trades) [{}]".format(unit), lambda df: sqn_for100trades(df["returns"]),
all_dfs, percentage_style=(self.initial_risk is None))
def sqn_per_year(returns: QFSeries):
sqn_per_year_value = sqn(returns) * sqrt(avg_nr_of_trades_per1y(returns, self.start_date, self.end_date))
return sqn_per_year_value
append_to_statistics("SQN (per year) [{}]".format(unit), lambda df: sqn_per_year(df["returns"]), all_dfs,
percentage_style=(self.initial_risk is None))
statistics_df = QFDataFrame.from_records(statistics, columns=["Measure", "All trades", "Long trades",
"Short trades"])
table = DFTable(statistics_df, css_classes=['table', 'left-align'])
table.add_columns_classes(["Measure"], 'wide-column')
self.document.add_element(table)