def test_api_locks(botclient):
ftbot, client = botclient
rc = client_get(client, f"{BASE_URI}/locks")
assert_response(rc)
assert 'locks' in rc.json()
assert rc.json()['lock_count'] == 0
assert rc.json()['lock_count'] == len(rc.json()['locks'])
PairLocks.lock_pair('ETH/BTC', datetime.now(timezone.utc) + timedelta(minutes=4), 'randreason')
PairLocks.lock_pair('XRP/BTC', datetime.now(timezone.utc) + timedelta(minutes=20), 'deadbeef')
rc = client_get(client, f"{BASE_URI}/locks")
assert_response(rc)
assert rc.json()['lock_count'] == 2
assert rc.json()['lock_count'] == len(rc.json()['locks'])
assert 'ETH/BTC' in (rc.json()['locks'][0]['pair'], rc.json()['locks'][1]['pair'])
assert 'randreason' in (rc.json()['locks'][0]['reason'], rc.json()['locks'][1]['reason'])
assert 'deadbeef' in (rc.json()['locks'][0]['reason'], rc.json()['locks'][1]['reason'])
# Test deletions
rc = client_delete(client, f"{BASE_URI}/locks/1")
assert_response(rc)
assert rc.json()['lock_count'] == 1
rc = client_post(client, f"{BASE_URI}/locks/delete",
data='{"pair": "XRP/BTC"}')
assert_response(rc)
assert rc.json()['lock_count'] == 0
def test_CooldownPeriod(mocker, default_conf, fee, caplog):
default_conf['protections'] = [{
"method": "CooldownPeriod",
"stop_duration": 60,
}]
freqtrade = get_patched_freqtradebot(mocker, default_conf)
message = r"Trading stopped due to .*"
assert not freqtrade.protections.global_stop()
assert not freqtrade.protections.stop_per_pair('XRP/BTC')
assert not log_has_re(message, caplog)
caplog.clear()
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value,
min_ago_open=200, min_ago_close=30,
))
assert not freqtrade.protections.global_stop()
assert freqtrade.protections.stop_per_pair('XRP/BTC')
assert PairLocks.is_pair_locked('XRP/BTC')
assert not PairLocks.is_global_lock()
Trade.query.session.add(generate_mock_trade(
'ETH/BTC', fee.return_value, False, sell_reason=SellType.ROI.value,
min_ago_open=205, min_ago_close=35,
))
assert not freqtrade.protections.global_stop()
assert not PairLocks.is_pair_locked('ETH/BTC')
assert freqtrade.protections.stop_per_pair('ETH/BTC')
assert PairLocks.is_pair_locked('ETH/BTC')
assert not PairLocks.is_global_lock()
def confirm_trade_entry(self, pair: str, order_type: str, amount: float, rate: float, time_in_force: str, **kwargs) -> bool:
bot_id = 1234567890
coin, currency = pair.split('/')
p3cw = Py3CW(
key='1',
secret='2',
)
logger.info(f"3Commas: Sending buy signal for {pair} to 3commas bot_id={bot_id}")
error, data = p3cw.request(
entity='bots',
action='start_new_deal',
action_id=f'{bot_id}',
payload={
"bot_id": bot_id,
"pair": f"{currency}_{coin}{currency}",
},
)
if error:
logger.error(f"3Commas: {error['msg']}")
PairLocks.lock_pair(
pair=pair,
until=datetime.now(timezone.utc) + timedelta(minutes=1),
reason="Send 3c buy order"
)
return False
def unlock_reason(self, reason: str) -> None:
"""
Unlocks all pairs previously locked using lock_pair with specified reason.
Not used by freqtrade itself, but intended to be used if users lock pairs
manually from within the strategy, to allow an easy way to unlock pairs.
:param reason: Unlock pairs to allow trading again
"""
PairLocks.unlock_reason(reason, datetime.now(timezone.utc))
def unlock_pair(self, pair: str) -> None:
"""
Unlocks a pair previously locked using lock_pair.
Not used by freqtrade itself, but intended to be used if users lock pairs
manually from within the strategy, to allow an easy way to unlock pairs.
:param pair: Unlock pair to allow trading again
"""
PairLocks.unlock_pair(pair, datetime.now(timezone.utc))
def prepare_backtest(self, enable_protections):
"""
Backtesting setup method - called once for every call to "backtest()".
"""
PairLocks.use_db = False
PairLocks.timeframe = self.config['timeframe']
Trade.use_db = False
PairLocks.reset_locks()
Trade.reset_trades()
def prepare_backtest(self, enable_protections):
"""
Backtesting setup method - called once for every call to "backtest()".
"""
PairLocks.use_db = False
Trade.use_db = False
if enable_protections:
# Reset persisted data - used for protections only
PairLocks.reset_locks()
Trade.reset_trades()
def test_stoploss_guard_perpair(mocker, default_conf, fee, caplog, only_per_pair):
default_conf['protections'] = [{
"method": "StoplossGuard",
"lookback_period": 60,
"trade_limit": 2,
"stop_duration": 60,
"only_per_pair": only_per_pair
}]
freqtrade = get_patched_freqtradebot(mocker, default_conf)
message = r"Trading stopped due to .*"
pair = 'XRP/BTC'
assert not freqtrade.protections.stop_per_pair(pair)
assert not freqtrade.protections.global_stop()
assert not log_has_re(message, caplog)
caplog.clear()
Trade.query.session.add(generate_mock_trade(
pair, fee.return_value, False, sell_reason=SellType.STOP_LOSS.value,
min_ago_open=200, min_ago_close=30, profit_rate=0.9,
))
assert not freqtrade.protections.stop_per_pair(pair)
assert not freqtrade.protections.global_stop()
assert not log_has_re(message, caplog)
caplog.clear()
# This trade does not count, as it's closed too long ago
Trade.query.session.add(generate_mock_trade(
pair, fee.return_value, False, sell_reason=SellType.STOP_LOSS.value,
min_ago_open=250, min_ago_close=100, profit_rate=0.9,
))
# Trade does not count for per pair stop as it's the wrong pair.
Trade.query.session.add(generate_mock_trade(
'ETH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value,
min_ago_open=240, min_ago_close=30, profit_rate=0.9,
))
# 3 Trades closed - but the 2nd has been closed too long ago.
assert not freqtrade.protections.stop_per_pair(pair)
assert freqtrade.protections.global_stop() != only_per_pair
if not only_per_pair:
assert log_has_re(message, caplog)
else:
assert not log_has_re(message, caplog)
caplog.clear()
# 2nd Trade that counts with correct pair
Trade.query.session.add(generate_mock_trade(
pair, fee.return_value, False, sell_reason=SellType.STOP_LOSS.value,
min_ago_open=180, min_ago_close=30, profit_rate=0.9,
))
assert freqtrade.protections.stop_per_pair(pair)
assert freqtrade.protections.global_stop() != only_per_pair
assert PairLocks.is_pair_locked(pair)
assert PairLocks.is_global_lock() != only_per_pair
def stop_per_pair(self, pair, now: Optional[datetime] = None) -> Optional[PairLock]:
if not now:
now = datetime.now(timezone.utc)
result = None
for protection_handler in self._protection_handlers:
if protection_handler.has_local_stop:
lock, until, reason = protection_handler.stop_per_pair(pair, now)
if lock and until:
if not PairLocks.is_pair_locked(pair, until):
result = PairLocks.lock_pair(pair, until, reason, now=now)
return result
def lock_pair(self, pair: str, until: datetime, reason: str = None) -> None:
"""
Locks pair until a given timestamp happens.
Locked pairs are not analyzed, and are prevented from opening new trades.
Locks can only count up (allowing users to lock pairs for a longer period of time).
To remove a lock from a pair, use `unlock_pair()`
:param pair: Pair to lock
:param until: datetime in UTC until the pair should be blocked from opening new trades.
Needs to be timezone aware `datetime.now(timezone.utc)`
:param reason: Optional string explaining why the pair was locked.
"""
PairLocks.lock_pair(pair, until, reason)
def prepare_backtest(self, enable_protections):
"""
Backtesting setup method - called once for every call to "backtest()".
"""
PairLocks.use_db = False
PairLocks.timeframe = self.config['timeframe']
Trade.use_db = False
PairLocks.reset_locks()
Trade.reset_trades()
self.rejected_trades = 0
self.dataprovider.clear_cache()
self._load_protections(self.strategy)
def global_stop(self, now: Optional[datetime] = None) -> Optional[PairLock]:
if not now:
now = datetime.now(timezone.utc)
result = None
for protection_handler in self._protection_handlers:
if protection_handler.has_global_stop:
lock, until, reason = protection_handler.global_stop(now)
# Early stopping - first positive result blocks further trades
if lock and until:
if not PairLocks.is_global_lock(until):
result = PairLocks.lock_pair('*', until, reason, now=now)
return result
def test_stoploss_guard(mocker, default_conf, fee, caplog):
default_conf['protections'] = [{
"method": "StoplossGuard",
"lookback_period": 60,
"stop_duration": 40,
"trade_limit": 3
}]
freqtrade = get_patched_freqtradebot(mocker, default_conf)
message = r"Trading stopped due to .*"
assert not freqtrade.protections.global_stop()
assert not log_has_re(message, caplog)
caplog.clear()
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value,
min_ago_open=200, min_ago_close=30,
))
assert not freqtrade.protections.global_stop()
assert not log_has_re(message, caplog)
caplog.clear()
# This trade does not count, as it's closed too long ago
Trade.query.session.add(generate_mock_trade(
'BCH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value,
min_ago_open=250, min_ago_close=100,
))
Trade.query.session.add(generate_mock_trade(
'ETH/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value,
min_ago_open=240, min_ago_close=30,
))
# 3 Trades closed - but the 2nd has been closed too long ago.
assert not freqtrade.protections.global_stop()
assert not log_has_re(message, caplog)
caplog.clear()
Trade.query.session.add(generate_mock_trade(
'LTC/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value,
min_ago_open=180, min_ago_close=30,
))
assert freqtrade.protections.global_stop()
assert log_has_re(message, caplog)
assert PairLocks.is_global_lock()
# Test 5m after lock-period - this should try and relock the pair, but end-time
# should be the previous end-time
end_time = PairLocks.get_pair_longest_lock('*').lock_end_time + timedelta(minutes=5)
assert freqtrade.protections.global_stop(end_time)
assert not PairLocks.is_global_lock(end_time)
def _rpc_locks(self) -> Dict[str, Any]:
""" Returns the current locks """
locks = PairLocks.get_pair_locks(None)
return {
'lock_count': len(locks),
'locks': [lock.to_json() for lock in locks]
}
def is_pair_locked(self, pair: str, candle_date: datetime = None) -> bool:
"""
Checks if a pair is currently locked
The 2nd, optional parameter ensures that locks are applied until the new candle arrives,
and not stop at 14:00:00 - while the next candle arrives at 14:00:02 leaving a gap
of 2 seconds for a buy to happen on an old signal.
:param: pair: "Pair to check"
:param candle_date: Date of the last candle. Optional, defaults to current date
:returns: locking state of the pair in question.
"""
if not candle_date:
# Simple call ...
return PairLocks.is_pair_locked(pair)
else:
lock_time = timeframe_to_next_date(self.timeframe, candle_date)
return PairLocks.is_pair_locked(pair, lock_time)
def backtest_one_strategy(self, strat: IStrategy, data: Dict[str, Any],
timerange: TimeRange):
logger.info("Running backtesting for Strategy %s",
strat.get_strategy_name())
backtest_start_time = datetime.now(timezone.utc)
self._set_strategy(strat)
strategy_safe_wrapper(self.strategy.bot_loop_start,
supress_error=True)()
# Use max_open_trades in backtesting, except --disable-max-market-positions is set
if self.config.get('use_max_market_positions', True):
# Must come from strategy config, as the strategy may modify this setting.
max_open_trades = self.strategy.config['max_open_trades']
else:
logger.info(
'Ignoring max_open_trades (--disable-max-market-positions was used) ...'
)
max_open_trades = 0
# need to reprocess data every time to populate signals
preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
# Trim startup period from analyzed dataframe
for pair, df in preprocessed.items():
preprocessed[pair] = trim_dataframe(
df, timerange, startup_candles=self.required_startup)
min_date, max_date = history.get_timerange(preprocessed)
logger.info(
f'Backtesting with data from {min_date.strftime(DATETIME_PRINT_FORMAT)} '
f'up to {max_date.strftime(DATETIME_PRINT_FORMAT)} '
f'({(max_date - min_date).days} days)..')
# Execute backtest and store results
results = self.backtest(
processed=preprocessed,
start_date=min_date.datetime,
end_date=max_date.datetime,
max_open_trades=max_open_trades,
position_stacking=self.config.get('position_stacking', False),
enable_protections=self.config.get('enable_protections', False),
)
backtest_end_time = datetime.now(timezone.utc)
self.all_results[self.strategy.get_strategy_name()] = {
'results':
results,
'config':
self.strategy.config,
'locks':
PairLocks.get_all_locks(),
'final_balance':
self.wallets.get_total(self.strategy.config['stake_currency']),
'backtest_start_time':
int(backtest_start_time.timestamp()),
'backtest_end_time':
int(backtest_end_time.timestamp()),
}
return min_date, max_date
def test_LowProfitPairs(mocker, default_conf, fee, caplog):
default_conf['protections'] = [{
"method": "LowProfitPairs",
"lookback_period": 400,
"stop_duration": 60,
"trade_limit": 2,
"required_profit": 0.0,
}]
freqtrade = get_patched_freqtradebot(mocker, default_conf)
message = r"Trading stopped due to .*"
assert not freqtrade.protections.global_stop()
assert not freqtrade.protections.stop_per_pair('XRP/BTC')
assert not log_has_re(message, caplog)
caplog.clear()
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value,
min_ago_open=800, min_ago_close=450, profit_rate=0.9,
))
# Not locked with 1 trade
assert not freqtrade.protections.global_stop()
assert not freqtrade.protections.stop_per_pair('XRP/BTC')
assert not PairLocks.is_pair_locked('XRP/BTC')
assert not PairLocks.is_global_lock()
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value,
min_ago_open=200, min_ago_close=120, profit_rate=0.9,
))
# Not locked with 1 trade (first trade is outside of lookback_period)
assert not freqtrade.protections.global_stop()
assert not freqtrade.protections.stop_per_pair('XRP/BTC')
assert not PairLocks.is_pair_locked('XRP/BTC')
assert not PairLocks.is_global_lock()
# Add positive trade
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=SellType.ROI.value,
min_ago_open=20, min_ago_close=10, profit_rate=1.15,
))
assert not freqtrade.protections.stop_per_pair('XRP/BTC')
assert not PairLocks.is_pair_locked('XRP/BTC')
Trade.query.session.add(generate_mock_trade(
'XRP/BTC', fee.return_value, False, sell_reason=SellType.STOP_LOSS.value,
min_ago_open=110, min_ago_close=20, profit_rate=0.8,
))
# Locks due to 2nd trade
assert not freqtrade.protections.global_stop()
assert freqtrade.protections.stop_per_pair('XRP/BTC')
assert PairLocks.is_pair_locked('XRP/BTC')
assert not PairLocks.is_global_lock()
def test_is_pair_locked(default_conf):
default_conf.update({'strategy': 'DefaultStrategy'})
PairLocks.timeframe = default_conf['timeframe']
PairLocks.use_db = True
strategy = StrategyResolver.load_strategy(default_conf)
# No lock should be present
assert len(PairLocks.get_pair_locks(None)) == 0
pair = 'ETH/BTC'
assert not strategy.is_pair_locked(pair)
strategy.lock_pair(pair, arrow.now(timezone.utc).shift(minutes=4).datetime)
# ETH/BTC locked for 4 minutes
assert strategy.is_pair_locked(pair)
# XRP/BTC should not be locked now
pair = 'XRP/BTC'
assert not strategy.is_pair_locked(pair)
# Unlocking a pair that's not locked should not raise an error
strategy.unlock_pair(pair)
# Unlock original pair
pair = 'ETH/BTC'
strategy.unlock_pair(pair)
assert not strategy.is_pair_locked(pair)
pair = 'BTC/USDT'
# Lock until 14:30
lock_time = datetime(2020, 5, 1, 14, 30, 0, tzinfo=timezone.utc)
# Subtract 2 seconds, as locking rounds up to the next candle.
strategy.lock_pair(pair, lock_time - timedelta(seconds=2))
assert not strategy.is_pair_locked(pair)
# latest candle is from 14:20, lock goes to 14:30
assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-10))
assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-50))
# latest candle is from 14:25 (lock should be lifted)
# Since this is the "new candle" available at 14:30
assert not strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-4))
# Should not be locked after time expired
assert not strategy.is_pair_locked(pair, lock_time + timedelta(minutes=10))
# Change timeframe to 15m
strategy.timeframe = '15m'
# Candle from 14:14 - lock goes until 14:30
assert strategy.is_pair_locked(pair, lock_time + timedelta(minutes=-16))
assert strategy.is_pair_locked(
pair, lock_time + timedelta(minutes=-15, seconds=-2))
# Candle from 14:15 - lock goes until 14:30
assert not strategy.is_pair_locked(pair,
lock_time + timedelta(minutes=-15))
def _rpc_delete_lock(self, lockid: Optional[int] = None,
pair: Optional[str] = None) -> Dict[str, Any]:
""" Delete specific lock(s) """
locks = []
if pair:
locks = PairLocks.get_pair_locks(pair)
if lockid:
locks = PairLock.query.filter(PairLock.id == lockid).all()
for lock in locks:
lock.active = False
lock.lock_end_time = datetime.now(timezone.utc)
PairLock.query.session.commit()
return self._rpc_locks()
def backtest(self,
processed: Dict,
start_date: datetime,
end_date: datetime,
max_open_trades: int = 0,
position_stacking: bool = False,
enable_protections: bool = False) -> Dict[str, Any]:
"""
Implement backtesting functionality
NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized.
Of course try to not have ugly code. By some accessor are sometime slower than functions.
Avoid extensive logging in this method and functions it calls.
:param processed: a processed dictionary with format {pair, data}, which gets cleared to
optimize memory usage!
:param start_date: backtesting timerange start datetime
:param end_date: backtesting timerange end datetime
:param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited
:param position_stacking: do we allow position stacking?
:param enable_protections: Should protections be enabled?
:return: DataFrame with trades (results of backtesting)
"""
trades: List[LocalTrade] = []
self.prepare_backtest(enable_protections)
# Use dict of lists with data for performance
# (looping lists is a lot faster than pandas DataFrames)
data: Dict = self._get_ohlcv_as_lists(processed)
# Indexes per pair, so some pairs are allowed to have a missing start.
indexes: Dict = defaultdict(int)
tmp = start_date + timedelta(minutes=self.timeframe_min)
open_trades: Dict[str, List[LocalTrade]] = defaultdict(list)
open_trade_count = 0
self.progress.init_step(
BacktestState.BACKTEST,
int((end_date - start_date) /
timedelta(minutes=self.timeframe_min)))
# Loop timerange and get candle for each pair at that point in time
while tmp <= end_date:
open_trade_count_start = open_trade_count
self.check_abort()
for i, pair in enumerate(data):
row_index = indexes[pair]
try:
# Row is treated as "current incomplete candle".
# Buy / sell signals are shifted by 1 to compensate for this.
row = data[pair][row_index]
except IndexError:
# missing Data for one pair at the end.
# Warnings for this are shown during data loading
continue
# Waits until the time-counter reaches the start of the data for this pair.
if row[DATE_IDX] > tmp:
continue
row_index += 1
indexes[pair] = row_index
self.dataprovider._set_dataframe_max_index(row_index)
# without positionstacking, we can only have one open trade per pair.
# max_open_trades must be respected
# don't open on the last row
if ((position_stacking or len(open_trades[pair]) == 0)
and self.trade_slot_available(max_open_trades,
open_trade_count_start)
and tmp != end_date and row[BUY_IDX] == 1
and row[SELL_IDX] != 1
and not PairLocks.is_pair_locked(pair, row[DATE_IDX])):
trade = self._enter_trade(pair, row)
if trade:
# TODO: hacky workaround to avoid opening > max_open_trades
# This emulates previous behaviour - not sure if this is correct
# Prevents buying if the trade-slot was freed in this candle
open_trade_count_start += 1
open_trade_count += 1
# logger.debug(f"{pair} - Emulate creation of new trade: {trade}.")
open_trades[pair].append(trade)
LocalTrade.add_bt_trade(trade)
for trade in list(open_trades[pair]):
# also check the buying candle for sell conditions.
trade_entry = self._get_sell_trade_entry(trade, row)
# Sell occurred
if trade_entry:
# logger.debug(f"{pair} - Backtesting sell {trade}")
open_trade_count -= 1
open_trades[pair].remove(trade)
LocalTrade.close_bt_trade(trade)
trades.append(trade_entry)
if enable_protections:
self.protections.stop_per_pair(pair, row[DATE_IDX])
self.protections.global_stop(tmp)
# Move time one configured time_interval ahead.
self.progress.increment()
tmp += timedelta(minutes=self.timeframe_min)
trades += self.handle_left_open(open_trades, data=data)
self.wallets.update()
results = trade_list_to_dataframe(trades)
return {
'results':
results,
'config':
self.strategy.config,
'locks':
PairLocks.get_all_locks(),
'rejected_signals':
self.rejected_trades,
'final_balance':
self.wallets.get_total(self.strategy.config['stake_currency']),
}
def __init__(self, config: Dict[str, Any]) -> None:
LoggingMixin.show_output = False
self.config = config
# Reset keys for backtesting
remove_credentials(self.config)
self.strategylist: List[IStrategy] = []
self.all_results: Dict[str, Dict] = {}
self.exchange = ExchangeResolver.load_exchange(
self.config['exchange']['name'], self.config)
self.dataprovider = DataProvider(self.config, None)
if self.config.get('strategy_list', None):
for strat in list(self.config['strategy_list']):
stratconf = deepcopy(self.config)
stratconf['strategy'] = strat
self.strategylist.append(
StrategyResolver.load_strategy(stratconf))
validate_config_consistency(stratconf)
else:
# No strategy list specified, only one strategy
self.strategylist.append(
StrategyResolver.load_strategy(self.config))
validate_config_consistency(self.config)
if "timeframe" not in self.config:
raise OperationalException(
"Timeframe (ticker interval) needs to be set in either "
"configuration or as cli argument `--timeframe 5m`")
self.timeframe = str(self.config.get('timeframe'))
self.timeframe_min = timeframe_to_minutes(self.timeframe)
self.pairlists = PairListManager(self.exchange, self.config)
if 'VolumePairList' in self.pairlists.name_list:
raise OperationalException(
"VolumePairList not allowed for backtesting.")
if 'PerformanceFilter' in self.pairlists.name_list:
raise OperationalException(
"PerformanceFilter not allowed for backtesting.")
if len(self.strategylist
) > 1 and 'PrecisionFilter' in self.pairlists.name_list:
raise OperationalException(
"PrecisionFilter not allowed for backtesting multiple strategies."
)
self.dataprovider.add_pairlisthandler(self.pairlists)
self.pairlists.refresh_pairlist()
if len(self.pairlists.whitelist) == 0:
raise OperationalException("No pair in whitelist.")
if config.get('fee', None) is not None:
self.fee = config['fee']
else:
self.fee = self.exchange.get_fee(
symbol=self.pairlists.whitelist[0])
Trade.use_db = False
Trade.reset_trades()
PairLocks.timeframe = self.config['timeframe']
PairLocks.use_db = False
PairLocks.reset_locks()
self.wallets = Wallets(self.config, self.exchange, log=False)
# Get maximum required startup period
self.required_startup = max(
[strat.startup_candle_count for strat in self.strategylist])
def test_PairLocks_reason(use_db):
PairLocks.timeframe = '5m'
PairLocks.use_db = use_db
# No lock should be present
if use_db:
assert len(PairLock.query.all()) == 0
assert PairLocks.use_db == use_db
PairLocks.lock_pair('XRP/USDT',
arrow.utcnow().shift(minutes=4).datetime, 'TestLock1')
PairLocks.lock_pair('ETH/USDT',
arrow.utcnow().shift(minutes=4).datetime, 'TestLock2')
assert PairLocks.is_pair_locked('XRP/USDT')
assert PairLocks.is_pair_locked('ETH/USDT')
PairLocks.unlock_reason('TestLock1')
assert not PairLocks.is_pair_locked('XRP/USDT')
assert PairLocks.is_pair_locked('ETH/USDT')
PairLocks.reset_locks()
PairLocks.use_db = True
def test_MaxDrawdown(mocker, default_conf, fee, caplog):
default_conf['protections'] = [{
"method": "MaxDrawdown",
"lookback_period": 1000,
"stop_duration": 60,
"trade_limit": 3,
"max_allowed_drawdown": 0.15
}]
freqtrade = get_patched_freqtradebot(mocker, default_conf)
message = r"Trading stopped due to Max.*"
assert not freqtrade.protections.global_stop()
assert not freqtrade.protections.stop_per_pair('XRP/BTC')
caplog.clear()
Trade.session.add(
generate_mock_trade(
'XRP/BTC',
fee.return_value,
False,
sell_reason=SellType.STOP_LOSS.value,
min_ago_open=1000,
min_ago_close=900,
profit_rate=1.1,
))
Trade.session.add(
generate_mock_trade(
'ETH/BTC',
fee.return_value,
False,
sell_reason=SellType.STOP_LOSS.value,
min_ago_open=1000,
min_ago_close=900,
profit_rate=1.1,
))
Trade.session.add(
generate_mock_trade(
'NEO/BTC',
fee.return_value,
False,
sell_reason=SellType.STOP_LOSS.value,
min_ago_open=1000,
min_ago_close=900,
profit_rate=1.1,
))
# No losing trade yet ... so max_drawdown will raise exception
assert not freqtrade.protections.global_stop()
assert not freqtrade.protections.stop_per_pair('XRP/BTC')
Trade.session.add(
generate_mock_trade(
'XRP/BTC',
fee.return_value,
False,
sell_reason=SellType.STOP_LOSS.value,
min_ago_open=500,
min_ago_close=400,
profit_rate=0.9,
))
# Not locked with one trade
assert not freqtrade.protections.global_stop()
assert not freqtrade.protections.stop_per_pair('XRP/BTC')
assert not PairLocks.is_pair_locked('XRP/BTC')
assert not PairLocks.is_global_lock()
Trade.session.add(
generate_mock_trade(
'XRP/BTC',
fee.return_value,
False,
sell_reason=SellType.STOP_LOSS.value,
min_ago_open=1200,
min_ago_close=1100,
profit_rate=0.5,
))
# Not locked with 1 trade (2nd trade is outside of lookback_period)
assert not freqtrade.protections.global_stop()
assert not freqtrade.protections.stop_per_pair('XRP/BTC')
assert not PairLocks.is_pair_locked('XRP/BTC')
assert not PairLocks.is_global_lock()
assert not log_has_re(message, caplog)
# Winning trade ... (should not lock, does not change drawdown!)
Trade.session.add(
generate_mock_trade(
'XRP/BTC',
fee.return_value,
False,
sell_reason=SellType.ROI.value,
min_ago_open=320,
min_ago_close=410,
profit_rate=1.5,
))
assert not freqtrade.protections.global_stop()
assert not PairLocks.is_global_lock()
caplog.clear()
# Add additional negative trade, causing a loss of > 15%
Trade.session.add(
generate_mock_trade(
'XRP/BTC',
fee.return_value,
False,
sell_reason=SellType.ROI.value,
min_ago_open=20,
min_ago_close=10,
profit_rate=0.8,
))
assert not freqtrade.protections.stop_per_pair('XRP/BTC')
# local lock not supported
assert not PairLocks.is_pair_locked('XRP/BTC')
assert freqtrade.protections.global_stop()
assert PairLocks.is_global_lock()
assert log_has_re(message, caplog)
def test_PairLocks(use_db):
PairLocks.timeframe = '5m'
# No lock should be present
if use_db:
assert len(PairLock.query.all()) == 0
else:
PairLocks.use_db = False
assert PairLocks.use_db == use_db
pair = 'ETH/BTC'
assert not PairLocks.is_pair_locked(pair)
PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime)
# ETH/BTC locked for 4 minutes
assert PairLocks.is_pair_locked(pair)
# XRP/BTC should not be locked now
pair = 'XRP/BTC'
assert not PairLocks.is_pair_locked(pair)
# Unlocking a pair that's not locked should not raise an error
PairLocks.unlock_pair(pair)
PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime)
assert PairLocks.is_pair_locked(pair)
# Get both locks from above
locks = PairLocks.get_pair_locks(None)
assert len(locks) == 2
# Unlock original pair
pair = 'ETH/BTC'
PairLocks.unlock_pair(pair)
assert not PairLocks.is_pair_locked(pair)
assert not PairLocks.is_global_lock()
pair = 'BTC/USDT'
# Lock until 14:30
lock_time = datetime(2020, 5, 1, 14, 30, 0, tzinfo=timezone.utc)
PairLocks.lock_pair(pair, lock_time)
assert not PairLocks.is_pair_locked(pair)
assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-10))
assert not PairLocks.is_global_lock(lock_time + timedelta(minutes=-10))
assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-50))
assert not PairLocks.is_global_lock(lock_time + timedelta(minutes=-50))
# Should not be locked after time expired
assert not PairLocks.is_pair_locked(pair,
lock_time + timedelta(minutes=10))
locks = PairLocks.get_pair_locks(pair, lock_time + timedelta(minutes=-2))
assert len(locks) == 1
assert 'PairLock' in str(locks[0])
# Unlock all
PairLocks.unlock_pair(pair, lock_time + timedelta(minutes=-2))
assert not PairLocks.is_global_lock(lock_time + timedelta(minutes=-50))
# Global lock
PairLocks.lock_pair('*', lock_time)
assert PairLocks.is_global_lock(lock_time + timedelta(minutes=-50))
# Global lock also locks every pair seperately
assert PairLocks.is_pair_locked(pair, lock_time + timedelta(minutes=-50))
assert PairLocks.is_pair_locked('XRP/USDT',
lock_time + timedelta(minutes=-50))
if use_db:
assert len(PairLock.query.all()) > 0
else:
# Nothing was pushed to the database
assert len(PairLock.query.all()) == 0
# Reset use-db variable
PairLocks.use_db = True
def backtest(self,
processed: Dict,
start_date: datetime,
end_date: datetime,
max_open_trades: int = 0,
position_stacking: bool = False,
enable_protections: bool = False) -> Dict[str, Any]:
"""
Implement backtesting functionality
NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized.
Of course try to not have ugly code. By some accessor are sometime slower than functions.
Avoid extensive logging in this method and functions it calls.
:param processed: a processed dictionary with format {pair, data}, which gets cleared to
optimize memory usage!
:param start_date: backtesting timerange start datetime
:param end_date: backtesting timerange end datetime
:param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited
:param position_stacking: do we allow position stacking?
:param enable_protections: Should protections be enabled?
:return: DataFrame with trades (results of backtesting)
"""
trades: List[LocalTrade] = []
self.prepare_backtest(enable_protections)
# Ensure wallets are uptodate (important for --strategy-list)
self.wallets.update()
# Use dict of lists with data for performance
# (looping lists is a lot faster than pandas DataFrames)
data: Dict = self._get_ohlcv_as_lists(processed)
# Indexes per pair, so some pairs are allowed to have a missing start.
indexes: Dict = defaultdict(int)
current_time = start_date + timedelta(minutes=self.timeframe_min)
open_trades: Dict[str, List[LocalTrade]] = defaultdict(list)
open_trade_count = 0
self.progress.init_step(
BacktestState.BACKTEST,
int((end_date - start_date) /
timedelta(minutes=self.timeframe_min)))
# Loop timerange and get candle for each pair at that point in time
while current_time <= end_date:
open_trade_count_start = open_trade_count
self.check_abort()
for i, pair in enumerate(data):
row_index = indexes[pair]
row = self.validate_row(data, pair, row_index, current_time)
if not row:
continue
row_index += 1
indexes[pair] = row_index
self.dataprovider._set_dataframe_max_index(row_index)
# 1. Process buys.
# without positionstacking, we can only have one open trade per pair.
# max_open_trades must be respected
# don't open on the last row
if ((position_stacking or len(open_trades[pair]) == 0)
and self.trade_slot_available(max_open_trades,
open_trade_count_start)
and current_time != end_date and row[BUY_IDX] == 1
and row[SELL_IDX] != 1
and not PairLocks.is_pair_locked(pair, row[DATE_IDX])):
trade = self._enter_trade(pair, row)
if trade:
# TODO: hacky workaround to avoid opening > max_open_trades
# This emulates previous behavior - not sure if this is correct
# Prevents buying if the trade-slot was freed in this candle
open_trade_count_start += 1
open_trade_count += 1
# logger.debug(f"{pair} - Emulate creation of new trade: {trade}.")
open_trades[pair].append(trade)
for trade in list(open_trades[pair]):
# 2. Process buy orders.
order = trade.select_order('buy', is_open=True)
if order and self._get_order_filled(order.price, row):
order.close_bt_order(current_time)
trade.open_order_id = None
LocalTrade.add_bt_trade(trade)
self.wallets.update()
# 3. Create sell orders (if any)
if not trade.open_order_id:
self._get_sell_trade_entry(
trade, row) # Place sell order if necessary
# 4. Process sell orders.
order = trade.select_order('sell', is_open=True)
if order and self._get_order_filled(order.price, row):
trade.open_order_id = None
trade.close_date = current_time
trade.close(order.price, show_msg=False)
# logger.debug(f"{pair} - Backtesting sell {trade}")
open_trade_count -= 1
open_trades[pair].remove(trade)
LocalTrade.close_bt_trade(trade)
trades.append(trade)
self.wallets.update()
self.run_protections(enable_protections, pair,
current_time)
# 5. Cancel expired buy/sell orders.
if self.check_order_cancel(trade, current_time):
# Close trade due to buy timeout expiration.
open_trade_count -= 1
open_trades[pair].remove(trade)
self.wallets.update()
# Move time one configured time_interval ahead.
self.progress.increment()
current_time += timedelta(minutes=self.timeframe_min)
trades += self.handle_left_open(open_trades, data=data)
self.wallets.update()
results = trade_list_to_dataframe(trades)
return {
'results':
results,
'config':
self.strategy.config,
'locks':
PairLocks.get_all_locks(),
'rejected_signals':
self.rejected_trades,
'timedout_entry_orders':
self.timedout_entry_orders,
'timedout_exit_orders':
self.timedout_exit_orders,
'final_balance':
self.wallets.get_total(self.strategy.config['stake_currency']),
}
def backtest(self,
processed: Dict,
stake_amount: float,
start_date: datetime,
end_date: datetime,
max_open_trades: int = 0,
position_stacking: bool = False,
enable_protections: bool = False) -> DataFrame:
"""
Implement backtesting functionality
NOTE: This method is used by Hyperopt at each iteration. Please keep it optimized.
Of course try to not have ugly code. By some accessor are sometime slower than functions.
Avoid extensive logging in this method and functions it calls.
:param processed: a processed dictionary with format {pair, data}
:param stake_amount: amount to use for each trade
:param start_date: backtesting timerange start datetime
:param end_date: backtesting timerange end datetime
:param max_open_trades: maximum number of concurrent trades, <= 0 means unlimited
:param position_stacking: do we allow position stacking?
:param enable_protections: Should protections be enabled?
:return: DataFrame with trades (results of backtesting)
"""
logger.debug(
f"Run backtest, stake_amount: {stake_amount}, "
f"start_date: {start_date}, end_date: {end_date}, "
f"max_open_trades: {max_open_trades}, position_stacking: {position_stacking}"
)
trades = []
self.prepare_backtest(enable_protections)
# Use dict of lists with data for performance
# (looping lists is a lot faster than pandas DataFrames)
data: Dict = self._get_ohlcv_as_lists(processed)
# Indexes per pair, so some pairs are allowed to have a missing start.
indexes: Dict = {}
tmp = start_date + timedelta(minutes=self.timeframe_min)
open_trades: Dict[str, List] = defaultdict(list)
open_trade_count = 0
# Loop timerange and get candle for each pair at that point in time
while tmp <= end_date:
open_trade_count_start = open_trade_count
for i, pair in enumerate(data):
if pair not in indexes:
indexes[pair] = 0
try:
row = data[pair][indexes[pair]]
except IndexError:
# missing Data for one pair at the end.
# Warnings for this are shown during data loading
continue
# Waits until the time-counter reaches the start of the data for this pair.
if row[DATE_IDX] > tmp:
continue
indexes[pair] += 1
# without positionstacking, we can only have one open trade per pair.
# max_open_trades must be respected
# don't open on the last row
if ((position_stacking or len(open_trades[pair]) == 0)
and (max_open_trades <= 0
or open_trade_count_start < max_open_trades)
and tmp != end_date and row[BUY_IDX] == 1
and row[SELL_IDX] != 1
and not PairLocks.is_pair_locked(pair, row[DATE_IDX])):
# Enter trade
trade = Trade(
pair=pair,
open_rate=row[OPEN_IDX],
open_date=row[DATE_IDX],
stake_amount=stake_amount,
amount=round(stake_amount / row[OPEN_IDX], 8),
fee_open=self.fee,
fee_close=self.fee,
is_open=True,
)
# TODO: hacky workaround to avoid opening > max_open_trades
# This emulates previous behaviour - not sure if this is correct
# Prevents buying if the trade-slot was freed in this candle
open_trade_count_start += 1
open_trade_count += 1
# logger.debug(f"{pair} - Backtesting emulates creation of new trade: {trade}.")
open_trades[pair].append(trade)
Trade.trades.append(trade)
for trade in open_trades[pair]:
# since indexes has been incremented before, we need to go one step back to
# also check the buying candle for sell conditions.
trade_entry = self._get_sell_trade_entry(trade, row)
# Sell occured
if trade_entry:
# logger.debug(f"{pair} - Backtesting sell {trade}")
open_trade_count -= 1
open_trades[pair].remove(trade)
trades.append(trade_entry)
if enable_protections:
self.protections.stop_per_pair(pair, row[DATE_IDX])
self.protections.global_stop(tmp)
# Move time one configured time_interval ahead.
tmp += timedelta(minutes=self.timeframe_min)
trades += self.handle_left_open(open_trades, data=data)
return DataFrame.from_records(trades, columns=BacktestResult._fields)
def test_PairLocks_getlongestlock(use_db):
PairLocks.timeframe = '5m'
# No lock should be present
PairLocks.use_db = use_db
if use_db:
assert len(PairLock.query.all()) == 0
assert PairLocks.use_db == use_db
pair = 'ETH/BTC'
assert not PairLocks.is_pair_locked(pair)
PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=4).datetime)
# ETH/BTC locked for 4 minutes
assert PairLocks.is_pair_locked(pair)
lock = PairLocks.get_pair_longest_lock(pair)
assert lock.lock_end_time.replace(
tzinfo=timezone.utc) > arrow.utcnow().shift(minutes=3)
assert lock.lock_end_time.replace(
tzinfo=timezone.utc) < arrow.utcnow().shift(minutes=14)
PairLocks.lock_pair(pair, arrow.utcnow().shift(minutes=15).datetime)
assert PairLocks.is_pair_locked(pair)
lock = PairLocks.get_pair_longest_lock(pair)
# Must be longer than above
assert lock.lock_end_time.replace(
tzinfo=timezone.utc) > arrow.utcnow().shift(minutes=14)
PairLocks.reset_locks()
PairLocks.use_db = True
