def __init__(self, config: dict) -> None:
self.config = config
# Dict to determine if analysis is necessary
self._last_candle_seen_per_pair: Dict[str, datetime] = {}
super().__init__(config)
# Gather informative pairs from @informative-decorated methods.
self._ft_informative: List[Tuple[InformativeData,
PopulateIndicators]] = []
for attr_name in dir(self.__class__):
cls_method = getattr(self.__class__, attr_name)
if not callable(cls_method):
continue
informative_data_list = getattr(cls_method, '_ft_informative',
None)
if not isinstance(informative_data_list, list):
# Type check is required because mocker would return a mock object that evaluates to
# True, confusing this code.
continue
strategy_timeframe_minutes = timeframe_to_minutes(self.timeframe)
for informative_data in informative_data_list:
if timeframe_to_minutes(informative_data.timeframe
) < strategy_timeframe_minutes:
raise OperationalException(
'Informative timeframe must be equal or higher than '
'strategy timeframe!')
self._ft_informative.append((informative_data, cls_method))
def merge_informative_pair(dataframe: pd.DataFrame,
informative: pd.DataFrame,
timeframe: str,
timeframe_inf: str,
ffill: bool = True) -> pd.DataFrame:
"""
Correctly merge informative samples to the original dataframe, avoiding lookahead bias.
Since dates are candle open dates, merging a 15m candle that starts at 15:00, and a
1h candle that starts at 15:00 will result in all candles to know the close at 16:00
which they should not know.
Moves the date of the informative pair by 1 time interval forward.
This way, the 14:00 1h candle is merged to 15:00 15m candle, since the 14:00 1h candle is the
last candle that's closed at 15:00, 15:15, 15:30 or 15:45.
Assuming inf_tf = '1d' - then the resulting columns will be:
date_1d, open_1d, high_1d, low_1d, close_1d, rsi_1d
:param dataframe: Original dataframe
:param informative: Informative pair, most likely loaded via dp.get_pair_dataframe
:param timeframe: Timeframe of the original pair sample.
:param timeframe_inf: Timeframe of the informative pair sample.
:param ffill: Forwardfill missing values - optional but usually required
"""
minutes_inf = timeframe_to_minutes(timeframe_inf)
minutes = timeframe_to_minutes(timeframe)
if minutes >= minutes_inf:
# No need to forwardshift if the timeframes are identical
informative['date_merge'] = informative["date"]
else:
informative['date_merge'] = informative["date"] + pd.to_timedelta(
minutes_inf, 'm')
# Rename columns to be unique
informative.columns = [
f"{col}_{timeframe_inf}" for col in informative.columns
]
# Combine the 2 dataframes
# all indicators on the informative sample MUST be calculated before this point
dataframe = pd.merge(dataframe,
informative,
left_on='date',
right_on=f'date_merge_{timeframe_inf}',
how='left')
dataframe = dataframe.drop(f'date_merge_{timeframe_inf}', axis=1)
if ffill:
dataframe = dataframe.ffill()
return dataframe
def populate_indicators(self, dataframe: DataFrame,
metadata: dict) -> DataFrame:
# Populate/update the trade data if there is any, set trades to false if not live/dry
self.custom_trade_info[metadata['pair']] = self.populate_trades(
metadata['pair'])
if self.config['runmode'].value in ('backtest', 'hyperopt'):
assert (timeframe_to_minutes(self.timeframe) <=
30), "Backtest this strategy in 5m or 1m timeframe."
if self.timeframe == self.informative_timeframe:
dataframe = self.do_indicators(dataframe, metadata)
else:
if not self.dp:
return dataframe
informative = self.dp.get_pair_dataframe(
pair=metadata['pair'], timeframe=self.informative_timeframe)
informative = self.do_indicators(informative.copy(), metadata)
dataframe = merge_informative_pair(dataframe,
informative,
self.timeframe,
self.informative_timeframe,
ffill=True)
skip_columns = [(s + "_" + self.informative_timeframe) for s in [
'date', 'open', 'high', 'low', 'close', 'volume', 'emac',
'emao'
]]
dataframe.rename(columns=lambda s: s.replace(
"_{}".format(self.informative_timeframe), "")
if (not s in skip_columns) else s,
inplace=True)
# Slam some indicators into the trade_info dict so we can dynamic roi and custom stoploss in backtest
if self.dp.runmode.value in ('backtest', 'hyperopt'):
self.custom_trade_info[metadata['pair']]['roc'] = dataframe[[
'date', 'roc'
]].copy().set_index('date')
self.custom_trade_info[metadata['pair']]['atr'] = dataframe[[
'date', 'atr'
]].copy().set_index('date')
self.custom_trade_info[metadata['pair']]['sroc'] = dataframe[[
'date', 'sroc'
]].copy().set_index('date')
self.custom_trade_info[metadata['pair']]['ssl-dir'] = dataframe[[
'date', 'ssl-dir'
]].copy().set_index('date')
self.custom_trade_info[
metadata['pair']]['rmi-up-trend'] = dataframe[[
'date', 'rmi-up-trend'
]].copy().set_index('date')
self.custom_trade_info[
metadata['pair']]['candle-up-trend'] = dataframe[[
'date', 'candle-up-trend'
]].copy().set_index('date')
return dataframe
def populate_indicators(self, dataframe: DataFrame,
metadata: dict) -> DataFrame:
assert (timeframe_to_minutes(
self.timeframe) == 5), "Run this strategy at 5m."
if self.timeframe == self.informative_timeframe:
dataframe = self.slow_tf_indicators(dataframe, metadata)
else:
assert self.dp, "DataProvider is required for multiple timeframes."
informative = self.dp.get_pair_dataframe(
pair=metadata['pair'], timeframe=self.informative_timeframe)
informative = self.slow_tf_indicators(informative.copy(), metadata)
dataframe = merge_informative_pair(dataframe,
informative,
self.timeframe,
self.informative_timeframe,
ffill=True)
# don't overwrite the base dataframe's OHLCV information
skip_columns = [
(s + "_" + self.informative_timeframe)
for s in ['date', 'open', 'high', 'low', 'close', 'volume']
]
dataframe.rename(columns=lambda s: s.replace(
"_{}".format(self.informative_timeframe), "")
if (not s in skip_columns) else s,
inplace=True)
dataframe = self.fast_tf_indicators(dataframe, metadata)
return dataframe
def __init__(self, config: Dict[str, Any],
protection_config: Dict[str, Any]) -> None:
self._config = config
self._protection_config = protection_config
self._stop_duration_candles: Optional[int] = None
self._lookback_period_candles: Optional[int] = None
tf_in_min = timeframe_to_minutes(config['timeframe'])
if 'stop_duration_candles' in protection_config:
self._stop_duration_candles = int(
protection_config.get('stop_duration_candles', 1))
self._stop_duration = (tf_in_min * self._stop_duration_candles)
else:
self._stop_duration_candles = None
self._stop_duration = protection_config.get('stop_duration', 60)
if 'lookback_period_candles' in protection_config:
self._lookback_period_candles = int(
protection_config.get('lookback_period_candles', 1))
self._lookback_period = tf_in_min * self._lookback_period_candles
else:
self._lookback_period_candles = None
self._lookback_period = int(
protection_config.get('lookback_period', 60))
LoggingMixin.__init__(self, logger)
def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str, pair: str) -> DataFrame:
"""
Fills up missing data with 0 volume rows,
using the previous close as price for "open", "high" "low" and "close", volume is set to 0
"""
from freqtrade.exchange import timeframe_to_minutes
ohlc_dict = {
'open': 'first',
'high': 'max',
'low': 'min',
'close': 'last',
'volume': 'sum'
}
ticker_minutes = timeframe_to_minutes(timeframe)
# Resample to create "NAN" values
df = dataframe.resample(f'{ticker_minutes}min', on='date').agg(ohlc_dict)
# Forwardfill close for missing columns
df['close'] = df['close'].fillna(method='ffill')
# Use close for "open, high, low"
df.loc[:, ['open', 'high', 'low']] = df[['open', 'high', 'low']].fillna(
value={'open': df['close'],
'high': df['close'],
'low': df['close'],
})
df.reset_index(inplace=True)
len_before = len(dataframe)
len_after = len(df)
if len_before != len_after:
logger.info(f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}")
return df
def _rpc_show_config(self) -> Dict[str, Any]:
"""
Return a dict of config options.
Explicitly does NOT return the full config to avoid leakage of sensitive
information via rpc.
"""
config = self._freqtrade.config
val = {
'dry_run': config['dry_run'],
'stake_currency': config['stake_currency'],
'stake_amount': config['stake_amount'],
'max_open_trades': config['max_open_trades'],
'minimal_roi': config['minimal_roi'].copy(),
'stoploss': config['stoploss'],
'trailing_stop': config['trailing_stop'],
'trailing_stop_positive': config.get('trailing_stop_positive'),
'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset'),
'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached'),
'ticker_interval': config['timeframe'], # DEPRECATED
'timeframe': config['timeframe'],
'timeframe_ms': timeframe_to_msecs(config['timeframe']),
'timeframe_min': timeframe_to_minutes(config['timeframe']),
'exchange': config['exchange']['name'],
'strategy': config['strategy'],
'forcebuy_enabled': config.get('forcebuy_enable', False),
'ask_strategy': config.get('ask_strategy', {}),
'bid_strategy': config.get('bid_strategy', {}),
'state': str(self._freqtrade.state)
}
return val
def populate_indicators(self, dataframe: DataFrame,
metadata: dict) -> DataFrame:
if self.config['runmode'].value in ('backtest', 'hyperopt'):
assert (timeframe_to_minutes(self.timeframe) <=
5), "Backtest this strategy in 5m or 1m timeframe."
if self.timeframe == self.informative_timeframe:
dataframe = self.do_indicators(dataframe, metadata)
else:
if not self.dp:
return dataframe
informative = self.dp.get_pair_dataframe(
pair=metadata['pair'], timeframe=self.informative_timeframe)
informative = self.do_indicators(informative.copy(), metadata)
dataframe = merge_informative_pair(dataframe,
informative,
self.timeframe,
self.informative_timeframe,
ffill=True)
# don't overwrite the base dataframe's OHLCV information
skip_columns = [
(s + "_" + self.informative_timeframe)
for s in ['date', 'open', 'high', 'low', 'close', 'volume']
]
dataframe.rename(columns=lambda s: s.replace(
"_{}".format(self.informative_timeframe), "")
if (not s in skip_columns) else s,
inplace=True)
return dataframe
def __init__(self, config: Dict[str, Any]) -> None:
self.config = config
# Reset keys for backtesting
remove_credentials(self.config)
self.strategylist: List[IStrategy] = []
self.exchange = ExchangeResolver.load_exchange(
self.config['exchange']['name'], self.config)
dataprovider = DataProvider(self.config, self.exchange)
IStrategy.dp = dataprovider
if self.config.get('strategy_list', None):
for strat in list(self.config['strategy_list']):
stratconf = deepcopy(self.config)
stratconf['strategy'] = strat
self.strategylist.append(
StrategyResolver.load_strategy(stratconf))
validate_config_consistency(stratconf)
else:
# No strategy list specified, only one strategy
self.strategylist.append(
StrategyResolver.load_strategy(self.config))
validate_config_consistency(self.config)
if "timeframe" not in self.config:
raise OperationalException(
"Timeframe (ticker interval) needs to be set in either "
"configuration or as cli argument `--timeframe 5m`")
self.timeframe = str(self.config.get('timeframe'))
self.timeframe_min = timeframe_to_minutes(self.timeframe)
self.pairlists = PairListManager(self.exchange, self.config)
if 'VolumePairList' in self.pairlists.name_list:
raise OperationalException(
"VolumePairList not allowed for backtesting.")
if len(self.strategylist
) > 1 and 'PrecisionFilter' in self.pairlists.name_list:
raise OperationalException(
"PrecisionFilter not allowed for backtesting multiple strategies."
)
self.pairlists.refresh_pairlist()
if len(self.pairlists.whitelist) == 0:
raise OperationalException("No pair in whitelist.")
if config.get('fee', None) is not None:
self.fee = config['fee']
else:
self.fee = self.exchange.get_fee(
symbol=self.pairlists.whitelist[0])
# Get maximum required startup period
self.required_startup = max(
[strat.startup_candle_count for strat in self.strategylist])
# Load one (first) strategy
self._set_strategy(self.strategylist[0])
def _rpc_show_config(config, botstate: Union[State, str]) -> Dict[str, Any]:
"""
Return a dict of config options.
Explicitly does NOT return the full config to avoid leakage of sensitive
information via rpc.
"""
val = {
'dry_run': config['dry_run'],
'stake_currency': config['stake_currency'],
'stake_amount': config['stake_amount'],
'max_open_trades': (config['max_open_trades']
if config['max_open_trades'] != float('inf') else -1),
'minimal_roi': config['minimal_roi'].copy() if 'minimal_roi' in config else {},
'stoploss': config.get('stoploss'),
'trailing_stop': config.get('trailing_stop'),
'trailing_stop_positive': config.get('trailing_stop_positive'),
'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset'),
'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached'),
'use_custom_stoploss': config.get('use_custom_stoploss'),
'bot_name': config.get('bot_name', 'freqtrade'),
'timeframe': config.get('timeframe'),
'timeframe_ms': timeframe_to_msecs(config['timeframe']
) if 'timeframe' in config else '',
'timeframe_min': timeframe_to_minutes(config['timeframe']
) if 'timeframe' in config else '',
'exchange': config['exchange']['name'],
'strategy': config['strategy'],
'forcebuy_enabled': config.get('forcebuy_enable', False),
'ask_strategy': config.get('ask_strategy', {}),
'bid_strategy': config.get('bid_strategy', {}),
'state': str(botstate),
'runmode': config['runmode'].value
}
return val
def get_signal(self, pair: str, timeframe: str, dataframe: DataFrame) -> Tuple[bool, bool]:
"""
Calculates current signal based based on the buy / sell columns of the dataframe.
Used by Bot to get the signal to buy or sell
:param pair: pair in format ANT/BTC
:param timeframe: timeframe to use
:param dataframe: Analyzed dataframe to get signal from.
:return: (Buy, Sell) A bool-tuple indicating buy/sell signal
"""
if not isinstance(dataframe, DataFrame) or dataframe.empty:
logger.warning(f'Empty candle (OHLCV) data for pair {pair}')
return False, False
latest_date = dataframe['date'].max()
latest = dataframe.loc[dataframe['date'] == latest_date].iloc[-1]
# Explicitly convert to arrow object to ensure the below comparison does not fail
latest_date = arrow.get(latest_date)
# Check if dataframe is out of date
timeframe_minutes = timeframe_to_minutes(timeframe)
offset = self.config.get('exchange', {}).get('outdated_offset', 5)
if latest_date < (arrow.utcnow().shift(minutes=-(timeframe_minutes * 2 + offset))):
logger.warning(
'Outdated history for pair %s. Last tick is %s minutes old',
pair, int((arrow.utcnow() - latest_date).total_seconds() // 60)
)
return False, False
(buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1
logger.debug('trigger: %s (pair=%s) buy=%s sell=%s',
latest['date'], pair, str(buy), str(sell))
return buy, sell
def analyze_trade_parallelism(results: pd.DataFrame,
timeframe: str) -> pd.DataFrame:
"""
Find overlapping trades by expanding each trade once per period it was open
and then counting overlaps.
:param results: Results Dataframe - can be loaded
:param timeframe: Timeframe used for backtest
:return: dataframe with open-counts per time-period in timeframe
"""
from freqtrade.exchange import timeframe_to_minutes
timeframe_min = timeframe_to_minutes(timeframe)
dates = [
pd.Series(
pd.date_range(row[1]['open_date'],
row[1]['close_date'],
freq=f"{timeframe_min}min"))
for row in results[['open_date', 'close_date']].iterrows()
]
deltas = [len(x) for x in dates]
dates = pd.Series(pd.concat(dates).values, name='date')
df2 = pd.DataFrame(np.repeat(results.values, deltas, axis=0),
columns=results.columns)
df2 = pd.concat([dates, df2], axis=1)
df2 = df2.set_index('date')
df_final = df2.resample(f"{timeframe_min}min")[['pair']].count()
df_final = df_final.rename({'pair': 'open_trades'}, axis=1)
return df_final
def start_list_data(args: Dict[str, Any]) -> None:
"""
List available backtest data
"""
config = setup_utils_configuration(args, RunMode.UTIL_NO_EXCHANGE)
from tabulate import tabulate
from freqtrade.data.history.idatahandler import get_datahandler
dhc = get_datahandler(config['datadir'], config['dataformat_ohlcv'])
paircombs = dhc.ohlcv_get_available_data(config['datadir'])
if args['pairs']:
paircombs = [comb for comb in paircombs if comb[0] in args['pairs']]
print(f"Found {len(paircombs)} pair / timeframe combinations.")
groupedpair = defaultdict(list)
for pair, timeframe in sorted(paircombs,
key=lambda x:
(x[0], timeframe_to_minutes(x[1]))):
groupedpair[pair].append(timeframe)
if groupedpair:
print(
tabulate([(pair, ', '.join(timeframes))
for pair, timeframes in groupedpair.items()],
headers=("Pair", "Timeframe"),
tablefmt='psql',
stralign='right'))
def trades_to_ohlcv(trades: TradeList, timeframe: str) -> DataFrame:
"""
Converts trades list to OHLCV list
:param trades: List of trades, as returned by ccxt.fetch_trades.
:param timeframe: Timeframe to resample data to
:return: OHLCV Dataframe.
:raises: ValueError if no trades are provided
"""
from freqtrade.exchange import timeframe_to_minutes
timeframe_minutes = timeframe_to_minutes(timeframe)
if not trades:
raise ValueError('Trade-list empty.')
df = pd.DataFrame(trades, columns=DEFAULT_TRADES_COLUMNS)
df['timestamp'] = pd.to_datetime(
df['timestamp'],
unit='ms',
utc=True,
)
df = df.set_index('timestamp')
df_new = df['price'].resample(f'{timeframe_minutes}min').ohlc()
df_new['volume'] = df['amount'].resample(f'{timeframe_minutes}min').sum()
df_new['date'] = df_new.index
# Drop 0 volume rows
df_new = df_new.dropna()
return df_new.loc[:, DEFAULT_DATAFRAME_COLUMNS]
def __init__(self, config):
super(TradingEnv, self).__init__()
self.config = config
self.config['strategy'] = self.config['gym_parameters']['indicator_strategy']
self.strategy = StrategyResolver.load_strategy(self.config)
self.fee = self.config['gym_parameters']['fee']
self.timeframe = str(config.get('ticker_interval'))
self.timeframe_min = timeframe_to_minutes(self.timeframe)
self.required_startup = self.strategy.startup_candle_count
data, timerange = self.load_bt_data()
# need to reprocess data every time to populate signals
preprocessed = self.strategy.ohlcvdata_to_dataframe(data)
del data
# Trim startup period from analyzed dataframe
dfs = []
for pair, df in preprocessed.items():
dfs.append(trim_dataframe(df, timerange))
del preprocessed
self.rest_idx = set()
idx = 0
for d in dfs:
idx += d.shape[0]
self.rest_idx.add(idx)
print(self.rest_idx)
df = pd.concat(dfs, ignore_index=True)
del dfs
# setting
df = df.dropna()
self.pair = pair
self.ticker = self._get_ticker(df)
del df
self.lookback_window_size = 40
# start
logger.info('Using stake_currency: %s ...', self.config['stake_currency'])
logger.info('Data Length: %s ...', len(self.ticker))
self.stake_amount = self.config['stake_amount']
self.reward_decay = 0.0005
self.not_complete_trade_decay = 0.5
self.game_loss = -0.5
self.game_win = 1.0
self.simulate_length = self.config['gym_parameters']['simulate_length']
# Actions
self.action_space = spaces.Discrete(3)
self.observation_space = spaces.Box(
low=np.full(24, -np.inf), high=np.full(24, np.inf), dtype=np.float)
def get_signal(self, pair: str, interval: str,
dataframe: DataFrame) -> Tuple[bool, bool]:
"""
Calculates current signal based several technical analysis indicators
:param pair: pair in format ANT/BTC
:param interval: Interval to use (in min)
:param dataframe: Dataframe to analyze
:return: (Buy, Sell) A bool-tuple indicating buy/sell signal
"""
if not isinstance(dataframe, DataFrame) or dataframe.empty:
logger.warning('Empty ticker history for pair %s', pair)
return False, False
try:
dataframe = self.analyze_ticker(dataframe, {'pair': pair})
except ValueError as error:
logger.warning(
'Unable to analyze ticker for pair %s: %s',
pair,
str(error)
)
return False, False
except Exception as error:
logger.exception(
'Unexpected error when analyzing ticker for pair %s: %s',
pair,
str(error)
)
return False, False
if dataframe.empty:
logger.warning('Empty dataframe for pair %s', pair)
return False, False
latest = dataframe.iloc[-1]
# Check if dataframe is out of date
signal_date = arrow.get(latest['date'])
interval_minutes = timeframe_to_minutes(interval)
offset = self.config.get('exchange', {}).get('outdated_offset', 5)
if signal_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + offset))):
logger.warning(
'Outdated history for pair %s. Last tick is %s minutes old',
pair,
(arrow.utcnow() - signal_date).seconds // 60
)
return False, False
(buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1
logger.debug(
'trigger: %s (pair=%s) buy=%s sell=%s',
latest['date'],
pair,
str(buy),
str(sell)
)
return buy, sell
def get_signal(
self,
pair: str,
timeframe: str,
dataframe: DataFrame
) -> Tuple[bool, bool, Optional[str], Optional[str]]:
"""
Calculates current signal based based on the buy / sell columns of the dataframe.
Used by Bot to get the signal to buy or sell
:param pair: pair in format ANT/BTC
:param timeframe: timeframe to use
:param dataframe: Analyzed dataframe to get signal from.
:return: (Buy, Sell) A bool-tuple indicating buy/sell signal
"""
if not isinstance(dataframe, DataFrame) or dataframe.empty:
logger.warning(f'Empty candle (OHLCV) data for pair {pair}')
return False, False, None, None
latest_date = dataframe['date'].max()
latest = dataframe.loc[dataframe['date'] == latest_date].iloc[-1]
# Explicitly convert to arrow object to ensure the below comparison does not fail
latest_date = arrow.get(latest_date)
# Check if dataframe is out of date
timeframe_minutes = timeframe_to_minutes(timeframe)
offset = self.config.get('exchange', {}).get('outdated_offset', 5)
if latest_date < (arrow.utcnow().shift(minutes=-(timeframe_minutes * 2 + offset))):
logger.warning(
'Outdated history for pair %s. Last tick is %s minutes old',
pair, int((arrow.utcnow() - latest_date).total_seconds() // 60)
)
return False, False, None, None
buy = latest[SignalType.BUY.value] == 1
sell = False
if SignalType.SELL.value in latest:
sell = latest[SignalType.SELL.value] == 1
buy_tag = latest.get(SignalTagType.BUY_TAG.value, None)
exit_tag = latest.get(SignalTagType.EXIT_TAG.value, None)
# Tags can be None, which does not resolve to False.
buy_tag = buy_tag if isinstance(buy_tag, str) else None
exit_tag = exit_tag if isinstance(exit_tag, str) else None
logger.debug('trigger: %s (pair=%s) buy=%s sell=%s',
latest['date'], pair, str(buy), str(sell))
timeframe_seconds = timeframe_to_seconds(timeframe)
if self.ignore_expired_candle(latest_date=latest_date,
current_time=datetime.now(timezone.utc),
timeframe_seconds=timeframe_seconds,
buy=buy):
return False, sell, buy_tag, exit_tag
return buy, sell, buy_tag, exit_tag
def init_backtest_detail(self):
# Load detail timeframe if specified
self.timeframe_detail = str(self.config.get('timeframe_detail', ''))
if self.timeframe_detail:
self.timeframe_detail_min = timeframe_to_minutes(self.timeframe_detail)
if self.timeframe_min <= self.timeframe_detail_min:
raise OperationalException(
"Detail timeframe must be smaller than strategy timeframe.")
else:
self.timeframe_detail_min = 0
self.detail_data: Dict[str, DataFrame] = {}
def get_signal(self, pair: str, interval: str, dataframe: DataFrame) -> Tuple[bool, bool]:
"""
Calculates current signal based several technical analysis indicators
:param pair: pair in format ANT/BTC
:param interval: Interval to use (in min)
:param dataframe: Dataframe to analyze
:return: (Buy, Sell) A bool-tuple indicating buy/sell signal
"""
if not isinstance(dataframe, DataFrame) or dataframe.empty:
logger.warning('Empty candle (OHLCV) data for pair %s', pair)
return False, False
try:
df_len, df_close, df_date = self.preserve_df(dataframe)
dataframe = strategy_safe_wrapper(
self._analyze_ticker_internal, message=""
)(dataframe, {'pair': pair})
self.assert_df(dataframe, df_len, df_close, df_date)
except StrategyError as error:
logger.warning(f"Unable to analyze candle (OHLCV) data for pair {pair}: {error}")
return False, False
if dataframe.empty:
logger.warning('Empty dataframe for pair %s', pair)
return False, False
latest_date = dataframe['date'].max()
latest = dataframe.loc[dataframe['date'] == latest_date].iloc[-1]
# Explicitly convert to arrow object to ensure the below comparison does not fail
latest_date = arrow.get(latest_date)
# Check if dataframe is out of date
interval_minutes = timeframe_to_minutes(interval)
offset = self.config.get('exchange', {}).get('outdated_offset', 5)
if latest_date < (arrow.utcnow().shift(minutes=-(interval_minutes * 2 + offset))):
logger.warning(
'Outdated history for pair %s. Last tick is %s minutes old',
pair,
(arrow.utcnow() - latest_date).seconds // 60
)
return False, False
(buy, sell) = latest[SignalType.BUY.value] == 1, latest[SignalType.SELL.value] == 1
logger.debug(
'trigger: %s (pair=%s) buy=%s sell=%s',
latest['date'],
pair,
str(buy),
str(sell)
)
return buy, sell
def _set_strategy(self, strategy):
"""
Load strategy into backtesting
"""
self.strategy = strategy
self.ticker_interval = self.config.get('ticker_interval')
self.ticker_interval_mins = timeframe_to_minutes(self.ticker_interval)
self.advise_buy = strategy.advise_buy
self.advise_sell = strategy.advise_sell
# Set stoploss_on_exchange to false for backtesting,
# since a "perfect" stoploss-sell is assumed anyway
# And the regular "stoploss" function would not apply to that case
self.strategy.order_types['stoploss_on_exchange'] = False
def test_fetch_ohlcv(self, exchange):
exchange, exchangename = exchange
pair = EXCHANGES[exchangename]['pair']
timeframe = EXCHANGES[exchangename]['timeframe']
pair_tf = (pair, timeframe)
ohlcv = exchange.refresh_latest_ohlcv([pair_tf])
assert isinstance(ohlcv, dict)
assert len(ohlcv[pair_tf]) == len(exchange.klines(pair_tf))
# assert len(exchange.klines(pair_tf)) > 200
# Assume 90% uptime ...
assert len(exchange.klines(pair_tf)) > exchange.ohlcv_candle_limit(timeframe) * 0.90
# Check if last-timeframe is within the last 2 intervals
now = datetime.now(timezone.utc) - timedelta(minutes=(timeframe_to_minutes(timeframe) * 2))
assert exchange.klines(pair_tf).iloc[-1]['date'] >= timeframe_to_prev_date(timeframe, now)
def _rpc_show_config(config, botstate: Union[State, str],
strategy_version: Optional[str] = None) -> Dict[str, Any]:
"""
Return a dict of config options.
Explicitly does NOT return the full config to avoid leakage of sensitive
information via rpc.
"""
val = {
'version': __version__,
'strategy_version': strategy_version,
'dry_run': config['dry_run'],
'trading_mode': config.get('trading_mode', 'spot'),
'short_allowed': config.get('trading_mode', 'spot') != 'spot',
'stake_currency': config['stake_currency'],
'stake_currency_decimals': decimals_per_coin(config['stake_currency']),
'stake_amount': str(config['stake_amount']),
'available_capital': config.get('available_capital'),
'max_open_trades': (config['max_open_trades']
if config['max_open_trades'] != float('inf') else -1),
'minimal_roi': config['minimal_roi'].copy() if 'minimal_roi' in config else {},
'stoploss': config.get('stoploss'),
'trailing_stop': config.get('trailing_stop'),
'trailing_stop_positive': config.get('trailing_stop_positive'),
'trailing_stop_positive_offset': config.get('trailing_stop_positive_offset'),
'trailing_only_offset_is_reached': config.get('trailing_only_offset_is_reached'),
'unfilledtimeout': config.get('unfilledtimeout'),
'use_custom_stoploss': config.get('use_custom_stoploss'),
'order_types': config.get('order_types'),
'bot_name': config.get('bot_name', 'freqtrade'),
'timeframe': config.get('timeframe'),
'timeframe_ms': timeframe_to_msecs(config['timeframe']
) if 'timeframe' in config else 0,
'timeframe_min': timeframe_to_minutes(config['timeframe']
) if 'timeframe' in config else 0,
'exchange': config['exchange']['name'],
'strategy': config['strategy'],
'force_entry_enable': config.get('force_entry_enable', False),
'exit_pricing': config.get('exit_pricing', {}),
'entry_pricing': config.get('entry_pricing', {}),
'state': str(botstate),
'runmode': config['runmode'].value,
'position_adjustment_enable': config.get('position_adjustment_enable', False),
'max_entry_position_adjustment': (
config.get('max_entry_position_adjustment', -1)
if config.get('max_entry_position_adjustment') != float('inf')
else -1)
}
return val
def populate_indicators(self, dataframe: DataFrame, metadata: dict) -> DataFrame:
dataframe['maShort'] = ta.EMA(dataframe, timeperiod=8)
dataframe['maMedium'] = ta.EMA(dataframe, timeperiod=21)
##################################################################################
# required for graphing
bollinger = qtpylib.bollinger_bands(dataframe['close'], window=20, stds=2)
dataframe['bb_lowerband'] = bollinger['lower']
dataframe['bb_upperband'] = bollinger['upper']
dataframe['bb_middleband'] = bollinger['mid']
self.resample_interval = timeframe_to_minutes(self.ticker_interval) * 12
dataframe_long = resample_to_interval(dataframe, self.resample_interval)
dataframe_long['sma'] = ta.SMA(dataframe_long, timeperiod=50, price='close')
dataframe = resampled_merge(dataframe, dataframe_long, fill_na=True)
return dataframe
def test_validate_backtest_data(default_conf, mocker, caplog) -> None:
patch_exchange(mocker)
strategy = DefaultStrategy(default_conf)
timerange = TimeRange('index', 'index', 200, 250)
data = strategy.tickerdata_to_dataframe(
history.load_data(datadir=None,
ticker_interval='5m',
pairs=['UNITTEST/BTC'],
timerange=timerange))
min_date, max_date = history.get_timeframe(data)
caplog.clear()
assert not history.validate_backtest_data(
data['UNITTEST/BTC'], 'UNITTEST/BTC', min_date, max_date,
timeframe_to_minutes('5m'))
assert len(caplog.record_tuples) == 0
def __init__(self, config: Dict[str, Any]) -> None:
self.config = config
# Reset keys for backtesting
remove_credentials(self.config)
self.strategylist: List[IStrategy] = []
self.exchange = ExchangeResolver.load_exchange(
self.config['exchange']['name'], self.config)
if config.get('fee'):
self.fee = config['fee']
else:
self.fee = self.exchange.get_fee(
symbol=self.config['exchange']['pair_whitelist'][0])
if self.config.get('runmode') != RunMode.HYPEROPT:
self.dataprovider = DataProvider(self.config, self.exchange)
IStrategy.dp = self.dataprovider
if self.config.get('strategy_list', None):
for strat in list(self.config['strategy_list']):
stratconf = deepcopy(self.config)
stratconf['strategy'] = strat
self.strategylist.append(
StrategyResolver.load_strategy(stratconf))
validate_config_consistency(stratconf)
else:
# No strategy list specified, only one strategy
self.strategylist.append(
StrategyResolver.load_strategy(self.config))
validate_config_consistency(self.config)
if "ticker_interval" not in self.config:
raise OperationalException(
"Ticker-interval needs to be set in either configuration "
"or as cli argument `--ticker-interval 5m`")
self.timeframe = str(self.config.get('ticker_interval'))
self.timeframe_min = timeframe_to_minutes(self.timeframe)
# Get maximum required startup period
self.required_startup = max(
[strat.startup_candle_count for strat in self.strategylist])
# Load one (first) strategy
self._set_strategy(self.strategylist[0])
def ohlcv_fill_up_missing_data(dataframe: DataFrame, timeframe: str,
pair: str) -> DataFrame:
"""
Fills up missing data with 0 volume rows,
using the previous close as price for "open", "high" "low" and "close", volume is set to 0
"""
from freqtrade.exchange import timeframe_to_minutes
ohlcv_dict = {
'open': 'first',
'high': 'max',
'low': 'min',
'close': 'last',
'volume': 'sum'
}
timeframe_minutes = timeframe_to_minutes(timeframe)
# Resample to create "NAN" values
df = dataframe.resample(f'{timeframe_minutes}min',
on='date').agg(ohlcv_dict)
# Forwardfill close for missing columns
df['close'] = df['close'].fillna(method='ffill')
# Use close for "open, high, low"
df.loc[:, ['open', 'high', 'low']] = df[['open', 'high',
'low']].fillna(value={
'open': df['close'],
'high': df['close'],
'low': df['close'],
})
df.reset_index(inplace=True)
len_before = len(dataframe)
len_after = len(df)
pct_missing = (len_after -
len_before) / len_before if len_before > 0 else 0
if len_before != len_after:
message = (
f"Missing data fillup for {pair}: before: {len_before} - after: {len_after}"
f" - {round(pct_missing * 100, 2)}%")
if pct_missing > 0.01:
logger.info(message)
else:
# Don't be verbose if only a small amount is missing
logger.debug(message)
return df
def populate_indicators(self, dataframe: DataFrame,
metadata: dict) -> DataFrame:
if self.config['runmode'].value in ('backtest', 'hyperopt'):
assert (timeframe_to_minutes(self.timeframe) <=
5), "Backtest this strategy in a timeframe of 5m or less."
assert self.dp, "DataProvider is required for multiple timeframes."
informative = self.get_informative_indicators(dataframe, metadata)
dataframe = merge_informative_pair(dataframe,
informative,
self.timeframe,
self.informative_timeframe,
ffill=True)
dataframe = self.get_main_indicators(dataframe, metadata)
return dataframe
def test_validate_backtest_data_warn(default_conf, mocker, caplog) -> None:
patch_exchange(mocker)
strategy = DefaultStrategy(default_conf)
data = strategy.tickerdata_to_dataframe(
history.load_data(datadir=None,
ticker_interval='1m',
pairs=['UNITTEST/BTC'],
fill_up_missing=False))
min_date, max_date = history.get_timeframe(data)
caplog.clear()
assert history.validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC',
min_date, max_date,
timeframe_to_minutes('1m'))
assert len(caplog.record_tuples) == 1
assert log_has(
"UNITTEST/BTC has missing frames: expected 14396, got 13680, that's 716 missing values",
caplog.record_tuples)
def trades_to_ohlcv(trades: list, timeframe: str) -> list:
"""
Converts trades list to ohlcv list
:param trades: List of trades, as returned by ccxt.fetch_trades.
:param timeframe: Ticker timeframe to resample data to
:return: ohlcv timeframe as list (as returned by ccxt.fetch_ohlcv)
"""
from freqtrade.exchange import timeframe_to_minutes
ticker_minutes = timeframe_to_minutes(timeframe)
df = pd.DataFrame(trades)
df['datetime'] = pd.to_datetime(df['datetime'])
df = df.set_index('datetime')
df_new = df['price'].resample(f'{ticker_minutes}min').ohlc()
df_new['volume'] = df['amount'].resample(f'{ticker_minutes}min').sum()
df_new['date'] = df_new.index.astype("int64") // 10 ** 6
# Drop 0 volume rows
df_new = df_new.dropna()
columns = ["date", "open", "high", "low", "close", "volume"]
return list(zip(*[df_new[x].values.tolist() for x in columns]))
def test_validate_backtest_data(default_conf, mocker, caplog,
testdatadir) -> None:
patch_exchange(mocker)
default_conf.update({'strategy': 'DefaultStrategy'})
strategy = StrategyResolver.load_strategy(default_conf)
timerange = TimeRange('index', 'index', 200, 250)
data = strategy.ohlcvdata_to_dataframe(
load_data(datadir=testdatadir,
timeframe='5m',
pairs=['UNITTEST/BTC'],
timerange=timerange))
min_date, max_date = get_timerange(data)
caplog.clear()
assert not validate_backtest_data(data['UNITTEST/BTC'], 'UNITTEST/BTC',
min_date, max_date,
timeframe_to_minutes('5m'))
assert len(caplog.record_tuples) == 0
