def test_deep_merge_dicts():
a = {'first': {'rows': {'pass': '******', 'number': '1', 'test': None}}}
b = {'first': {'rows': {'fail': 'cat', 'number': '5', 'test': 'asdf'}}}
res = {
'first': {
'rows': {
'pass': '******',
'fail': 'cat',
'number': '5',
'test': 'asdf'
}
}
}
res2 = {
'first': {
'rows': {
'pass': '******',
'fail': 'cat',
'number': '1',
'test': None
}
}
}
assert deep_merge_dicts(b, deepcopy(a)) == res
assert deep_merge_dicts(a, deepcopy(b)) == res2
res2['first']['rows']['test'] = 'asdf'
assert deep_merge_dicts(a, deepcopy(b), allow_null_overrides=False) == res2
def load_from_files(self, files: List[str]) -> Dict[str, Any]:
# Keep this method as staticmethod, so it can be used from interactive environments
config: Dict[str, Any] = {}
if not files:
return deepcopy(constants.MINIMAL_CONFIG)
# We expect here a list of config filenames
for path in files:
logger.info(f'Using config: {path} ...')
# Merge config options, overwriting old values
config = deep_merge_dicts(load_config_file(path), config)
# Load environment variables
env_data = enironment_vars_to_dict()
config = deep_merge_dicts(env_data, config)
config['config_files'] = files
# Normalize config
if 'internals' not in config:
config['internals'] = {}
if 'ask_strategy' not in config:
config['ask_strategy'] = {}
if 'pairlists' not in config:
config['pairlists'] = []
return config
def _load_hyper_params(self, hyperopt: bool = False) -> None:
"""
Load Hyperoptable parameters
"""
params = self.load_params_from_file()
params = params.get('params', {})
self._ft_params_from_file = params
buy_params = deep_merge_dicts(params.get('buy', {}),
getattr(self, 'buy_params', None))
sell_params = deep_merge_dicts(params.get('sell', {}),
getattr(self, 'sell_params', None))
self._load_params(buy_params, 'buy', hyperopt)
self._load_params(sell_params, 'sell', hyperopt)
def load_from_files(self, files: List[str]) -> Dict[str, Any]:
# Keep this method as staticmethod, so it can be used from interactive environments
config: Dict[str, Any] = {}
if not files:
return deepcopy(constants.MINIMAL_CONFIG)
# We expect here a list of config filenames
for path in files:
logger.info(f'Using config: {path} ...')
# Merge config options, overwriting old values
config = deep_merge_dicts(load_config_file(path), config)
# Normalize config
if 'internals' not in config:
config['internals'] = {}
# TODO: This can be deleted along with removal of deprecated
# experimental settings
if 'ask_strategy' not in config:
config['ask_strategy'] = {}
if 'pairlists' not in config:
config['pairlists'] = []
return config
def from_files(files: List[str]) -> Dict[str, Any]:
"""
Iterate through the config files passed in, loading all of them
and merging their contents.
Files are loaded in sequence, parameters in later configuration files
override the same parameter from an earlier file (last definition wins).
:param files: List of file paths
:return: configuration dictionary
"""
# Keep this method as staticmethod, so it can be used from interactive environments
config: Dict[str, Any] = {}
if not files:
return constants.MINIMAL_CONFIG.copy()
# We expect here a list of config filenames
for path in files:
logger.info(f'Using config: {path} ...')
# Merge config options, overwriting old values
config = deep_merge_dicts(load_config_file(path), config)
# Normalize config
if 'internals' not in config:
config['internals'] = {}
# validate configuration before returning
logger.info('Validating configuration ...')
validate_config_schema(config)
return config
def get_tickers(self,
symbols: List[str] = None,
cached: bool = False) -> Dict:
tickers = super().get_tickers(symbols=symbols, cached=cached)
if self.trading_mode == TradingMode.FUTURES:
# Binance's future result has no bid/ask values.
# Therefore we must fetch that from fetch_bids_asks and combine the two results.
bidsasks = self.fetch_bids_asks(symbols, cached)
tickers = deep_merge_dicts(bidsasks,
tickers,
allow_null_overrides=False)
return tickers
def load_from_files(files: List[str],
base_path: Path = None,
level: int = 0) -> Dict[str, Any]:
"""
Recursively load configuration files if specified.
Sub-files are assumed to be relative to the initial config.
"""
config: Dict[str, Any] = {}
if level > 5:
raise OperationalException("Config loop detected.")
if not files:
return deepcopy(MINIMAL_CONFIG)
files_loaded = []
# We expect here a list of config filenames
for filename in files:
logger.info(f'Using config: {filename} ...')
if filename == '-':
# Immediately load stdin and return
return load_config_file(filename)
file = Path(filename)
if base_path:
# Prepend basepath to allow for relative assignments
file = base_path / file
config_tmp = load_config_file(str(file))
if 'add_config_files' in config_tmp:
config_sub = load_from_files(config_tmp['add_config_files'],
file.resolve().parent, level + 1)
files_loaded.extend(config_sub.get('config_files', []))
config_tmp = deep_merge_dicts(config_tmp, config_sub)
files_loaded.insert(0, str(file))
# Merge config options, overwriting prior values
config = deep_merge_dicts(config_tmp, config)
config['config_files'] = files_loaded
return config
def _load_config_files(self) -> Dict[str, Any]:
"""
Iterate through the config files passed in the args,
loading all of them and merging their contents.
"""
config: Dict[str, Any] = {}
# We expect here a list of config filenames
for path in self.args.config:
logger.info('Using config: %s ...', path)
# Merge config options, overwriting old values
config = deep_merge_dicts(self._load_config_file(path), config)
return config
def load_config(self) -> Dict[str, Any]:
"""
Extract information for sys.argv and load the bot configuration
:return: Configuration dictionary
"""
# Load all configs
config: Dict[str, Any] = load_from_files(self.args.get("config", []))
# Load environment variables
env_data = enironment_vars_to_dict()
config = deep_merge_dicts(env_data, config)
# Normalize config
if 'internals' not in config:
config['internals'] = {}
if 'pairlists' not in config:
config['pairlists'] = []
# Keep a copy of the original configuration file
config['original_config'] = deepcopy(config)
self._process_logging_options(config)
self._process_runmode(config)
self._process_common_options(config)
self._process_trading_options(config)
self._process_optimize_options(config)
self._process_plot_options(config)
self._process_data_options(config)
# Check if the exchange set by the user is supported
check_exchange(
config,
config.get('experimental', {}).get('block_bad_exchanges', True))
self._resolve_pairs_list(config)
process_temporary_deprecated_settings(config)
return config
def export_params(params, strategy_name: str, filename: Path):
"""
Generate files
"""
final_params = deepcopy(params['params_not_optimized'])
final_params = deep_merge_dicts(params['params_details'], final_params)
final_params = {
'strategy_name': strategy_name,
'params': final_params,
'ft_stratparam_v': 1,
'export_time': datetime.now(timezone.utc),
}
logger.info(f"Dumping parameters to {filename}")
rapidjson.dump(final_params, filename.open('w'), indent=2,
default=hyperopt_serializer,
number_mode=rapidjson.NM_NATIVE | rapidjson.NM_NAN
)
def load_config(self) -> Dict[str, Any]:
"""
Extract information for sys.argv and load the bot configuration
:return: Configuration dictionary
"""
config: Dict[str, Any] = {}
# Now expecting a list of config filenames here, not a string
for path in self.args.config:
logger.info('Using config: %s ...', path)
# Merge config options, overwriting old values
config = deep_merge_dicts(self._load_config_file(path), config)
if 'internals' not in config:
config['internals'] = {}
logger.info('Validating configuration ...')
self._validate_config_schema(config)
self._validate_config_consistency(config)
# Set strategy if not specified in config and or if it's non default
if self.args.strategy != constants.DEFAULT_STRATEGY or not config.get('strategy'):
config.update({'strategy': self.args.strategy})
if self.args.strategy_path:
config.update({'strategy_path': self.args.strategy_path})
# Load Common configuration
config = self._load_common_config(config)
# Load Backtesting
config = self._load_backtesting_config(config)
# Load Edge
config = self._load_edge_config(config)
# Load Hyperopt
config = self._load_hyperopt_config(config)
# Set runmode
if not self.runmode:
# Handle real mode, infer dry/live from config
self.runmode = RunMode.DRY_RUN if config.get('dry_run', True) else RunMode.LIVE
config.update({'runmode': self.runmode})
return config
def _get_results_dict(self, backtesting_results, min_date, max_date,
params_dict,
processed: Dict[str, DataFrame]) -> Dict[str, Any]:
params_details = self._get_params_details(params_dict)
strat_stats = generate_strategy_stats(
processed,
self.backtesting.strategy.get_strategy_name(),
backtesting_results,
min_date,
max_date,
market_change=0)
results_explanation = HyperoptTools.format_results_explanation_string(
strat_stats, self.config['stake_currency'])
not_optimized = self.backtesting.strategy.get_no_optimize_params()
not_optimized = deep_merge_dicts(not_optimized,
self._get_no_optimize_details())
trade_count = strat_stats['total_trades']
total_profit = strat_stats['profit_total']
# If this evaluation contains too short amount of trades to be
# interesting -- consider it as 'bad' (assigned max. loss value)
# in order to cast this hyperspace point away from optimization
# path. We do not want to optimize 'hodl' strategies.
loss: float = MAX_LOSS
if trade_count >= self.config['hyperopt_min_trades']:
loss = self.calculate_loss(results=backtesting_results['results'],
trade_count=trade_count,
min_date=min_date,
max_date=max_date,
config=self.config,
processed=processed,
backtest_stats=strat_stats)
return {
'loss': loss,
'params_dict': params_dict,
'params_details': params_details,
'params_not_optimized': not_optimized,
'results_metrics': strat_stats,
'results_explanation': results_explanation,
'total_profit': total_profit,
}
def flat_vars_to_nested_dict(env_dict: Dict[str, Any],
prefix: str) -> Dict[str, Any]:
"""
Environment variables must be prefixed with FREQTRADE.
FREQTRADE__{section}__{key}
:param env_dict: Dictionary to validate - usually os.environ
:param prefix: Prefix to consider (usually FREQTRADE__)
:return: Nested dict based on available and relevant variables.
"""
relevant_vars: Dict[str, Any] = {}
for env_var, val in sorted(env_dict.items()):
if env_var.startswith(prefix):
logger.info(f"Loading variable '{env_var}'")
key = env_var.replace(prefix, '')
for k in reversed(key.split('__')):
val = {
k.lower(): get_var_typed(val) if type(val) != dict else val
}
relevant_vars = deep_merge_dicts(val, relevant_vars)
return relevant_vars
def load_from_files(self, files: List[str]) -> Dict[str, Any]:
# Keep this method as staticmethod, so it can be used from interactive environments
config: Dict[str, Any] = {}
if not files:
return deepcopy(constants.MINIMAL_CONFIG)
# We expect here a list of config filenames
for path in files:
logger.info(f'Using config: {path} ...')
# Merge config options, overwriting old values
config = deep_merge_dicts(load_config_file(path), config)
# Normalize config
if 'internals' not in config:
config['internals'] = {}
# validate configuration before returning
logger.info('Validating configuration ...')
validate_config_schema(config)
return config
def __init__(self, config: dict, validate: bool = True) -> None:
"""
Initializes this module with the given config,
it does basic validation whether the specified exchange and pairs are valid.
:return: None
"""
self._api: ccxt.Exchange = None
self._api_async: ccxt_async.Exchange = None
self._config.update(config)
self._cached_ticker: Dict[str, Any] = {}
# Holds last candle refreshed time of each pair
self._pairs_last_refresh_time: Dict[Tuple[str, str], int] = {}
# Timestamp of last markets refresh
self._last_markets_refresh: int = 0
# Holds candles
self._klines: Dict[Tuple[str, str], DataFrame] = {}
# Holds all open sell orders for dry_run
self._dry_run_open_orders: Dict[str, Any] = {}
if config['dry_run']:
logger.info('Instance is running with dry_run enabled')
exchange_config = config['exchange']
# Deep merge ft_has with default ft_has options
self._ft_has = deep_merge_dicts(self._ft_has,
deepcopy(self._ft_has_default))
if exchange_config.get("_ft_has_params"):
self._ft_has = deep_merge_dicts(
exchange_config.get("_ft_has_params"), self._ft_has)
logger.info(
"Overriding exchange._ft_has with config params, result: %s",
self._ft_has)
# Assign this directly for easy access
self._ohlcv_candle_limit = self._ft_has['ohlcv_candle_limit']
self._ohlcv_partial_candle = self._ft_has['ohlcv_partial_candle']
self._trades_pagination = self._ft_has['trades_pagination']
self._trades_pagination_arg = self._ft_has['trades_pagination_arg']
# Initialize ccxt objects
ccxt_config = self._ccxt_config.copy()
ccxt_config = deep_merge_dicts(exchange_config.get('ccxt_config', {}),
ccxt_config)
self._api = self._init_ccxt(exchange_config, ccxt_kwargs=ccxt_config)
ccxt_async_config = self._ccxt_config.copy()
ccxt_async_config = deep_merge_dicts(
exchange_config.get('ccxt_async_config', {}), ccxt_async_config)
self._api_async = self._init_ccxt(exchange_config,
ccxt_async,
ccxt_kwargs=ccxt_async_config)
logger.info('Using Exchange "%s"', self.name)
if validate:
# Check if timeframe is available
self.validate_timeframes(config.get('ticker_interval'))
# Initial markets load
self._load_markets()
# Check if all pairs are available
self.validate_stakecurrency(config['stake_currency'])
self.validate_pairs(config['exchange']['pair_whitelist'])
self.validate_ordertypes(config.get('order_types', {}))
self.validate_order_time_in_force(
config.get('order_time_in_force', {}))
self.validate_required_startup_candles(
config.get('startup_candle_count', 0))
# Converts the interval provided in minutes in config to seconds
self.markets_refresh_interval: int = exchange_config.get(
"markets_refresh_interval", 60) * 60
args = arguments.parse_args(no_default_config=True)
# Use bittrex as default exchange
exchange_name = args.exchange or 'bittrex'
pairs: List = []
configuration = Configuration(args)
config: Dict[str, Any] = {}
if args.config:
# Now expecting a list of config filenames here, not a string
for path in args.config:
logger.info(f"Using config: {path}...")
# Merge config options, overwriting old values
config = deep_merge_dicts(configuration._load_config_file(path),
config)
config['stake_currency'] = ''
# Ensure we do not use Exchange credentials
config['exchange']['dry_run'] = True
config['exchange']['key'] = ''
config['exchange']['secret'] = ''
pairs = config['exchange']['pair_whitelist']
if config.get('ticker_interval'):
timeframes = args.timeframes or [config.get('ticker_interval')]
else:
timeframes = args.timeframes or ['1m', '5m']
else: