def download_symbol(cls,
symbol: tp.Label,
start: tp.DatetimeLike = 0,
end: tp.DatetimeLike = 'now',
freq: tp.Union[None, str, pd.DateOffset] = None,
date_range_kwargs: tp.KwargsLike = None,
**kwargs) -> tp.SeriesFrame:
"""Download the symbol.
Generates datetime index and passes it to `SyntheticData.generate_symbol` to fill
the Series/DataFrame with generated data."""
if date_range_kwargs is None:
date_range_kwargs = {}
index = pd.date_range(start=to_tzaware_datetime(start,
tz=get_utc_tz()),
end=to_tzaware_datetime(end, tz=get_utc_tz()),
freq=freq,
**date_range_kwargs)
if len(index) == 0:
raise ValueError("Date range is empty")
return cls.generate_symbol(symbol, index, **kwargs)
drop_duplicates=True,
keep='last',
drop_redundant=True,
ignore_default=True
),
array_wrapper=dict(
column_only_select=False,
group_select=True,
freq=None
),
datetime=dict(
naive_tz=get_local_tz(),
to_py_timezone=True
),
data=dict(
tz_localize=get_utc_tz(),
tz_convert=get_utc_tz(),
missing_index='nan',
missing_columns='raise',
binance=Config( # flex
dict(
api_key=None,
api_secret=None
)
),
ccxt=Config( # flex
dict(
enableRateLimit=True
)
)
),
"""
# Datetime
datetime = Config(dict(naive_tz=get_local_tz(), to_py_timezone=True),
frozen=True)
"""_"""
__pdoc__['datetime'] = f"""Parameters for datetime.
```plaintext
{json.dumps(datetime, indent=2, default=str)}
```
"""
# Data
data = Config(dict(tz_localize=get_utc_tz(),
tz_convert=get_utc_tz(),
missing_index='nan',
missing_columns='raise',
binance=Config(dict(api_key=None, api_secret=None)),
ccxt=Config(enableRateLimit=True)),
frozen=True)
"""_"""
__pdoc__['data'] = f"""Parameters for data.
```plaintext
{json.dumps(data, indent=2, default=str)}
```
For `data['binance']`, see `binance.client.Client`.
def _ts_to_str(ts):
return str(pd.Timestamp(to_tzaware_datetime(ts, tz=get_utc_tz())))
def download_symbol(cls,
symbol: str,
exchange: tp.Union[str, "ExchangeT"] = 'binance',
config: tp.Optional[dict] = None,
timeframe: str = '1d',
start: tp.DatetimeLike = 0,
end: tp.DatetimeLike = 'now UTC',
delay: tp.Optional[float] = None,
limit: tp.Optional[int] = 500,
retries: int = 3,
show_progress: bool = True,
params: tp.Optional[dict] = None) -> tp.Frame:
"""Download the symbol.
Args:
symbol (str): Symbol.
exchange (str or object): Exchange identifier or an exchange object of type
`ccxt.base.exchange.Exchange`.
config (dict): Config passed to the exchange upon instantiation.
Will raise an exception if exchange has been already instantiated.
timeframe (str): Timeframe supported by the exchange.
start (any): Start datetime.
See `vectorbt.utils.datetime.to_tzaware_datetime`.
end (any): End datetime.
See `vectorbt.utils.datetime.to_tzaware_datetime`.
delay (float): Time to sleep after each request (in milliseconds).
!!! note
Use only if `enableRateLimit` is not set.
limit (int): The maximum number of returned items.
retries (int): The number of retries on failure to fetch data.
show_progress (bool): Whether to show the progress bar.
params (dict): Exchange-specific key-value parameters.
For defaults, see `data.ccxt` in `vectorbt._settings.settings`.
"""
import ccxt
from vectorbt._settings import settings
ccxt_cfg = settings['data']['ccxt']
if config is None:
config = {}
if params is None:
params = {}
if isinstance(exchange, str):
if not hasattr(ccxt, exchange):
raise ValueError(f"Exchange {exchange} not found")
# Resolve config
default_config = {}
for k, v in ccxt_cfg.items():
# Get general (not per exchange) settings
if k in ccxt.exchanges:
continue
default_config[k] = v
if exchange in ccxt_cfg:
default_config = merge_dicts(default_config,
ccxt_cfg[exchange])
config = merge_dicts(default_config, config)
exchange = getattr(ccxt, exchange)(config)
else:
if len(config) > 0:
raise ValueError(
"Cannot apply config after instantiation of the exchange")
if not exchange.has['fetchOHLCV']:
raise ValueError(f"Exchange {exchange} does not support OHLCV")
if timeframe not in exchange.timeframes:
raise ValueError(
f"Exchange {exchange} does not support {timeframe} timeframe")
if exchange.has['fetchOHLCV'] == 'emulated':
warnings.warn("Using emulated OHLCV candles", stacklevel=2)
def _retry(method):
@wraps(method)
def retry_method(*args, **kwargs):
for i in range(retries):
try:
return method(*args, **kwargs)
except (ccxt.NetworkError, ccxt.ExchangeError) as e:
if i == retries - 1:
raise e
if delay is not None:
time.sleep(delay / 1000)
return retry_method
@_retry
def _fetch(_since, _limit):
return exchange.fetch_ohlcv(symbol,
timeframe=timeframe,
since=_since,
limit=_limit,
params=params)
# Establish the timestamps
start_ts = datetime_to_ms(to_tzaware_datetime(start, tz=get_utc_tz()))
try:
first_data = _fetch(0, 1)
first_valid_ts = first_data[0][0]
next_start_ts = start_ts = max(start_ts, first_valid_ts)
except:
next_start_ts = start_ts
end_ts = datetime_to_ms(to_tzaware_datetime(end, tz=get_utc_tz()))
def _ts_to_str(ts):
return str(pd.Timestamp(to_tzaware_datetime(ts, tz=get_utc_tz())))
# Iteratively collect the data
data: tp.List[list] = []
with tqdm(disable=not show_progress) as pbar:
pbar.set_description(_ts_to_str(start_ts))
while True:
# Fetch the klines for the next interval
next_data = _fetch(next_start_ts, limit)
if len(data) > 0:
next_data = list(
filter(lambda d: next_start_ts < d[0] < end_ts,
next_data))
else:
next_data = list(filter(lambda d: d[0] < end_ts,
next_data))
# Update the timestamps and the progress bar
if not len(next_data):
break
data += next_data
pbar.set_description("{} - {}".format(
_ts_to_str(start_ts), _ts_to_str(next_data[-1][0])))
pbar.update(1)
next_start_ts = next_data[-1][0]
if delay is not None:
time.sleep(delay / 1000) # be kind to api
# Convert data to a DataFrame
df = pd.DataFrame(
data,
columns=['Open time', 'Open', 'High', 'Low', 'Close', 'Volume'])
df.index = pd.to_datetime(df['Open time'], unit='ms', utc=True)
del df['Open time']
df['Open'] = df['Open'].astype(float)
df['High'] = df['High'].astype(float)
df['Low'] = df['Low'].astype(float)
df['Close'] = df['Close'].astype(float)
df['Volume'] = df['Volume'].astype(float)
return df
def _ts_to_str(ts: tp.DatetimeLike) -> str:
return str(pd.Timestamp(to_tzaware_datetime(ts, tz=get_utc_tz())))
def download_symbol(cls,
symbol: str,
client: tp.Optional["ClientT"] = None,
interval: str = '1d',
start: tp.DatetimeLike = 0,
end: tp.DatetimeLike = 'now UTC',
delay: tp.Optional[float] = 500,
limit: int = 500,
show_progress: bool = True) -> tp.Frame:
"""Download the symbol.
Args:
symbol (str): Symbol.
client (binance.client.Client): Binance client of type `binance.client.Client`.
interval (str): Kline interval.
See `binance.enums`.
start (any): Start datetime.
See `vectorbt.utils.datetime.to_tzaware_datetime`.
end (any): End datetime.
See `vectorbt.utils.datetime.to_tzaware_datetime`.
delay (float): Time to sleep after each request (in milliseconds).
limit (int): The maximum number of returned items.
show_progress (bool): Whether to show the progress bar.
For defaults, see `data.binance` in `vectorbt._settings.settings`.
"""
if client is None:
raise ValueError("client must be provided")
# Establish the timestamps
start_ts = datetime_to_ms(to_tzaware_datetime(start, tz=get_utc_tz()))
try:
first_data = client.get_klines(symbol=symbol,
interval=interval,
limit=1,
startTime=0,
endTime=None)
first_valid_ts = first_data[0][0]
next_start_ts = start_ts = max(start_ts, first_valid_ts)
except:
next_start_ts = start_ts
end_ts = datetime_to_ms(to_tzaware_datetime(end, tz=get_utc_tz()))
def _ts_to_str(ts: tp.DatetimeLike) -> str:
return str(pd.Timestamp(to_tzaware_datetime(ts, tz=get_utc_tz())))
# Iteratively collect the data
data: tp.List[list] = []
with tqdm(disable=not show_progress) as pbar:
pbar.set_description(_ts_to_str(start_ts))
while True:
# Fetch the klines for the next interval
next_data = client.get_klines(symbol=symbol,
interval=interval,
limit=limit,
startTime=next_start_ts,
endTime=end_ts)
if len(data) > 0:
next_data = list(
filter(lambda d: next_start_ts < d[0] < end_ts,
next_data))
else:
next_data = list(filter(lambda d: d[0] < end_ts,
next_data))
# Update the timestamps and the progress bar
if not len(next_data):
break
data += next_data
pbar.set_description("{} - {}".format(
_ts_to_str(start_ts), _ts_to_str(next_data[-1][0])))
pbar.update(1)
next_start_ts = next_data[-1][0]
if delay is not None:
time.sleep(delay / 1000) # be kind to api
# Convert data to a DataFrame
df = pd.DataFrame(data,
columns=[
'Open time', 'Open', 'High', 'Low', 'Close',
'Volume', 'Close time', 'Quote volume',
'Number of trades', 'Taker base volume',
'Taker quote volume', 'Ignore'
])
df.index = pd.to_datetime(df['Open time'], unit='ms', utc=True)
del df['Open time']
df['Open'] = df['Open'].astype(float)
df['High'] = df['High'].astype(float)
df['Low'] = df['Low'].astype(float)
df['Close'] = df['Close'].astype(float)
df['Volume'] = df['Volume'].astype(float)
df['Close time'] = pd.to_datetime(df['Close time'],
unit='ms',
utc=True)
df['Quote volume'] = df['Quote volume'].astype(float)
df['Number of trades'] = df['Number of trades'].astype(int)
df['Taker base volume'] = df['Taker base volume'].astype(float)
df['Taker quote volume'] = df['Taker quote volume'].astype(float)
del df['Ignore']
return df
