def orders() -> List[Union[List[str], List[Union[CharField, str, FloatField]]]]:
array = []
# headers
array.append(['symbol', 'side', 'type', 'qty', 'price', 'flag', 'status', 'created_at'])
route_orders = []
for r in router.routes:
r_orders = store.orders.get_orders(r.exchange, r.symbol)
for o in r_orders:
route_orders.append(o)
if not len(route_orders):
return None
route_orders.sort(key=lambda x: x.created_at, reverse=False)
for o in route_orders[::-1][0:5]:
array.append([
o.symbol if o.is_active else jh.color(o.symbol, 'gray'),
jh.color(o.side, 'red') if o.side == 'sell' else jh.color(o.side, 'green'),
o.type if o.is_active else jh.color(o.type, 'gray'),
o.qty if o.is_active else jh.color(str(o.qty), 'gray'),
o.price if o.is_active else jh.color(str(o.price), 'gray'),
o.flag if o.is_active else jh.color(o.flag, 'gray'),
o.status if o.is_active else jh.color(o.status, 'gray'),
jh.timestamp_to_time(o.created_at)[:19] if o.is_active else jh.color(
jh.timestamp_to_time(o.created_at)[:19], 'gray'),
])
return array
def fetch(self, symbol: str, start_timestamp: int):
"""
note1: unlike Bitfinex, Binance does NOT skip candles with volume=0.
note2: like Bitfinex, start_time includes the candle and so does the end_time.
"""
end_timestamp = start_timestamp + (self.count - 1) * 60000
payload = {
'granularity': '60',
'start': jh.timestamp_to_time(start_timestamp),
'end': jh.timestamp_to_time(end_timestamp),
}
response = requests.get(self.endpoint + '/{}/candles'.format(symbol),
params=payload)
self._handle_errors(response)
data = response.json()
candles = []
for d in data:
candles.append({
'id': jh.generate_unique_id(),
'symbol': symbol,
'exchange': self.name,
'timestamp': int(d[0]) * 1000,
'open': float(d[3]),
'close': float(d[4]),
'high': float(d[2]),
'low': float(d[1]),
'volume': float(d[5])
})
return candles
def __init__(self, training_candles, testing_candles, optimal_total, cpu_cores):
if len(router.routes) != 1:
raise NotImplementedError('optimize_mode mode only supports one route at the moment')
self.strategy_name = router.routes[0].strategy_name
self.optimal_total = optimal_total
self.exchange = router.routes[0].exchange
self.symbol = router.routes[0].symbol
self.timeframe = router.routes[0].timeframe
StrategyClass = jh.get_strategy_class(self.strategy_name)
self.strategy_hp = StrategyClass.hyperparameters(None)
solution_len = len(self.strategy_hp)
if solution_len == 0:
raise exceptions.InvalidStrategy('Targeted strategy does not implement a valid hyperparameters() method.')
super().__init__(
iterations=2000 * solution_len,
population_size=solution_len * 100,
solution_len=solution_len,
options={
'strategy_name': self.strategy_name,
'exchange': self.exchange,
'symbol': self.symbol,
'timeframe': self.timeframe
}
)
if cpu_cores > cpu_count():
raise ValueError('Entered cpu cores number is more than available on this machine which is {}'.format(
cpu_count()
))
elif cpu_cores == 0:
self.cpu_cores = cpu_count()
else:
self.cpu_cores = cpu_cores
self.training_candles = training_candles
self.testing_candles = testing_candles
key = jh.key(self.exchange, self.symbol)
training_candles_start_date = jh.timestamp_to_time(self.training_candles[key]['candles'][0][0]).split('T')[0]
training_candles_finish_date = jh.timestamp_to_time(self.training_candles[key]['candles'][-1][0]).split('T')[0]
testing_candles_start_date = jh.timestamp_to_time(self.testing_candles[key]['candles'][0][0]).split('T')[0]
testing_candles_finish_date = jh.timestamp_to_time(self.testing_candles[key]['candles'][-1][0]).split('T')[0]
self.training_initial_candles = []
self.testing_initial_candles = []
for c in config['app']['considering_candles']:
self.training_initial_candles.append(
required_candles.load_required_candles(c[0], c[1], training_candles_start_date,
training_candles_finish_date))
self.testing_initial_candles.append(
required_candles.load_required_candles(c[0], c[1], testing_candles_start_date,
testing_candles_finish_date))
def info(msg):
from jesse.store import store
store.logs.info.append({'time': jh.now(), 'message': msg})
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(jh.color('[{}]: {}'.format(jh.timestamp_to_time(jh.now()), msg), 'magenta'))
if jh.is_live():
msg = '[INFO | {}] '.format(jh.timestamp_to_time(jh.now())[:19]) + str(msg)
import logging
logging.info(msg)
def error(msg):
from jesse.store import store
if jh.is_live() and jh.get_config('env.notifications.events.errors', True):
notify('ERROR:\n{}'.format(msg))
if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data():
print(jh.color('[{}]: {}'.format(jh.timestamp_to_time(jh.now()), msg), 'red'))
store.logs.errors.append({'time': jh.now(), 'message': msg})
if jh.is_live():
msg = '[ERROR | {}] '.format(jh.timestamp_to_time(jh.now())[:19]) + str(msg)
import logging
logging.error(msg)
def async_save():
Trade.insert(**d).on_conflict_ignore().execute()
print(
jh.color(
'trade: {}-{}-{}: {}'.format(
jh.timestamp_to_time(d['timestamp']), exchange, symbol,
trade), 'green'))
def async_save():
Candle.insert(**d).on_conflict_ignore().execute()
print(
jh.color(
'candle: {}-{}-{}: {}'.format(
jh.timestamp_to_time(d['timestamp']), exchange, symbol,
candle), 'blue'))
def errors() -> List[List[Union[str, Any]]]:
array = []
for w in store.logs.errors[::-1][0:5]:
array.append([jh.timestamp_to_time(w['time'])[11:19],
f"{w['message'][:70]}.." if len(w['message']) > 70 else w['message']])
return array
def async_save():
Ticker.insert(**d).on_conflict_ignore().execute()
print(
jh.color('ticker: {}-{}-{}: {}'.format(
jh.timestamp_to_time(d['timestamp']), exchange, symbol, ticker
), 'yellow')
)
def info() -> List[List[Union[str, Any]]]:
array = []
for w in store.logs.info[::-1][0:5]:
array.append(
[jh.timestamp_to_time(w['time'])[11:19],
(w['message'][:70] + '..') if len(w['message']) > 70 else w['message']])
return array
def errors():
array = []
for w in store.logs.errors[::-1][0:5]:
array.append([jh.timestamp_to_time(w['time'])[11:19],
(w['message'][:70] + '..') if len(w['message']) > 70 else w['message']])
return array
def candles():
"""
:return:
"""
array = []
candle_keys = []
# add routes
for e in router.routes:
if e.strategy is None:
return
candle_keys.append({
'exchange': e.exchange,
'symbol': e.symbol,
'timeframe': e.timeframe
})
# add extra_routes
for e in router.extra_candles:
candle_keys.append({
'exchange': e[0],
'symbol': e[1],
'timeframe': e[2]
})
# headers
array.append([
'exchange-symbol-timeframe', 'timestamp', 'open', 'close', 'high',
'low'
])
for k in candle_keys:
try:
current_candle = store.candles.get_current_candle(
k['exchange'], k['symbol'], k['timeframe'])
green = is_bullish(current_candle)
bold = k['symbol'] in config['app']['trading_symbols'] and k[
'timeframe'] in config['app']['trading_timeframes']
key = jh.key(k['exchange'], k['symbol'], k['timeframe'])
array.append([
jh.style(key, 'underline' if bold else None),
jh.color(jh.timestamp_to_time(current_candle[0]),
'green' if green else 'red'),
jh.color(str(current_candle[1]), 'green' if green else 'red'),
jh.color(str(current_candle[2]), 'green' if green else 'red'),
jh.color(str(current_candle[3]), 'green' if green else 'red'),
jh.color(str(current_candle[4]), 'green' if green else 'red'),
])
except IndexError:
return
except Exception:
raise
return array
def _fill_absent_candles(temp_candles, start_timestamp, end_timestamp):
if len(temp_candles) == 0:
raise CandleNotFoundInExchange(
'No candles exists in the market for this day: {} \n'
'Try another start_date'.format(
jh.timestamp_to_time(start_timestamp)[:10],
)
)
symbol = temp_candles[0]['symbol']
exchange = temp_candles[0]['exchange']
candles = []
first_candle = temp_candles[0]
started = False
loop_length = ((end_timestamp - start_timestamp) / 60000) + 1
i = 0
while i < loop_length:
candle_for_timestamp = pydash.find(
temp_candles, lambda c: c['timestamp'] == start_timestamp)
if candle_for_timestamp is None:
if started:
last_close = candles[-1]['close']
candles.append({
'id': jh.generate_unique_id(),
'symbol': symbol,
'exchange': exchange,
'timestamp': start_timestamp,
'open': last_close,
'high': last_close,
'low': last_close,
'close': last_close,
'volume': 0
})
else:
candles.append({
'id': jh.generate_unique_id(),
'symbol': symbol,
'exchange': exchange,
'timestamp': start_timestamp,
'open': first_candle['open'],
'high': first_candle['open'],
'low': first_candle['open'],
'close': first_candle['open'],
'volume': 0
})
# candle is present
else:
started = True
candles.append(candle_for_timestamp)
start_timestamp += 60000
i += 1
return candles
def __init__(self, training_candles, testing_candles):
if len(router.routes) != 1:
raise NotImplementedError(
'optimize_mode mode only supports one route at the moment')
self.strategy_name = router.routes[0].strategy_name
self.exchange = router.routes[0].exchange
self.symbol = router.routes[0].symbol
self.timeframe = router.routes[0].timeframe
StrategyClass = jh.get_strategy_class(self.strategy_name)
self.strategy_hp = StrategyClass.hyper_parameters()
solution_len = len(self.strategy_hp)
if solution_len == 0:
raise InvalidStrategy(
'Targeted strategy does not implement a valid hyper_parameters() method.'
)
super().__init__(iterations=2000 * solution_len,
population_size=solution_len * 100,
solution_len=solution_len,
options={
'strategy_name': self.strategy_name,
'exchange': self.exchange,
'symbol': self.symbol,
'timeframe': self.timeframe
})
self.training_candles = training_candles
self.testing_candles = testing_candles
key = jh.key(self.exchange, self.symbol)
# training
self.required_initial_training_candles = required_candles.load_required_candles(
self.exchange, self.symbol,
jh.timestamp_to_time(self.training_candles[key]['candles'][0][0]),
jh.timestamp_to_time(self.training_candles[key]['candles'][-1][0]))
# testing
self.required_initial_testing_candles = required_candles.load_required_candles(
self.exchange, self.symbol,
jh.timestamp_to_time(self.testing_candles[key]['candles'][0][0]),
jh.timestamp_to_time(self.testing_candles[key]['candles'][-1][0]))
def errors() -> List[List[Union[str, Any]]]:
return [
[
jh.timestamp_to_time(w['time'])[11:19],
f"{w['message'][:70]}.."
if len(w['message']) > 70
else w['message'],
]
for w in store.logs.errors[::-1][0:5]
]
def log_exchange_message(exchange, message):
# if the type of message is not str, convert it to str
if not isinstance(message, str):
message = str(message)
formatted_time = jh.timestamp_to_time(jh.now())[:19]
message = f'[{formatted_time} - {exchange}]: ' + message
if 'exchange-streams' not in LOGGERS:
create_disposable_logger('exchange-streams')
LOGGERS['exchange-streams'].info(message)
def livetrade():
"""
:return:
"""
# sum up balance of all trading exchanges
starting_balance = 0
current_balance = 0
for e in store.exchanges.storage:
starting_balance += store.exchanges.storage[e].starting_balance
current_balance += store.exchanges.storage[e].balance
starting_balance = round(starting_balance, 2)
current_balance = round(current_balance, 2)
arr = [[
'started/current balance', '{}/{}'.format(starting_balance,
current_balance)
], ['started at',
jh.get_arrow(store.app.starting_time).humanize()],
['current time',
jh.timestamp_to_time(jh.now())[:19]],
[
'errors/info', '{}/{}'.format(len(store.logs.errors),
len(store.logs.info))
], ['active orders',
store.orders.count_all_active_orders()],
['open positions',
store.positions.count_open_positions()]]
# short trades summary
if len(store.completed_trades.trades):
df = pd.DataFrame.from_records(
[t.to_dict() for t in store.completed_trades.trades])
total = len(df)
winning_trades = df.loc[df['PNL'] > 0]
losing_trades = df.loc[df['PNL'] < 0]
pnl = round(df['PNL'].sum(), 2)
pnl_percentage = round((pnl / starting_balance) * 100, 2)
arr.append([
'total/winning/losing trades',
'{}/{}/{}'.format(total, len(winning_trades), len(losing_trades))
])
arr.append(['PNL (%)', '${} ({}%)'.format(pnl, pnl_percentage)])
if config['app']['debug_mode']:
arr.append(['debug mode', config['app']['debug_mode']])
if config['app']['is_test_driving']:
arr.append(['Test Drive', config['app']['is_test_driving']])
return arr
def log_optimize_mode(message):
# if the type of message is not str, convert it to str
if not isinstance(message, str):
message = str(message)
formatted_time = jh.timestamp_to_time(jh.now())[:19]
message = f'[{formatted_time}]: ' + message
file_name = 'optimize-mode'
if file_name not in LOGGERS:
create_logger_file(file_name)
LOGGERS[file_name].info(message)
def async_save():
Orderbook.insert(**d).on_conflict_ignore().execute()
print(
jh.color(
'orderbook: {}-{}-{}: [{}, {}], [{}, {}]'.format(
jh.timestamp_to_time(d['timestamp']), exchange, symbol,
# best ask
orderbook[0][0][0], orderbook[0][0][1],
# best bid
orderbook[1][0][0], orderbook[1][0][1]
),
'magenta'
)
)
def candles(candles_array):
period = jh.date_diff_in_days(jh.get_arrow(candles_array[0][0]), jh.get_arrow(candles_array[-1][0])) + 1
if period > 365:
duration = '{} days ({} years)'.format(period, round(period / 365, 2))
elif period > 30:
duration = '{} days ({} months)'.format(period, round(period / 30, 2))
else:
duration = '{} days'.format(period)
return [
['period', duration],
['starting-ending date', '{} => {}'.format(jh.timestamp_to_time(candles_array[0][0])[:10],
jh.timestamp_to_time(candles_array[-1][0] + 60_000)[:10])],
]
def info():
"""
:return:
"""
array = []
for w in store.logs.info[::-1][0:5]:
array.append([
jh.timestamp_to_time(w['time'])[11:19],
(w['message'][:70] +
'..') if len(w['message']) > 70 else w['message']
])
return array
def run(exchange: str,
symbol: str,
start_date_str: str,
skip_confirmation: bool = False,
mode: str = 'candles') -> None:
config['app']['trading_mode'] = mode
# first, create and set session_id
store.app.set_session_id()
register_custom_exception_handler()
# close database connection
from jesse.services.db import database
database.open_connection()
# at every second, we check to see if it's time to execute stuff
status_checker = Timeloop()
@status_checker.job(interval=timedelta(seconds=1))
def handle_time():
if process_status() != 'started':
raise exceptions.Termination
status_checker.start()
try:
start_timestamp = jh.arrow_to_timestamp(
arrow.get(start_date_str, 'YYYY-MM-DD'))
except:
raise ValueError(
'start_date must be a string representing a date before today. ex: 2020-01-17'
)
# more start_date validations
today = arrow.utcnow().floor('day').int_timestamp * 1000
if start_timestamp == today:
raise ValueError(
"Today's date is not accepted. start_date must be a string a representing date BEFORE today."
)
elif start_timestamp > today:
raise ValueError(
"Future's date is not accepted. start_date must be a string a representing date BEFORE today."
)
# We just call this to throw a exception in case of a symbol without dash
jh.quote_asset(symbol)
click.clear()
symbol = symbol.upper()
until_date = arrow.utcnow().floor('day')
start_date = arrow.get(start_timestamp / 1000)
days_count = jh.date_diff_in_days(start_date, until_date)
candles_count = days_count * 1440
try:
driver: CandleExchange = drivers[exchange]()
except KeyError:
raise ValueError(f'{exchange} is not a supported exchange')
except TypeError:
raise FileNotFoundError('You are missing the "plugins.py" file')
loop_length = int(candles_count / driver.count) + 1
# ask for confirmation
if not skip_confirmation:
click.confirm(
f'Importing {days_count} days candles from "{exchange}" for "{symbol}". Duplicates will be skipped. All good?',
abort=True,
default=True)
progressbar = Progressbar(loop_length)
for i in range(candles_count):
temp_start_timestamp = start_date.int_timestamp * 1000
temp_end_timestamp = temp_start_timestamp + (driver.count - 1) * 60000
# to make sure it won't try to import candles from the future! LOL
if temp_start_timestamp > jh.now_to_timestamp():
break
# prevent duplicates calls to boost performance
count = Candle.select().where(
Candle.timestamp.between(temp_start_timestamp, temp_end_timestamp),
Candle.symbol == symbol, Candle.exchange == exchange).count()
already_exists = count == driver.count
if not already_exists:
# it's today's candles if temp_end_timestamp < now
if temp_end_timestamp > jh.now_to_timestamp():
temp_end_timestamp = arrow.utcnow().floor(
'minute').int_timestamp * 1000 - 60000
# fetch from market
candles = driver.fetch(symbol, temp_start_timestamp)
# check if candles have been returned and check those returned start with the right timestamp.
# Sometimes exchanges just return the earliest possible candles if the start date doesn't exist.
if not len(candles) or arrow.get(
candles[0]['timestamp'] / 1000) > start_date:
click.clear()
first_existing_timestamp = driver.get_starting_time(symbol)
# if driver can't provide accurate get_starting_time()
if first_existing_timestamp is None:
raise CandleNotFoundInExchange(
f'No candles exists in the market for this day: {jh.timestamp_to_time(temp_start_timestamp)[:10]} \n'
'Try another start_date')
# handle when there's missing candles during the period
if temp_start_timestamp > first_existing_timestamp:
# see if there are candles for the same date for the backup exchange,
# if so, get those, if not, download from that exchange.
if driver.backup_exchange is not None:
candles = _get_candles_from_backup_exchange(
exchange, driver.backup_exchange, symbol,
temp_start_timestamp, temp_end_timestamp)
else:
temp_start_time = jh.timestamp_to_time(
temp_start_timestamp)[:10]
temp_existing_time = jh.timestamp_to_time(
first_existing_timestamp)[:10]
sync_publish(
'alert', {
'message':
f'No candle exists in the market for {temp_start_time}. So '
f'Jesse started importing since the first existing date which is {temp_existing_time}',
'type':
'success'
})
run(exchange, symbol,
jh.timestamp_to_time(first_existing_timestamp)[:10],
True)
return
# fill absent candles (if there's any)
candles = _fill_absent_candles(candles, temp_start_timestamp,
temp_end_timestamp)
# store in the database
if skip_confirmation:
store_candles(candles)
else:
threading.Thread(target=store_candles, args=[candles]).start()
# add as much as driver's count to the temp_start_time
start_date = start_date.shift(minutes=driver.count)
progressbar.update()
sync_publish(
'progressbar', {
'current':
progressbar.current,
'estimated_remaining_seconds':
progressbar.estimated_remaining_seconds
})
# sleep so that the exchange won't get angry at us
if not already_exists:
time.sleep(driver.sleep_time)
# stop the status_checker time loop
status_checker.stop()
sync_publish(
'alert', {
'message':
f'Successfully imported candles since {jh.timestamp_to_date(start_timestamp)} until today ({days_count} days). ',
'type': 'success'
})
# if it is to skip, then it's being called from another process hence we should leave the database be
if not skip_confirmation:
# close database connection
from jesse.services.db import database
database.close_connection()
def test_timestamp_to_time():
assert jh.timestamp_to_time(1558770180000) == '2019-05-25T07:43:00+00:00'
def livetrade() -> List[Union[List[Union[str, Any]], List[str], List[Union[
str, int]], List[Union[str, Dict[str, Union[str, int]], Dict[
str, str], Dict[str, bool], Dict[str, Union[Dict[str, Union[
int, str, List[Dict[str, Union[str, int]]]]], Dict[str, Union[
float, str, int, List[Dict[str, Union[str, int]]]]]]], Dict[
str, int]]]]]:
# sum up balance of all trading exchanges
starting_balance = 0
current_balance = 0
for e in store.exchanges.storage:
starting_balance += store.exchanges.storage[e].starting_assets[
jh.app_currency()]
current_balance += store.exchanges.storage[e].assets[jh.app_currency()]
starting_balance = round(starting_balance, 2)
current_balance = round(current_balance, 2)
arr = [[
'started at',
jh.timestamp_to_arrow(store.app.starting_time).humanize()
], ['current time',
jh.timestamp_to_time(jh.now_to_timestamp())[:19]],
['errors/info', f'{len(store.logs.errors)}/{len(store.logs.info)}'],
['active orders',
store.orders.count_all_active_orders()],
['open positions',
store.positions.count_open_positions()]]
# TODO: for now, we assume that we trade on one exchange only. Later, we need to support for more than one exchange at a time
first_exchange = selectors.get_exchange(router.routes[0].exchange)
if first_exchange.type == 'futures':
arr.append([
'started/current balance', f'{starting_balance}/{current_balance}'
])
else:
# loop all trading exchanges
for exchange in selectors.get_all_exchanges():
# loop all assets
for asset_name, asset_balance in exchange.assets.items():
if asset_name == jh.base_asset(router.routes[0].symbol):
current_price = selectors.get_current_price(
router.routes[0].exchange, router.routes[0].symbol)
arr.append([
f'{asset_name}',
f'{round(exchange.available_assets[asset_name], 5)}/{round(asset_balance, 5)} ({jh.format_currency(round(asset_balance * current_price, 2))} { jh.quote_asset(router.routes[0].symbol)})'
])
else:
arr.append([
f'{asset_name}',
f'{round(exchange.available_assets[asset_name], 5)}/{round(asset_balance, 5)}'
])
# short trades summary
if len(store.completed_trades.trades):
df = pd.DataFrame.from_records(
[t.to_dict() for t in store.completed_trades.trades])
total = len(df)
winning_trades = df.loc[df['PNL'] > 0]
losing_trades = df.loc[df['PNL'] < 0]
pnl = round(df['PNL'].sum(), 2)
pnl_percentage = round((pnl / starting_balance) * 100, 2)
arr.append([
'total/winning/losing trades',
f'{total}/{len(winning_trades)}/{len(losing_trades)}'
])
arr.append(['PNL (%)', f'${pnl} ({pnl_percentage}%)'])
if config['app']['debug_mode']:
arr.append(['debug mode', config['app']['debug_mode']])
if config['app']['is_test_driving']:
arr.append(['Test Drive', config['app']['is_test_driving']])
return arr
def _get_candles_from_backup_exchange(exchange: str,
backup_driver: CandleExchange,
symbol: str, start_timestamp: int,
end_timestamp: int):
total_candles = []
# try fetching from database first
backup_candles = Candle.select(
Candle.timestamp, Candle.open, Candle.close, Candle.high, Candle.low,
Candle.volume).where(
Candle.timestamp.between(start_timestamp, end_timestamp),
Candle.exchange == backup_driver.name,
Candle.symbol == symbol).order_by(Candle.timestamp.asc()).tuples()
already_exists = len(
backup_candles) == (end_timestamp - start_timestamp) / 60_000 + 1
if already_exists:
# loop through them and set new ID and exchange
for c in backup_candles:
total_candles.append({
'id': jh.generate_unique_id(),
'symbol': symbol,
'exchange': exchange,
'timestamp': c[0],
'open': c[1],
'close': c[2],
'high': c[3],
'low': c[4],
'volume': c[5]
})
return total_candles
# try fetching from market now
days_count = jh.date_diff_in_days(jh.timestamp_to_arrow(start_timestamp),
jh.timestamp_to_arrow(end_timestamp))
# make sure it's rounded up so that we import maybe more candles, but not less
if days_count < 1:
days_count = 1
if type(days_count) is float and not days_count.is_integer():
days_count = math.ceil(days_count)
candles_count = days_count * 1440
start_date = jh.timestamp_to_arrow(start_timestamp).floor('day')
for _ in range(candles_count):
temp_start_timestamp = start_date.int_timestamp * 1000
temp_end_timestamp = temp_start_timestamp + (backup_driver.count -
1) * 60000
# to make sure it won't try to import candles from the future! LOL
if temp_start_timestamp > jh.now_to_timestamp():
break
# prevent duplicates
count = Candle.select().where(
Candle.timestamp.between(temp_start_timestamp, temp_end_timestamp),
Candle.symbol == symbol,
Candle.exchange == backup_driver.name).count()
already_exists = count == backup_driver.count
if not already_exists:
# it's today's candles if temp_end_timestamp < now
if temp_end_timestamp > jh.now_to_timestamp():
temp_end_timestamp = arrow.utcnow().floor(
'minute').int_timestamp * 1000 - 60000
# fetch from market
candles = backup_driver.fetch(symbol, temp_start_timestamp)
if not len(candles):
raise CandleNotFoundInExchange(
'No candles exists in the market for this day: {} \n'
'Try another start_date'.format(
jh.timestamp_to_time(temp_start_timestamp)[:10], ))
# fill absent candles (if there's any)
candles = _fill_absent_candles(candles, temp_start_timestamp,
temp_end_timestamp)
# store in the database
_insert_to_database(candles)
# add as much as driver's count to the temp_start_time
start_date = start_date.shift(minutes=backup_driver.count)
# sleep so that the exchange won't get angry at us
if not already_exists:
time.sleep(backup_driver.sleep_time)
# now try fetching from database again. Why? because we might have fetched more
# than what's needed, but we only want as much was requested. Don't worry, the next
# request will probably fetch from database and there won't be any waste!
backup_candles = Candle.select(
Candle.timestamp, Candle.open, Candle.close, Candle.high, Candle.low,
Candle.volume).where(
Candle.timestamp.between(start_timestamp, end_timestamp),
Candle.exchange == backup_driver.name,
Candle.symbol == symbol).order_by(Candle.timestamp.asc()).tuples()
already_exists = len(
backup_candles) == (end_timestamp - start_timestamp) / 60_000 + 1
if already_exists:
# loop through them and set new ID and exchange
for c in backup_candles:
total_candles.append({
'id': jh.generate_unique_id(),
'symbol': symbol,
'exchange': exchange,
'timestamp': c[0],
'open': c[1],
'close': c[2],
'high': c[3],
'low': c[4],
'volume': c[5]
})
return total_candles
def run(exchange: str,
symbol: str,
start_date_str: str,
skip_confirmation=False):
try:
start_timestamp = jh.arrow_to_timestamp(
arrow.get(start_date_str, 'YYYY-MM-DD'))
except:
raise ValueError(
'start_date must be a string representing a date before today. ex: 2020-01-17'
)
# more start_date validations
today = arrow.utcnow().floor('day').int_timestamp * 1000
if start_timestamp == today:
raise ValueError(
"Today's date is not accepted. start_date must be a string a representing date BEFORE today."
)
elif start_timestamp > today:
raise ValueError(
"Future's date is not accepted. start_date must be a string a representing date BEFORE today."
)
click.clear()
symbol = symbol.upper()
until_date = arrow.utcnow().floor('day')
start_date = arrow.get(start_timestamp / 1000)
days_count = jh.date_diff_in_days(start_date, until_date)
candles_count = days_count * 1440
exchange = exchange.title()
try:
driver: CandleExchange = drivers[exchange]()
except KeyError:
raise ValueError('{} is not a supported exchange'.format(exchange))
loop_length = int(candles_count / driver.count) + 1
# ask for confirmation
if not skip_confirmation:
click.confirm(
'Importing {} days candles from "{}" for "{}". Duplicates will be skipped. All good?'
.format(days_count, exchange, symbol),
abort=True,
default=True)
with click.progressbar(length=loop_length,
label='Importing candles...') as progressbar:
for _ in range(candles_count):
temp_start_timestamp = start_date.int_timestamp * 1000
temp_end_timestamp = temp_start_timestamp + (driver.count -
1) * 60000
# to make sure it won't try to import candles from the future! LOL
if temp_start_timestamp > jh.now_to_timestamp():
break
# prevent duplicates calls to boost performance
count = Candle.select().where(
Candle.timestamp.between(temp_start_timestamp,
temp_end_timestamp),
Candle.symbol == symbol, Candle.exchange == exchange).count()
already_exists = count == driver.count
if not already_exists:
# it's today's candles if temp_end_timestamp < now
if temp_end_timestamp > jh.now_to_timestamp():
temp_end_timestamp = arrow.utcnow().floor(
'minute').int_timestamp * 1000 - 60000
# fetch from market
candles = driver.fetch(symbol, temp_start_timestamp)
if not len(candles):
click.clear()
first_existing_timestamp = driver.get_starting_time(symbol)
# if driver can't provide accurate get_starting_time()
if first_existing_timestamp is None:
raise CandleNotFoundInExchange(
'No candles exists in the market for this day: {} \n'
'Try another start_date'.format(
jh.timestamp_to_time(temp_start_timestamp)
[:10], ))
# handle when there's missing candles during the period
if temp_start_timestamp > first_existing_timestamp:
# see if there are candles for the same date for the backup exchange,
# if so, get those, if not, download from that exchange.
driver.init_backup_exchange()
if driver.backup_exchange is not None:
candles = _get_candles_from_backup_exchange(
exchange, driver.backup_exchange, symbol,
temp_start_timestamp, temp_end_timestamp)
else:
if not skip_confirmation:
print(
jh.color(
'No candle exists in the market for {}\n'.
format(
jh.timestamp_to_time(
temp_start_timestamp)[:10]),
'yellow'))
click.confirm(
'First present candle is since {}. Would you like to continue?'
.format(
jh.timestamp_to_time(
first_existing_timestamp)[:10]),
abort=True,
default=True)
run(
exchange, symbol,
jh.timestamp_to_time(first_existing_timestamp)
[:10], True)
return
# fill absent candles (if there's any)
candles = _fill_absent_candles(candles, temp_start_timestamp,
temp_end_timestamp)
# store in the database
if skip_confirmation:
_insert_to_database(candles)
else:
threading.Thread(target=_insert_to_database,
args=[candles]).start()
# add as much as driver's count to the temp_start_time
start_date = start_date.shift(minutes=driver.count)
progressbar.update(1)
# sleep so that the exchange won't get angry at us
if not already_exists:
time.sleep(driver.sleep_time)
