def test_get_current_and_past_ticker(): set_up() # add 4 tickers t1 = np.array([jh.now(), 1, 2, 3, 4], dtype=np.float64) t2 = np.array([jh.now() + 1000, 2, 2, 3, 4], dtype=np.float64) t3 = np.array([jh.now() + 2000, 3, 2, 3, 4], dtype=np.float64) t4 = np.array([jh.now() + 3000, 4, 2, 3, 4], dtype=np.float64) store.tickers.add_ticker(t1, 'Sandbox', 'BTCUSD') store.app.time += 1000 store.tickers.add_ticker(t2, 'Sandbox', 'BTCUSD') store.app.time += 1000 store.tickers.add_ticker(t3, 'Sandbox', 'BTCUSD') store.app.time += 1000 store.tickers.add_ticker(t4, 'Sandbox', 'BTCUSD') np.testing.assert_equal(store.tickers.get_tickers('Sandbox', 'BTCUSD'), np.array([t1, t2, t3, t4])) # get the previous one np.testing.assert_equal( store.tickers.get_past_ticker('Sandbox', 'BTCUSD', 1), t3) # get current np.testing.assert_equal( store.tickers.get_current_ticker('Sandbox', 'BTCUSD'), t4)
def add_orderbook(self, exchange: str, symbol: str, asks: list, bids: list): """ :param exchange: :param symbol: :param asks: :param bids: """ key = jh.key(exchange, symbol) self.temp_storage[key]['asks'] = asks self.temp_storage[key]['bids'] = bids # generate new numpy formatted orderbook if it is # either the first time, or that it has passed # 1000 milliseconds since the last time if self.temp_storage[key]['last_updated_timestamp'] is None or jh.now( ) - self.temp_storage[key]['last_updated_timestamp'] >= 1000: self.temp_storage[key]['last_updated_timestamp'] = jh.now() formatted_orderbook = self.format_orderbook(exchange, symbol) if jh.is_collecting_data(): store_orderbook_into_db(exchange, symbol, formatted_orderbook) else: self.storage[key].append(formatted_orderbook)
def get_config(client_config: dict, has_live=False) -> dict: from jesse.services.db import database database.open_connection() from jesse.models.Option import Option try: o = Option.get(Option.type == 'config') # merge it with client's config (because it could include new keys added), # update it in the database, and then return it data = jh.merge_dicts(client_config, json.loads(o.json)) # make sure the list of BACKTEST exchanges is up to date from jesse.modes.import_candles_mode.drivers import drivers for k in list(data['backtest']['exchanges'].keys()): if k not in drivers: del data['backtest']['exchanges'][k] # make sure the list of LIVE exchanges is up to date if has_live: from jesse_live.info import SUPPORTED_EXCHANGES_NAMES live_exchanges = list(sorted(SUPPORTED_EXCHANGES_NAMES)) for k in list(data['live']['exchanges'].keys()): if k not in live_exchanges: del data['live']['exchanges'][k] # fix the settlement_currency of exchanges for k, e in data['live']['exchanges'].items(): e['settlement_currency'] = jh.get_settlement_currency_from_exchange( e['name']) for k, e in data['backtest']['exchanges'].items(): e['settlement_currency'] = jh.get_settlement_currency_from_exchange( e['name']) o.updated_at = jh.now() o.save() except peewee.DoesNotExist: # if not found, that means it's the first time. Store in the DB and # then return what was sent from the client side without changing it o = Option({ 'id': jh.generate_unique_id(), 'updated_at': jh.now(), 'type': 'config', 'json': json.dumps(client_config) }) o.save(force_insert=True) data = client_config database.close_connection() return {'data': data}
def info(msg): from jesse.store import store store.logs.info.append({'time': jh.now(), 'message': msg}) if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data(): print(jh.color('[{}]: {}'.format(jh.timestamp_to_time(jh.now()), msg), 'magenta')) if jh.is_live(): msg = '[INFO | {}] '.format(jh.timestamp_to_time(jh.now())[:19]) + str(msg) import logging logging.info(msg)
def test_can_log_info_by_firing_event(): set_up() # fire first info event logger.info('first info!!!!!') first_logged_info = {'time': jh.now(), 'message': 'first info!!!!!'} assert store.logs.info == [first_logged_info] # fire second info event logger.info('second info!!!!!') second_logged_info = {'time': jh.now(), 'message': 'second info!!!!!'} assert store.logs.info == [first_logged_info, second_logged_info]
def test_can_log_error_by_firing_event(): set_up() # fire first error event logger.error('first error!!!!!') first_logged_error = {'time': jh.now(), 'message': 'first error!!!!!'} assert store.logs.errors == [first_logged_error] # fire second error event logger.error('second error!!!!!') second_logged_error = {'time': jh.now(), 'message': 'second error!!!!!'} assert store.logs.errors == [first_logged_error, second_logged_error]
def error(msg): from jesse.store import store if jh.is_live() and jh.get_config('env.notifications.events.errors', True): notify('ERROR:\n{}'.format(msg)) if (jh.is_backtesting() and jh.is_debugging()) or jh.is_collecting_data(): print(jh.color('[{}]: {}'.format(jh.timestamp_to_time(jh.now()), msg), 'red')) store.logs.errors.append({'time': jh.now(), 'message': msg}) if jh.is_live(): msg = '[ERROR | {}] '.format(jh.timestamp_to_time(jh.now())[:19]) + str(msg) import logging logging.error(msg)
def _open(self, qty, price, change_balance=True): if self.is_open: raise OpenPositionError( 'an already open position cannot be opened') if change_balance: size = abs(qty) * price if self.exchange: self.exchange.decrease_balance(self, size) self.entry_price = price self.exit_price = None self.qty = qty self.opened_at = jh.now() info_text = 'OPENED {} position: {}, {}, {}, ${}'.format( self.type, self.exchange_name, self.symbol, self.qty, round(self.entry_price, 2)) if jh.is_debuggable('position_opened'): logger.info(info_text) if jh.is_live( ) and config['env']['notifications']['events']['updated_position']: notifier.notify(info_text)
def _close(self, close_price): if self.is_open is False: raise EmptyPosition('The position is already closed.') # just to prevent confusion close_qty = abs(self.qty) estimated_profit = jh.estimate_PNL(close_qty, self.entry_price, close_price, self.type) entry = self.entry_price trade_type = self.type self.exit_price = close_price if self.exchange: self.exchange.increase_balance( self, close_qty * self.entry_price + estimated_profit) self.qty = 0 self.entry_price = None self.closed_at = jh.now() info_text = 'CLOSED {} position: {}, {}. PNL: ${}, entry: {}, exit: {}'.format( trade_type, self.exchange_name, self.symbol, round(estimated_profit, 2), entry, close_price) if jh.is_debuggable('position_closed'): logger.info(info_text) if jh.is_live( ) and config['env']['notifications']['events']['updated_position']: notifier.notify(info_text)
def cancel(self): """ :return: """ if self.is_canceled or self.is_executed: return self.canceled_at = jh.now() self.status = order_statuses.CANCELED if jh.is_debuggable('order_cancellation'): logger.info( 'CANCELED order: {}, {}, {}, {}, ${}'.format( self.symbol, self.type, self.side, self.qty, round(self.price, 2) ) ) # notify if jh.is_live() and config['env']['notifications']['events']['cancelled_orders']: notify( 'CANCELED order: {}, {}, {}, {}, {}'.format( self.symbol, self.type, self.side, self.qty, round(self.price, 2) ) ) p = selectors.get_position(self.exchange, self.symbol) if p: p._on_canceled_order(self)
def execute(self): if self.is_canceled or self.is_executed: return self.executed_at = jh.now() self.status = order_statuses.EXECUTED # log if jh.is_debuggable('order_execution'): logger.info('EXECUTED order: {}, {}, {}, {}, ${}'.format( self.symbol, self.type, self.side, self.qty, round(self.price, 2))) # notify if jh.is_live( ) and config['env']['notifications']['events']['executed_orders']: notify('EXECUTED order: {}, {}, {}, {}, {}'.format( self.symbol, self.type, self.side, self.qty, round(self.price, 2))) p = selectors.get_position(self.exchange, self.symbol) if p: p._on_executed_order(self) # handle exchange balance for ordered asset e = selectors.get_exchange(self.exchange) e.on_order_execution(self)
def test_trade_size(): trade = CompletedTrade({ 'type': 'long', 'exchange': 'Sandbox', 'entry_price': 10, 'exit_price': 20, 'take_profit_at': 20, 'stop_loss_at': 5, 'qty': 1, 'orders': [], 'symbol': 'BTCUSD', 'opened_at': jh.now(), 'closed_at': jh.now() }) assert trade.size == 10
def get_starting_time(self, symbol: str) -> int: formatted_symbol = symbol.replace('USD', 'PERP') end_timestamp = jh.now() start_timestamp = end_timestamp - (86400_000 * 365 * 8) payload = { 'resolution': 86400, 'start_time': start_timestamp / 1000, 'end_time': end_timestamp / 1000, } response = requests.get( f'https://ftx.com/api/markets/{formatted_symbol}/candles', params=payload) self._handle_errors(response) data = response.json()['result'] # since the first timestamp doesn't include all the 1m # candles, let's start since the second day then first_timestamp = int(data[0]['time']) # second_timestamp: return first_timestamp + 60_000 * 1440
def save_daily_portfolio_balance() -> None: balances = [] # add exchange balances for key, e in store.exchanges.storage.items(): balances.append(e.assets[jh.app_currency()]) # store daily_balance of assets into database if jh.is_livetrading(): for asset_key, asset_value in e.assets.items(): store_daily_balance_into_db({ 'id': jh.generate_unique_id(), 'timestamp': jh.now(), 'identifier': jh.get_config('env.identifier', 'main'), 'exchange': e.name, 'asset': asset_key, 'balance': asset_value, }) # add open position values for key, pos in store.positions.storage.items(): if pos.is_open: balances.append(pos.pnl) total = sum(balances) store.app.daily_balance.append(total) logger.info('Saved daily portfolio balance: {}'.format(round(total, 2)))
def store_orderbook_into_db(exchange: str, symbol: str, orderbook: np.ndarray): d = { 'id': jh.generate_unique_id(), 'timestamp': jh.now(), 'data': orderbook.dumps(), 'symbol': symbol, 'exchange': exchange, } def async_save(): Orderbook.insert(**d).on_conflict_ignore().execute() print( jh.color( 'orderbook: {}-{}-{}: [{}, {}], [{}, {}]'.format( jh.timestamp_to_time(d['timestamp']), exchange, symbol, # best ask orderbook[0][0][0], orderbook[0][0][1], # best bid orderbook[1][0][0], orderbook[1][0][1] ), 'magenta' ) ) # async call threading.Thread(target=async_save).start()
def __init__( self, iterations, population_size, solution_len, charset='()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvw', fitness_goal=1, options=None): """ :param iterations: int :param population_size: int :param solution_len: int :param charset: str default= '()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvw' which is 40-119 (len=80) :param fitness_goal: """ # used for naming the files related to this session self.session_id = str(jh.now()) self.started_index = 0 self.start_time = jh.now() self.population = [] self.iterations = iterations self.population_size = population_size self.solution_len = solution_len self.charset = charset self.fitness_goal = fitness_goal if options is None: self.options = {} else: self.options = options os.makedirs('./storage/temp/optimize', exist_ok=True) self.temp_path = './storage/temp/optimize/{}-{}-{}-{}.pickle'.format( self.options['strategy_name'], self.options['exchange'], self.options['symbol'], self.options['timeframe']) if fitness_goal > 1 or fitness_goal < 0: raise ValueError('fitness scores must be between 0 and 1') # if temp file exists, load data to resume previous session if jh.file_exists(self.temp_path): if click.confirm( 'Previous session detected. Do you want to resume?', default=True): self.load_progress()
def test_PNL_percentage(): no_fee() trade = CompletedTrade({ 'type': 'long', 'exchange': 'Sandbox', 'entry_price': 10, 'exit_price': 12, 'take_profit_at': 20, 'stop_loss_at': 5, 'qty': 1, 'orders': [], 'symbol': 'BTCUSD', 'opened_at': jh.now(), 'closed_at': jh.now() }) assert trade.PNL_percentage == 20
def add_candle(self, candle: np.ndarray, exchange: str, symbol: str, timeframe: str, with_execution: bool = True, with_generation: bool = True) -> None: if jh.is_collecting_data(): # make sure it's a complete (and not a forming) candle if jh.now_to_timestamp() >= (candle[0] + 60000): store_candle_into_db(exchange, symbol, candle) return arr: DynamicNumpyArray = self.get_storage(exchange, symbol, timeframe) if jh.is_live(): self.update_position(exchange, symbol, candle) # ignore new candle at the time of execution because it messes # the count of candles without actually having an impact if candle[0] >= jh.now(): return # initial if len(arr) == 0: arr.append(candle) # if it's new, add elif candle[0] > arr[-1][0]: # in paper mode, check to see if the new candle causes any active orders to be executed if with_execution and jh.is_paper_trading(): self.simulate_order_execution(exchange, symbol, timeframe, candle) arr.append(candle) # generate other timeframes if with_generation and timeframe == '1m': self.generate_bigger_timeframes(candle, exchange, symbol, with_execution) # if it's the last candle again, update elif candle[0] == arr[-1][0]: # in paper mode, check to see if the new candle causes any active orders to get executed if with_execution and jh.is_paper_trading(): self.simulate_order_execution(exchange, symbol, timeframe, candle) arr[-1] = candle # regenerate other timeframes if with_generation and timeframe == '1m': self.generate_bigger_timeframes(candle, exchange, symbol, with_execution) # past candles will be ignored (dropped) elif candle[0] < arr[-1][0]: return
def test_cancel_order(): order = Order({ 'id': jh.generate_unique_id(), 'symbol': 'BTCUSD', 'type': order_types.LIMIT, 'price': 129.33, 'qty': 10.2041, 'side': sides.BUY, 'status': order_statuses.ACTIVE, 'created_at': jh.now(), }) assert order.is_canceled is False order.cancel() assert order.is_canceled is True assert order.canceled_at == jh.now()
def add_ticker(self, ticker: np.ndarray, exchange: str, symbol: str): key = jh.key(exchange, symbol) # only process once per second if len(self.storage[key][:]) == 0 or jh.now() - self.storage[key][-1][0] >= 1000: self.storage[key].append(ticker) if jh.is_collecting_data(): store_ticker_into_db(exchange, symbol, ticker) return
def test_can_add_new_ticker(): set_up() np.testing.assert_equal(store.tickers.get_tickers('Sandbox', 'BTCUSD'), np.zeros((0, 5))) # add first ticker t1 = np.array([jh.now(), 1, 2, 3, 4], dtype=np.float64) store.tickers.add_ticker(t1, 'Sandbox', 'BTCUSD') np.testing.assert_equal( store.tickers.get_tickers('Sandbox', 'BTCUSD')[0], t1) # fake 1 second store.app.time += 1000 # add second ticker t2 = np.array([jh.now() + 1, 11, 22, 33, 44], dtype=np.float64) store.tickers.add_ticker(t2, 'Sandbox', 'BTCUSD') np.testing.assert_equal(store.tickers.get_tickers('Sandbox', 'BTCUSD'), np.array([t1, t2]))
def test_PNL_with_fee(): # set fee (0.20%) config['env']['exchanges']['Sandbox']['fee'] = 0.002 trade = CompletedTrade({ 'type': 'long', 'exchange': 'Sandbox', 'entry_price': 10, 'exit_price': 20, 'take_profit_at': 20, 'stop_loss_at': 5, 'qty': 1, 'orders': [], 'symbol': 'BTCUSD', 'opened_at': jh.now(), 'closed_at': jh.now() }) assert trade.fee == 0.06 assert trade.PNL == 9.94
def log_exchange_message(exchange, message): # if the type of message is not str, convert it to str if not isinstance(message, str): message = str(message) formatted_time = jh.timestamp_to_time(jh.now())[:19] message = f'[{formatted_time} - {exchange}]: ' + message if 'exchange-streams' not in LOGGERS: create_disposable_logger('exchange-streams') LOGGERS['exchange-streams'].info(message)
def livetrade(): """ :return: """ # sum up balance of all trading exchanges starting_balance = 0 current_balance = 0 for e in store.exchanges.storage: starting_balance += store.exchanges.storage[e].starting_balance current_balance += store.exchanges.storage[e].balance starting_balance = round(starting_balance, 2) current_balance = round(current_balance, 2) arr = [[ 'started/current balance', '{}/{}'.format(starting_balance, current_balance) ], ['started at', jh.get_arrow(store.app.starting_time).humanize()], ['current time', jh.timestamp_to_time(jh.now())[:19]], [ 'errors/info', '{}/{}'.format(len(store.logs.errors), len(store.logs.info)) ], ['active orders', store.orders.count_all_active_orders()], ['open positions', store.positions.count_open_positions()]] # short trades summary if len(store.completed_trades.trades): df = pd.DataFrame.from_records( [t.to_dict() for t in store.completed_trades.trades]) total = len(df) winning_trades = df.loc[df['PNL'] > 0] losing_trades = df.loc[df['PNL'] < 0] pnl = round(df['PNL'].sum(), 2) pnl_percentage = round((pnl / starting_balance) * 100, 2) arr.append([ 'total/winning/losing trades', '{}/{}/{}'.format(total, len(winning_trades), len(losing_trades)) ]) arr.append(['PNL (%)', '${} ({}%)'.format(pnl, pnl_percentage)]) if config['app']['debug_mode']: arr.append(['debug mode', config['app']['debug_mode']]) if config['app']['is_test_driving']: arr.append(['Test Drive', config['app']['is_test_driving']]) return arr
def test_execute_order(): set_up_without_fee() order = Order({ 'id': jh.generate_unique_id(), 'symbol': 'BTCUSDT', 'exchange': exchange.name, 'type': order_types.LIMIT, 'price': 129.33, 'qty': 10.2041, 'side': sides.BUY, 'status': order_statuses.ACTIVE, 'created_at': jh.now(), }) assert order.is_executed is False assert order.executed_at is None order.execute() assert order.is_executed is True assert order.executed_at == jh.now()
def log_optimize_mode(message): # if the type of message is not str, convert it to str if not isinstance(message, str): message = str(message) formatted_time = jh.timestamp_to_time(jh.now())[:19] message = f'[{formatted_time}]: ' + message file_name = 'optimize-mode' if file_name not in LOGGERS: create_logger_file(file_name) LOGGERS[file_name].info(message)
def get_candles(exchange: str, symbol: str, timeframe: str): from jesse.services.db import database database.open_connection() from jesse.services.candle import generate_candle_from_one_minutes from jesse.models.utils import fetch_candles_from_db symbol = symbol.upper() num_candles = 210 one_min_count = jh.timeframe_to_one_minutes(timeframe) finish_date = jh.now(force_fresh=True) start_date = finish_date - (num_candles * one_min_count * 60_000) # fetch 1m candles from database candles = np.array( fetch_candles_from_db(exchange, symbol, start_date, finish_date)) # if there are no candles in the database, return [] if candles.size == 0: database.close_connection() return [] # leave out first candles until the timestamp of the first candle is the beginning of the timeframe timeframe_duration = one_min_count * 60_000 while candles[0][0] % timeframe_duration != 0: candles = candles[1:] # generate bigger candles from 1m candles if timeframe != '1m': generated_candles = [] for i in range(len(candles)): if (i + 1) % one_min_count == 0: bigger_candle = generate_candle_from_one_minutes( timeframe, candles[(i - (one_min_count - 1)):(i + 1)], True) generated_candles.append(bigger_candle) candles = generated_candles database.close_connection() return [{ 'time': int(c[0] / 1000), 'open': c[1], 'close': c[2], 'high': c[3], 'low': c[4], 'volume': c[5], } for c in candles]
def update_config(client_config: dict): from jesse.services.db import database database.open_connection() from jesse.models.Option import Option # at this point there must already be one option record for "config" existing, so: o = Option.get(Option.type == 'config') o.json = json.dumps(client_config) o.updated_at = jh.now() o.save() database.close_connection()
def __init__(self, attributes=None): # id generated by Jesse for database usage self.id = '' # id generated by market, used in live-trade mode self.exchange_id = '' # some exchanges might require even further info self.vars = {} self.symbol = '' self.exchange = '' self.side = '' self.type = '' self.flag = '' self.qty = 0 self.price = 0 self.status = order_statuses.ACTIVE self.created_at = None self.executed_at = None self.canceled_at = None self.role = None if attributes is None: attributes = {} for a in attributes: setattr(self, a, attributes[a]) if self.created_at is None: self.created_at = jh.now() p = selectors.get_position(self.exchange, self.symbol) if p: if jh.is_live( ) and config['env']['notifications']['events']['submitted_orders']: self.notify_submission() if jh.is_debuggable('order_submission'): logger.info('{} order: {}, {}, {}, {}, ${}'.format( 'QUEUED' if self.is_queued else 'SUBMITTED', self.symbol, self.type, self.side, self.qty, round(self.price, 2))) p._on_opened_order(self) # handle exchange balance for ordered asset e = selectors.get_exchange(self.exchange) e.on_order_submission(self)
def positions(): """ :return: """ array = [] # headers array.append([ 'type', 'strategy', 'symbol', 'opened at', 'qty', 'entry', 'current price', 'PNL (%)' ]) for p in store.positions.storage: pos = store.positions.storage[p] if pos.pnl_percentage > 0: pnl_color = 'green' elif pos.pnl_percentage < 0: pnl_color = 'red' else: pnl_color = 'black' if pos.type == 'long': type_color = 'green' elif pos.type == 'short': type_color = 'red' else: type_color = 'black' array.append([ jh.color(pos.type, type_color), pos.strategy.name, pos.symbol, '' if pos.is_close else '{} ago'.format( jh.readable_duration((jh.now() - pos.opened_at) / 1000, 3)), pos.qty if abs(pos.qty) > 0 else None, pos.entry_price, pos.current_price, '' if pos.is_close else '{} ({}%)'.format( jh.color(str(round(pos.pnl, 2)), pnl_color), jh.color(str(round(pos.pnl_percentage, 4)), pnl_color)), ]) return array