def run(cls, stdcsr): """Returns: void This is the main program loop. """ # initialize user interface cls._curses_init(stdcsr) # Read in existing trades while not cls.stopped and cls.recover_trades() == None: Log.write('"daemon.py" run(): Recovering trades...') cls._curses_refresh(stdcsr) # logging if Config.live_trading: Log.write('"daemon.py" start(): Using live account.') else: Log.write('"daemon.py" start(): Using practice mode.') """ Main loop: 1. Gather opportunities from each strategy. 2. Decide which opportunities to execute. 3. Clear the opportunity list. """ while not cls.stopped: # refresh user interface cls._curses_refresh(stdcsr) # Let each strategy suggest an order for s in cls.strategies: new_opp = s.refresh() if new_opp == None: Log.write('daemon.py run(): {} has nothing to offer now.' .format(s.get_name())) pass else: cls.opportunities.push(new_opp) # Decide which opportunity (or opportunities) to execute Log.write('"daemon.py" run(): Picking best opportunity...') best_opp = cls.opportunities.pick() if best_opp == None: # Nothing is being suggested. pass else: # An order was suggested by a strategy, so place the order. # Don't use all the money available. SLIPPAGE_WIGGLE = 0.95 ###available_money = Broker.get_margin_available(Config.account_id) * SLIPPAGE_WIGGLE available_money = 100 # USD - testing # Get the current price of one unit. instrument_price = 0 Log.write('best opp: {}'.format(best_opp)) go_long = best_opp.order.units > 0 if go_long: instrument_price = Broker.get_ask(best_opp.order.instrument) else: instrument_price = Broker.get_bid(best_opp.order.instrument) # How much leverage available: margin_rate = Broker.get_margin_rate(best_opp.order.instrument) # TODO: A bit awkward, but overwrite the existing value that was used to # determine long/short. units = available_money units /= cls.num_strategies_with_no_positions() # save money for other strategies units /= margin_rate units = int(units) # floor if units <= 0: # verify Log.write('daemon.py run(): units <= 0') raise Exception # abort if not go_long: # negative means short units = -units best_opp.order.units = units Log.write('daemon.py run(): Executing opportunity:\n{}'.format(best_opp)) order_result = Broker.place_order(best_opp.order) # Notify the strategies. if 'orderFillTransaction' in order_result: try: opened_trade_id = order_result['orderFillTransaction']['tradeOpened']['tradeID'] best_opp.strategy.trade_opened(trade_id=opened_trade_id) except: Log.write( 'daemon.py run(): Failed to extract opened trade from order result:\n{}' .format(order_result) ) raise Exception elif 'tradesClosed' in order_result: try: for trade in order_result['orderFillTransaction']['tradesClosed']: best_opp.strategy.trade_closed(trade_id=trade['tradeID']) except: Log.write( 'daemon.py run(): Failed to extract closed trades from order result:\n{}' .format(order_result) ) raise Exception elif 'tradeReduced' in order_result: try: closed_trade_id = order_result['orderFillTransaction']['tradeReduced']['tradeID'] best_opp.strategy.trade_reduced( closed_trade_id, instrument_id=Instrument.get_id_from_name(order_result['instrument']) ) except: Log.write( 'daemon.py run(): Failed to extract reduced trades from order result:\n{}' .format(order_result) ) raise Exception else: Log.write( '"daemon.py" run(): Unrecognized order result:\n{}' .format(order_result) ) raise Exception """ Clear opportunity list. Opportunities should be considered to exist only in the moment, so there is no need to save them for later. """ cls.opportunities.clear() """ Shutdown stuff. This runs after shutdown() is called, and is the last code that runs before returning to algo.py. """ DB.shutdown() # atexit() used in db.py, but call to be safe.
class Server: """Server routes system events amongst worker components via a queue in an event handling loop. The queue is processed at the start of each minute. Event loop lifecycle: 1. A new minute begins - Tick data is parsed into 1 min bars. 2. Datahander wraps new bars and other data in Market Events. 3. Datahandler pushes Market Events into event queue. 4. Market Events are consumed by Strategy object. 5. Strategy creates a Signal event and places it in event queque. 6. Signal events consumed by Portfolio. 7. Portfolio creates Order event from Signal, places it in queue. 8. Broker executes Order events, creates Fill event post-transaction. 9. Portfolio consumes Fill event, updates values. 10. Repeat 1-9 until queue empty. 11. Strategy prepares data for the next minutes calculuations. 12. Sleep until current minute elapses.""" VERSION = "0.1" DB_URL = 'mongodb://127.0.0.1:27017/' DB_NAME = 'asset_price_master' DB_TIMEOUT_MS = 10 DIAG_DELAY = 30 # mins between diagnostics def __init__(self): self.live_trading = True # set False for backtesting. self.log_level = logging.DEBUG self.logger = self.setup_logger() # Don't connect to live data feeds if backtesting. if self.live_trading: self.exchanges = self.load_exchanges(self.logger) # Connect to database. print("Connecting to database...") self.db_client = MongoClient( self.DB_URL, serverSelectionTimeoutMS=self.DB_TIMEOUT_MS) self.db = self.db_client[self.DB_NAME] self.check_db_connection() # Event queue and producer/consumer worker classes. self.events = queue.Queue(0) self.data = Datahandler(self.exchanges, self.logger, self.db, self.db_client) self.strategy = Strategy(self.exchanges, self.logger, self.db, self.db_client) self.portfolio = Portfolio(self.logger) self.broker = Broker(self.exchanges, self.logger) # Processing performance variables. self.start_processing = None self.end_processing = None self.run() def run(self): """Core event handling loop.""" self.data.set_live_trading(self.live_trading) self.broker.set_live_trading(self.live_trading) # Check data is current and complete print("Updating price database...") self.data.run_data_diagnostics(1) # Build working datasets print("Building working datasets...") self.strategy.init_dataframes() self.logger.debug("Initialised datasets.") # Start UI self.shell = Shell(self.VERSION, self.logger, self.data, self.exchanges, self.strategy, self.portfolio, self.broker) count = 0 sleep(self.seconds_til_next_minute()) while True: if self.live_trading: # Only update data after at least one minute of new data # has been collected, datahandler and strategy ready. if count >= 1 and self.data.ready: self.start_processing = time.time() self.logger.debug("Started processing events.") # Parse and queue market data (new Market Events) self.events = self.data.update_market_data(self.events) # Data is ready, route events to worker classes self.clear_event_queue() # run diagnostics at 5 min mark to fix early null bars if (count == 5): thread = Thread( target=self.data.run_data_diagnostics(0)) thread.start() # Check data integrity every 30 mins thereafter if (count % self.DIAG_DELAY == 0): thread = Thread( target=self.data.run_data_diagnostics(0)) thread.start() # Sleep til the next minute begins sleep(self.seconds_til_next_minute()) count += 1 elif not self.live_trading: # Update data w/o delay when backtesting, don't run diagnostics self.events = self.data.update_market_data(self.events) self.clear_event_queue() def clear_event_queue(self): """Routes events to worker classes for processing.""" count = 0 while True: try: # Get events from queue event = self.events.get(False) except queue.Empty: # Log processing performance stats self.end_processing = time.time() duration = round(self.end_processing - self.start_processing, 5) self.logger.debug("Processed " + str(count) + " events in " + str(duration) + " seconds.") # Store new data now that time-critical work is complete self.data.save_new_bars_to_db() break else: if event is not None: count += 1 if event.type == "MARKET": self.strategy.parse_new_data(event) elif event.type == "SIGNAL": self.portfolio.update_signal(event) elif event.type == "ORDER": self.broker.place_order(event) elif event.type == "FILL": self.portfolio.update_fill(event) # finished all jobs in queue self.events.task_done() def setup_logger(self): """Create and configure logger""" logger = logging.getLogger() logger.setLevel(self.log_level) log_file = logging.FileHandler('log.log', 'w+') formatter = logging.Formatter( "%(asctime)s:%(levelname)s:%(module)s - %(message)s") log_file.setFormatter(formatter) logger.addHandler(log_file) # supress requests/urlib3 messages as logging.DEBUG produces messages # with every single http request. logging.getLogger("urllib3").propagate = False requests_log = logging.getLogger("requests") requests_log.addHandler(logging.NullHandler()) requests_log.propagate = False return logger def load_exchanges(self, logger): """Create and return list of all exchange objects""" exchanges = [] exchanges.append(Bitmex(logger)) self.logger.debug("Initialised exchanges.") return exchanges def seconds_til_next_minute(self): """ Return number of seconds to next minute.""" now = datetime.datetime.utcnow().second delay = 60 - now return delay def check_db_connection(self): """Raise exception if DB connection not active.""" try: time.sleep(self.DB_TIMEOUT_MS) self.db_client.server_info() self.logger.debug("Connected to " + self.DB_NAME + " at " + self.DB_URL + ".") except errors.ServerSelectionTimeoutError as e: self.logger.debug("Failed to connect to " + self.DB_NAME + " at " + self.DB_URL + ".") raise Exception()
def run(cls, stdcsr): """Returns: void This is the main program loop. """ # initialize user interface cls._curses_init(stdcsr) # Read in existing trades while not cls.stopped and cls.recover_trades() == None: Log.write('"daemon.py" run(): Recovering trades...') cls._curses_refresh(stdcsr) # logging if Config.live_trading: Log.write('"daemon.py" start(): Using live account.') else: Log.write('"daemon.py" start(): Using practice mode.') """ Main loop: 1. Gather opportunities from each strategy. 2. Decide which opportunities to execute. 3. Clear the opportunity list. """ while not cls.stopped: # refresh user interface cls._curses_refresh(stdcsr) # Let each strategy suggest an order for s in cls.strategies: new_opp = s.refresh() if new_opp == None: Log.write( 'daemon.py run(): {} has nothing to offer now.'.format( s.get_name())) pass else: cls.opportunities.push(new_opp) # Decide which opportunity (or opportunities) to execute Log.write('"daemon.py" run(): Picking best opportunity...') best_opp = cls.opportunities.pick() if best_opp == None: # Nothing is being suggested. pass else: # An order was suggested by a strategy, so place the order. # Don't use all the money available. SLIPPAGE_WIGGLE = 0.95 ###available_money = Broker.get_margin_available(Config.account_id) * SLIPPAGE_WIGGLE available_money = 100 # USD - testing # Get the current price of one unit. instrument_price = 0 Log.write('best opp: {}'.format(best_opp)) go_long = best_opp.order.units > 0 if go_long: instrument_price = Broker.get_ask( best_opp.order.instrument) else: instrument_price = Broker.get_bid( best_opp.order.instrument) # How much leverage available: margin_rate = Broker.get_margin_rate(best_opp.order.instrument) # TODO: A bit awkward, but overwrite the existing value that was used to # determine long/short. units = available_money units /= cls.num_strategies_with_no_positions( ) # save money for other strategies units /= margin_rate units = int(units) # floor if units <= 0: # verify Log.write('daemon.py run(): units <= 0') raise Exception # abort if not go_long: # negative means short units = -units best_opp.order.units = units Log.write('daemon.py run(): Executing opportunity:\n{}'.format( best_opp)) order_result = Broker.place_order(best_opp.order) # Notify the strategies. if 'orderFillTransaction' in order_result: try: opened_trade_id = order_result['orderFillTransaction'][ 'tradeOpened']['tradeID'] best_opp.strategy.trade_opened( trade_id=opened_trade_id) except: Log.write( 'daemon.py run(): Failed to extract opened trade from order result:\n{}' .format(order_result)) raise Exception elif 'tradesClosed' in order_result: try: for trade in order_result['orderFillTransaction'][ 'tradesClosed']: best_opp.strategy.trade_closed( trade_id=trade['tradeID']) except: Log.write( 'daemon.py run(): Failed to extract closed trades from order result:\n{}' .format(order_result)) raise Exception elif 'tradeReduced' in order_result: try: closed_trade_id = order_result['orderFillTransaction'][ 'tradeReduced']['tradeID'] best_opp.strategy.trade_reduced( closed_trade_id, instrument_id=Instrument.get_id_from_name( order_result['instrument'])) except: Log.write( 'daemon.py run(): Failed to extract reduced trades from order result:\n{}' .format(order_result)) raise Exception else: Log.write( '"daemon.py" run(): Unrecognized order result:\n{}'. format(order_result)) raise Exception """ Clear opportunity list. Opportunities should be considered to exist only in the moment, so there is no need to save them for later. """ cls.opportunities.clear() """ Shutdown stuff. This runs after shutdown() is called, and is the last code that runs before returning to algo.py. """ DB.shutdown() # atexit() used in db.py, but call to be safe.