class Network(object): def __init__(self, concurrency, timeout, ui=None): self._executor = ThreadPoolExecutor(concurrency) self._timeout = timeout self.session = requests.Session() self._ui = ui or logger self.futures = [] self.concurrency = concurrency def _request(self, request): prepared = self.session.prepare_request(request) try: self.session.send(prepared, timeout=self._timeout) except requests.exceptions.ReadTimeout: self._ui.warning(textwrap.dedent("""The server did not send any data in the allotted amount of time. You might want to decrease the "--n_concurrent" parameters or increase "--timeout" parameter. """)) except Exception as exc: self._ui.debug('Exception {}: {}'.format(type(exc), exc)) try: callback = request.kwargs['hooks']['response'] except AttributeError: callback = request.hooks['response'][0] response = FakeResponse(400, 'No Response') callback(response) def perform_requests(self, requests): for r in requests: while True: self.futures = [i for i in self.futures if not i.done()] if len(self.futures) < self.concurrency: self.futures.append(self._executor.submit(self._request, r)) break else: sleep(0.1) yield # wait for all batches to finish before returning while self.futures: f_len = len(self.futures) self.futures = [i for i in self.futures if not i.done()] if f_len != len(self.futures): self._ui.debug('Waiting for final requests to finish. ' 'remaining requests: {}' ''.format(len(self.futures))) sleep(0.1) def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self._executor.shutdown(wait=False)
class MyWidget(DisplayWidget): def __init__(self, parent=None): super(MyWidget, self).__init__(parent) self._executor = ThreadPoolExecutor(max_workers=4) def _button_clicked(self): future = self._executor.submit(download_data) future.add_done_callback(self._populate_textarea) def _populate_textarea(self, future): self._textarea.setPlainText(future.result())
class Arbitrer(object): def __init__(self): self.markets = [] self.observers = [] self.updated_markets = {} self.init_markets(config.markets) self.init_observers(config.observers) self.threadpool = ThreadPoolExecutor(max_workers=10) def init_markets(self, markets): self.market_names = markets for market_name in markets: try: # importlib.import_module('public_markets.' + market_name.lower()) exec('from public_markets import ' + market_name.lower()) market = eval( 'public_markets.' + market_name.lower() + '.' + market_name + '()') self.markets.append(market) except (ImportError, AttributeError) as e: print e print("%s market name is invalid: Ignored (you should check your config file)" % (market_name)) def init_observers(self, _observers): self.observer_names = _observers for observer_name in _observers: try: exec('import observers.' + observer_name.lower()) observer = eval('observers.' + observer_name.lower() + '.' + observer_name + '()') self.observers.append(observer) except (ImportError, AttributeError) as e: print("%s observer name is invalid: Ignored (you should check your config file)" % (observer_name)) def get_profit_for(self, mi, mj, kask, kbid): buy_market = self.updated_markets[kask] sell_market = self.updated_markets[kbid] if buy_market.ask(mi) >= sell_market.bid(mj): return 0, 0, 0, 0, 0 max_amount_buy = buy_market.cum_asize(mi) max_amount_sell = sell_market.cum_bsize(mj) max_amount = min(max_amount_buy, max_amount_sell, config.max_tx_volume) w_buyprice, buy_total = buy_market.wavg_ask(max_amount) w_sellprice, sell_total = sell_market.wavg_bid(max_amount) profit = sell_total * w_sellprice - buy_total * w_buyprice comm = (sell_total * w_sellprice + buy_total * w_buyprice) * (0.2 / 100) profit -= comm return profit, comm, sell_total, w_buyprice, w_sellprice def get_max_depth(self, kask, kbid, max_depth_levels=5): """ :param kask: Market name where we can supposed buy (ask is lower than nbbo bid) :param kbid: Market name where we can supposed sell (bid is higher than nbbo ask) :return: (i, j) where i = number of levels of kask market lower than nbbo bid j = number of levels of kbid market lower than nbbo ask """ buy_market = self.updated_markets[kask] # Buy at this market's ask sell_market = self.updated_markets[kbid] # Sell at this market's bid # Find all prices that we can buy at (< ref_price) ref_price = sell_market.bid() for i, ask in enumerate(buy_market.iter_asks()): if ref_price < ask or i >= max_depth_levels: break # Find all the prices we can sell at (> ref_price) ref_price = buy_market.ask() for j, bid in enumerate(sell_market.iter_bids()): if ref_price > bid or j >= max_depth_levels: break return i, j def arbitrage_depth_opportunity(self, kask, kbid): """ :param kask: Market name to buy at :param kbid: Market name to sell at :return: """ maxi, maxj = self.get_max_depth(kask, kbid) buy_market = self.updated_markets[kask] # Buy at this market's ask sell_market = self.updated_markets[kbid] # Sell at this market's bid max_trade_size = min(buy_market.cum_asize(maxi), sell_market.cum_bsize(maxj), config.max_tx_volume) w_buyprice, buy_total = buy_market.wavg_ask(max_trade_size) w_sellprice, sell_total = sell_market.wavg_bid(max_trade_size) profit = sell_total * w_sellprice - buy_total * w_buyprice comm = (sell_total * w_sellprice + buy_total * w_buyprice) * (0.2 / 100) profit -= comm return profit, comm, max_trade_size, \ self.updated_markets[kask].ask(), \ self.updated_markets[kbid].bid(), \ w_buyprice, w_sellprice def arbitrage_opportunity(self, kask, ask, kbid, bid): """ :param kask: Market name to buy at :param ask: buy price :param kbid: Market name to sell at :param bid: sell price :return: """ profit, comm, volume, buyprice, sellprice, weighted_buyprice,\ weighted_sellprice = self.arbitrage_depth_opportunity(kask, kbid) if profit < 0: return if volume == 0 or buyprice == 0: return perc2 = (1 - (volume - (profit / buyprice)) / volume) * 100 for observer in self.observers: observer.opportunity( profit, comm, volume, buyprice, kask, sellprice, kbid, perc2, weighted_buyprice, weighted_sellprice) def __get_market_depth(self, market, depths): _ = market.get_depth() depths[market.name] = market def update_depths(self): depths = {} futures = [] for market in self.markets: futures.append(self.threadpool.submit(self.__get_market_depth, market, depths)) wait(futures, timeout=20) return depths def tickers(self): for market in self.markets: logging.verbose("ticker: " + market.name + " - " + str( market.get_ticker())) def replay_history(self, directory): import os import json import pprint files = os.listdir(directory) files.sort() for f in files: depths = json.load(open(directory + '/' + f, 'r')) self.updated_markets = {} for market in self.market_names: if market in depths: self.updated_markets[market] = depths[market] self.tick() def tick(self): for observer in self.observers: observer.begin_opportunity_finder(self.updated_markets) for kmarket1 in self.updated_markets: for kmarket2 in self.updated_markets: if kmarket1 == kmarket2: # same market continue market1 = self.updated_markets[kmarket1] market2 = self.updated_markets[kmarket2] # is market1.ask < market2.bid ? if market1.is_valid() and market2.is_valid(): if market2.bid() > market1.ask(): self.arbitrage_opportunity(kmarket1, market1.ask(), kmarket2, market2.bid()) for observer in self.observers: observer.end_opportunity_finder() for observer in self.observers: observer.end_opportunity_finder() def loop(self): while True: self.updated_markets = self.update_depths() self.tickers() self.tick() time.sleep(config.refresh_rate)
class Pool(object): __metaclass__ = ABCMeta class debugger(threading.Thread): def __init__(self, pool, interval = 5): self.pool = pool self.interval = interval threading.Thread.__init__(self) def start(self): self._running = True self.startTime = time.time() self.lastTime = time.time() self.lastNumber = 0 self.numberAtStart = self.pool.processed threading.Thread.start(self) def stop(self): self._running = False def debug(self): meanSpeed = (self.pool.processed - self.numberAtStart) / (time.time() - self.startTime) instantSpeed = (self.pool.processed - self.lastNumber) / (time.time() - self.lastTime) print "%s Threads: %s Remaining: %s Speed: %s / %s Done: %s" % ( ("["+self.pool.name+"]").ljust(15), str(self.pool.maxWorkers).ljust(4), str(self.pool.getQueueSize()).ljust(3), ("%.2f" % instantSpeed).ljust(9), ("%.2f" % meanSpeed).ljust(9), str(self.pool.processed) ) self.lastTime = time.time() self.lastNumber = self.pool.processed def run(self): while(self._running): self.debug() time.sleep(self.interval) def __init__(self, maxWorkers, queueSize): self.maxWorkers = maxWorkers self._pool = ThreadPoolExecutor(max_workers=maxWorkers) self._pool._work_queue.maxsize = queueSize #self._pool = ProcessPoolExecutor(max_workers=20) #self._pool._work_ids.maxsize = 2 self.processed = 0 self.debugger = self.__class__.debugger(self) self.debugger.start() def getQueueSize(self): return self._pool._work_queue.qsize() #return self._pool._work_ids.qsize()*self.maxWorkers @property def name(self): return self.__class__.__name__ def submit(self, task, *args, **kwargs): def handleSubmit(): try: result = task(*args, **kwargs) except Exception as e: self.handleError(task, e) else: self.agregate(task, result) self.processed += 1 self._pool.submit(handleSubmit) def waitAndShutdown(self): self._pool.shutdown(wait = True) self.debugger.stop() @abstractmethod def handleError(self, task, e): pass @abstractmethod def agregate(self, task, result): pass