def compile_asset_pair_market_info(): """Compiles the pair-level statistics that show on the View Prices page of counterwallet, for instance""" #loop through all open orders, and compile a listing of pairs, with a count of open orders for each pair mongo_db = config.mongo_db end_dt = datetime.datetime.utcnow() start_dt = end_dt - datetime.timedelta(days=1) start_block_index, end_block_index = util.get_block_indexes_for_dates(start_dt=start_dt, end_dt=end_dt) open_orders = util.call_jsonrpc_api("get_orders", { 'filters': [ {'field': 'give_remaining', 'op': '>', 'value': 0}, {'field': 'get_remaining', 'op': '>', 'value': 0}, {'field': 'fee_required_remaining', 'op': '>=', 'value': 0}, {'field': 'fee_provided_remaining', 'op': '>=', 'value': 0}, ], 'status': 'open', 'show_expired': False, }, abort_on_error=True)['result'] pair_data = {} asset_info = {} def get_price(base_quantity_normalized, quote_quantity_normalized): return float(D(quote_quantity_normalized / base_quantity_normalized )) #COMPOSE order depth, lowest ask, and highest bid column data for o in open_orders: (base_asset, quote_asset) = util.assets_to_asset_pair(o['give_asset'], o['get_asset']) pair = '%s/%s' % (base_asset, quote_asset) base_asset_info = asset_info.get(base_asset, mongo_db.tracked_assets.find_one({ 'asset': base_asset })) if base_asset not in asset_info: asset_info[base_asset] = base_asset_info quote_asset_info = asset_info.get(quote_asset, mongo_db.tracked_assets.find_one({ 'asset': quote_asset })) if quote_asset not in asset_info: asset_info[quote_asset] = quote_asset_info pair_data.setdefault(pair, {'open_orders_count': 0, 'lowest_ask': None, 'highest_bid': None, 'completed_trades_count': 0, 'vol_base': 0, 'vol_quote': 0}) #^ highest ask = open order selling base, highest bid = open order buying base #^ we also initialize completed_trades_count, vol_base, vol_quote because every pair inited here may # not have cooresponding data out of the trades_data_by_pair aggregation below pair_data[pair]['open_orders_count'] += 1 base_quantity_normalized = util_bitcoin.normalize_quantity(o['give_quantity'] if base_asset == o['give_asset'] else o['get_quantity'], base_asset_info['divisible']) quote_quantity_normalized = util_bitcoin.normalize_quantity(o['give_quantity'] if quote_asset == o['give_asset'] else o['get_quantity'], quote_asset_info['divisible']) order_price = get_price(base_quantity_normalized, quote_quantity_normalized) if base_asset == o['give_asset']: #selling base if pair_data[pair]['lowest_ask'] is None or order_price < pair_data[pair]['lowest_ask']: pair_data[pair]['lowest_ask'] = order_price elif base_asset == o['get_asset']: #buying base if pair_data[pair]['highest_bid'] is None or order_price > pair_data[pair]['highest_bid']: pair_data[pair]['highest_bid'] = order_price #COMPOSE volume data (in XCP and BTC), and % change data #loop through all trade volume over the past 24h, and match that to the open orders trades_data_by_pair = mongo_db.trades.aggregate([ {"$match": { "block_time": {"$gte": start_dt, "$lte": end_dt } } }, {"$project": { "base_asset": 1, "quote_asset": 1, "base_quantity_normalized": 1, #to derive base volume "quote_quantity_normalized": 1 #to derive quote volume }}, {"$group": { "_id": {"base_asset": "$base_asset", "quote_asset": "$quote_asset"}, "vol_base": {"$sum": "$base_quantity_normalized"}, "vol_quote": {"$sum": "$quote_quantity_normalized"}, "count": {"$sum": 1}, }} ]) trades_data_by_pair = [] if not trades_data_by_pair['ok'] else trades_data_by_pair['result'] for e in trades_data_by_pair: pair = '%s/%s' % (e['_id']['base_asset'], e['_id']['quote_asset']) pair_data.setdefault(pair, {'open_orders_count': 0, 'lowest_ask': None, 'highest_bid': None}) #^ initialize an empty pair in the event there are no open orders for that pair, but there ARE completed trades for it pair_data[pair]['completed_trades_count'] = e['count'] pair_data[pair]['vol_base'] = e['vol_base'] pair_data[pair]['vol_quote'] = e['vol_quote'] #compose price data, relative to BTC and XCP mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives() for pair, e in pair_data.iteritems(): base_asset, quote_asset = pair.split('/') _24h_vol_in_btc = None _24h_vol_in_xcp = None #derive asset price data, expressed in BTC and XCP, for the given volumes if base_asset == config.XCP: _24h_vol_in_xcp = e['vol_base'] _24h_vol_in_btc = util_bitcoin.round_out(e['vol_base'] * xcp_btc_price) if xcp_btc_price else 0 elif base_asset == config.BTC: _24h_vol_in_xcp = util_bitcoin.round_out(e['vol_base'] * btc_xcp_price) if btc_xcp_price else 0 _24h_vol_in_btc = e['vol_base'] else: #base is not XCP or BTC price_summary_in_xcp, price_summary_in_btc, price_in_xcp, price_in_btc, aggregated_price_in_xcp, aggregated_price_in_btc = \ get_xcp_btc_price_info(base_asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price, with_last_trades=0, start_dt=start_dt, end_dt=end_dt) if price_in_xcp: _24h_vol_in_xcp = util_bitcoin.round_out(e['vol_base'] * price_in_xcp) if price_in_btc: _24h_vol_in_btc = util_bitcoin.round_out(e['vol_base'] * price_in_btc) if _24h_vol_in_xcp is None or _24h_vol_in_btc is None: #the base asset didn't have price data against BTC or XCP, or both...try against the quote asset instead price_summary_in_xcp, price_summary_in_btc, price_in_xcp, price_in_btc, aggregated_price_in_xcp, aggregated_price_in_btc = \ get_xcp_btc_price_info(quote_asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price, with_last_trades=0, start_dt=start_dt, end_dt=end_dt) if _24h_vol_in_xcp is None and price_in_xcp: _24h_vol_in_xcp = util_bitcoin.round_out(e['vol_quote'] * price_in_xcp) if _24h_vol_in_btc is None and price_in_btc: _24h_vol_in_btc = util_bitcoin.round_out(e['vol_quote'] * price_in_btc) pair_data[pair]['24h_vol_in_{}'.format(config.XCP.lower())] = _24h_vol_in_xcp #might still be None pair_data[pair]['24h_vol_in_{}'.format(config.BTC.lower())] = _24h_vol_in_btc #might still be None #get % change stats -- start by getting the first trade directly before the 24h period starts prev_trade = mongo_db.trades.find({ "base_asset": base_asset, "quote_asset": quote_asset, "block_time": {'$lt': start_dt}}).sort('block_time', pymongo.DESCENDING).limit(1) latest_trade = mongo_db.trades.find({ "base_asset": base_asset, "quote_asset": quote_asset}).sort('block_time', pymongo.DESCENDING).limit(1) if not prev_trade.count(): #no previous trade before this 24hr period pair_data[pair]['24h_pct_change'] = None else: prev_trade = prev_trade[0] latest_trade = latest_trade[0] prev_trade_price = get_price(prev_trade['base_quantity_normalized'], prev_trade['quote_quantity_normalized']) latest_trade_price = get_price(latest_trade['base_quantity_normalized'], latest_trade['quote_quantity_normalized']) pair_data[pair]['24h_pct_change'] = ((latest_trade_price - prev_trade_price) / prev_trade_price) * 100 pair_data[pair]['last_updated'] = end_dt #print "PRODUCED", pair, pair_data[pair] mongo_db.asset_pair_market_info.update( {'base_asset': base_asset, 'quote_asset': quote_asset}, {"$set": pair_data[pair]}, upsert=True) #remove any old pairs that were not just updated mongo_db.asset_pair_market_info.remove({'last_updated': {'$lt': end_dt}}) logging.info("Recomposed 24h trade statistics for %i asset pairs: %s" % (len(pair_data), ', '.join(pair_data.keys())))
def compile_asset_pair_market_info(): """Compiles the pair-level statistics that show on the View Prices page of counterwallet, for instance""" #loop through all open orders, and compile a listing of pairs, with a count of open orders for each pair mongo_db = config.mongo_db end_dt = datetime.datetime.utcnow() start_dt = end_dt - datetime.timedelta(days=1) start_block_index, end_block_index = util.get_block_indexes_for_dates( start_dt=start_dt, end_dt=end_dt) open_orders = util.call_jsonrpc_api("get_orders", { 'filters': [ { 'field': 'give_remaining', 'op': '>', 'value': 0 }, { 'field': 'get_remaining', 'op': '>', 'value': 0 }, { 'field': 'fee_required_remaining', 'op': '>=', 'value': 0 }, { 'field': 'fee_provided_remaining', 'op': '>=', 'value': 0 }, ], 'status': 'open', 'show_expired': False, }, abort_on_error=True)['result'] pair_data = {} asset_info = {} def get_price(base_quantity_normalized, quote_quantity_normalized): return float(D(quote_quantity_normalized / base_quantity_normalized)) #COMPOSE order depth, lowest ask, and highest bid column data for o in open_orders: (base_asset, quote_asset) = util.assets_to_asset_pair(o['give_asset'], o['get_asset']) pair = '%s/%s' % (base_asset, quote_asset) base_asset_info = asset_info.get( base_asset, mongo_db.tracked_assets.find_one({'asset': base_asset})) if base_asset not in asset_info: asset_info[base_asset] = base_asset_info quote_asset_info = asset_info.get( quote_asset, mongo_db.tracked_assets.find_one({'asset': quote_asset})) if quote_asset not in asset_info: asset_info[quote_asset] = quote_asset_info pair_data.setdefault( pair, { 'open_orders_count': 0, 'lowest_ask': None, 'highest_bid': None, 'completed_trades_count': 0, 'vol_base': 0, 'vol_quote': 0 }) #^ highest ask = open order selling base, highest bid = open order buying base #^ we also initialize completed_trades_count, vol_base, vol_quote because every pair inited here may # not have cooresponding data out of the trades_data_by_pair aggregation below pair_data[pair]['open_orders_count'] += 1 base_quantity_normalized = util_bitcoin.normalize_quantity( o['give_quantity'] if base_asset == o['give_asset'] else o['get_quantity'], base_asset_info['divisible']) quote_quantity_normalized = util_bitcoin.normalize_quantity( o['give_quantity'] if quote_asset == o['give_asset'] else o['get_quantity'], quote_asset_info['divisible']) order_price = get_price(base_quantity_normalized, quote_quantity_normalized) if base_asset == o['give_asset']: #selling base if pair_data[pair]['lowest_ask'] is None or order_price < pair_data[ pair]['lowest_ask']: pair_data[pair]['lowest_ask'] = order_price elif base_asset == o['get_asset']: #buying base if pair_data[pair][ 'highest_bid'] is None or order_price > pair_data[pair][ 'highest_bid']: pair_data[pair]['highest_bid'] = order_price #COMPOSE volume data (in XDP and DOGE), and % change data #loop through all trade volume over the past 24h, and match that to the open orders trades_data_by_pair = mongo_db.trades.aggregate([ { "$match": { "block_time": { "$gte": start_dt, "$lte": end_dt } } }, { "$project": { "base_asset": 1, "quote_asset": 1, "base_quantity_normalized": 1, #to derive base volume "quote_quantity_normalized": 1 #to derive quote volume } }, { "$group": { "_id": { "base_asset": "$base_asset", "quote_asset": "$quote_asset" }, "vol_base": { "$sum": "$base_quantity_normalized" }, "vol_quote": { "$sum": "$quote_quantity_normalized" }, "count": { "$sum": 1 }, } } ]) trades_data_by_pair = [] if not trades_data_by_pair[ 'ok'] else trades_data_by_pair['result'] for e in trades_data_by_pair: pair = '%s/%s' % (e['_id']['base_asset'], e['_id']['quote_asset']) pair_data.setdefault(pair, { 'open_orders_count': 0, 'lowest_ask': None, 'highest_bid': None }) #^ initialize an empty pair in the event there are no open orders for that pair, but there ARE completed trades for it pair_data[pair]['completed_trades_count'] = e['count'] pair_data[pair]['vol_base'] = e['vol_base'] pair_data[pair]['vol_quote'] = e['vol_quote'] #compose price data, relative to DOGE and XDP mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives() for pair, e in pair_data.iteritems(): base_asset, quote_asset = pair.split('/') _24h_vol_in_btc = None _24h_vol_in_xcp = None #derive asset price data, expressed in DOGE and XDP, for the given volumes if base_asset == config.XCP: _24h_vol_in_xcp = e['vol_base'] _24h_vol_in_btc = util_bitcoin.round_out( e['vol_base'] * xcp_btc_price) if xcp_btc_price else 0 elif base_asset == config.BTC: _24h_vol_in_xcp = util_bitcoin.round_out( e['vol_base'] * btc_xcp_price) if btc_xcp_price else 0 _24h_vol_in_btc = e['vol_base'] else: #base is not XDP or DOGE price_summary_in_xcp, price_summary_in_btc, price_in_xcp, price_in_btc, aggregated_price_in_xcp, aggregated_price_in_btc = \ get_xcp_btc_price_info(base_asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price, with_last_trades=0, start_dt=start_dt, end_dt=end_dt) if price_in_xcp: _24h_vol_in_xcp = util_bitcoin.round_out(e['vol_base'] * price_in_xcp) if price_in_btc: _24h_vol_in_btc = util_bitcoin.round_out(e['vol_base'] * price_in_btc) if _24h_vol_in_xcp is None or _24h_vol_in_btc is None: #the base asset didn't have price data against DOGE or XDP, or both...try against the quote asset instead price_summary_in_xcp, price_summary_in_btc, price_in_xcp, price_in_btc, aggregated_price_in_xcp, aggregated_price_in_btc = \ get_xcp_btc_price_info(quote_asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price, with_last_trades=0, start_dt=start_dt, end_dt=end_dt) if _24h_vol_in_xcp is None and price_in_xcp: _24h_vol_in_xcp = util_bitcoin.round_out(e['vol_quote'] * price_in_xcp) if _24h_vol_in_btc is None and price_in_btc: _24h_vol_in_btc = util_bitcoin.round_out(e['vol_quote'] * price_in_btc) pair_data[pair]['24h_vol_in_{}'.format( config.XCP.lower())] = _24h_vol_in_xcp #might still be None pair_data[pair]['24h_vol_in_{}'.format( config.BTC.lower())] = _24h_vol_in_btc #might still be None #get % change stats -- start by getting the first trade directly before the 24h period starts prev_trade = mongo_db.trades.find({ "base_asset": base_asset, "quote_asset": quote_asset, "block_time": { '$lt': start_dt } }).sort('block_time', pymongo.DESCENDING).limit(1) latest_trade = mongo_db.trades.find({ "base_asset": base_asset, "quote_asset": quote_asset }).sort('block_time', pymongo.DESCENDING).limit(1) if not prev_trade.count(): #no previous trade before this 24hr period pair_data[pair]['24h_pct_change'] = None else: prev_trade = prev_trade[0] latest_trade = latest_trade[0] prev_trade_price = get_price( prev_trade['base_quantity_normalized'], prev_trade['quote_quantity_normalized']) latest_trade_price = get_price( latest_trade['base_quantity_normalized'], latest_trade['quote_quantity_normalized']) pair_data[pair]['24h_pct_change'] = ( (latest_trade_price - prev_trade_price) / prev_trade_price) * 100 pair_data[pair]['last_updated'] = end_dt #print "PRODUCED", pair, pair_data[pair] mongo_db.asset_pair_market_info.update( { 'base_asset': base_asset, 'quote_asset': quote_asset }, {"$set": pair_data[pair]}, upsert=True) #remove any old pairs that were not just updated mongo_db.asset_pair_market_info.remove({'last_updated': {'$lt': end_dt}}) logging.info("Recomposed 24h trade statistics for %i asset pairs: %s" % (len(pair_data), ', '.join(pair_data.keys())))