def decorate_message_for_feed(msg, msg_data=None): """This function takes a message from counterpartyd's message feed and mutates it a bit to be suitable to be sent through the counterblockd message feed to an end-client""" if not msg_data: msg_data = json.loads(msg['bindings']) message = copy.deepcopy(msg_data) message['_message_index'] = msg['message_index'] message['_command'] = msg['command'] message['_block_index'] = msg['block_index'] message['_block_time'] = database.get_block_time(msg['block_index']) message['_category'] = msg['category'] message['_status'] = msg_data.get('status', 'valid') message = decorate_message(message) return message
def decorate_message(message, for_txn_history=False): # insert custom fields in certain events... # even invalid actions need these extra fields for proper reporting to the client (as the reporting message # is produced via PendingActionViewModel.calcText) -- however make it able to deal with the queried data not existing in this case assert '_category' in message if for_txn_history: message['_command'] = 'insert' # history data doesn't include this block_index = message['block_index'] if 'block_index' in message else message['tx1_block_index'] message['_block_time'] = database.get_block_time(block_index) message['_tx_index'] = message['tx_index'] if 'tx_index' in message else message.get('tx1_index', None) if message['_category'] in ['bet_expirations', 'order_expirations', 'bet_match_expirations', 'order_match_expirations']: message['_tx_index'] = 0 # add tx_index to all entries (so we can sort on it secondarily in history view), since these lack it if message['_category'] in ['credits', 'debits']: # find the last balance change on record bal_change = config.mongo_db.balance_changes.find_one( {'address': message['address'], 'asset': message['asset']}, sort=[("block_time", pymongo.DESCENDING)]) message['_quantity_normalized'] = abs(bal_change['quantity_normalized']) if bal_change else None message['_balance'] = bal_change['new_balance'] if bal_change else None message['_balance_normalized'] = bal_change['new_balance_normalized'] if bal_change else None if message['_category'] in ['orders', ] and message['_command'] == 'insert': get_asset_info = config.mongo_db.tracked_assets.find_one({'asset': message['get_asset']}) give_asset_info = config.mongo_db.tracked_assets.find_one({'asset': message['give_asset']}) message['_get_asset_divisible'] = get_asset_info['divisible'] if get_asset_info else None message['_give_asset_divisible'] = give_asset_info['divisible'] if give_asset_info else None if message['_category'] in ['order_matches', ] and message['_command'] == 'insert': forward_asset_info = config.mongo_db.tracked_assets.find_one({'asset': message['forward_asset']}) backward_asset_info = config.mongo_db.tracked_assets.find_one({'asset': message['backward_asset']}) message['_forward_asset_divisible'] = forward_asset_info['divisible'] if forward_asset_info else None message['_backward_asset_divisible'] = backward_asset_info['divisible'] if backward_asset_info else None if message['_category'] in ['orders', 'order_matches', ]: message['_btc_below_dust_limit'] = ( ('forward_asset' in message and message['forward_asset'] == config.BTC and message['forward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF) or ('backward_asset' in message and message['backward_asset'] == config.BTC and message['backward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF) ) if message['_category'] in ['dividends', 'sends', ]: asset_info = config.mongo_db.tracked_assets.find_one({'asset': message['asset']}) message['_divisible'] = asset_info['divisible'] if asset_info else None if message['_category'] in ['issuances', ]: message['_quantity_normalized'] = blockchain.normalize_quantity(message['quantity'], message['divisible']) return message
def decorate_message(message, for_txn_history=False): # insert custom fields in certain events... # even invalid actions need these extra fields for proper reporting to the client (as the reporting message # is produced via PendingActionViewModel.calcText) -- however make it able to deal with the queried data not existing in this case assert '_category' in message if for_txn_history: message['_command'] = 'insert' # history data doesn't include this block_index = message[ 'block_index'] if 'block_index' in message else message[ 'tx1_block_index'] message['_block_time'] = database.get_block_time(block_index) message['_tx_index'] = message[ 'tx_index'] if 'tx_index' in message else message.get( 'tx1_index', None) if message['_category'] in [ 'bet_expirations', 'order_expirations', 'bet_match_expirations', 'order_match_expirations' ]: message[ '_tx_index'] = 0 # add tx_index to all entries (so we can sort on it secondarily in history view), since these lack it if message['_category'] in ['credits', 'debits']: # find the last balance change on record bal_change = config.mongo_db.balance_changes.find_one( { 'address': message['address'], 'asset': message['asset'] }, sort=[("block_time", pymongo.DESCENDING)]) message['_quantity_normalized'] = abs( bal_change['quantity_normalized']) if bal_change else None message['_balance'] = bal_change['new_balance'] if bal_change else None message['_balance_normalized'] = bal_change[ 'new_balance_normalized'] if bal_change else None if message['_category'] in [ 'orders', ] and message['_command'] == 'insert': get_asset_info = config.mongo_db.tracked_assets.find_one( {'asset': message['get_asset']}) give_asset_info = config.mongo_db.tracked_assets.find_one( {'asset': message['give_asset']}) message['_get_asset_divisible'] = get_asset_info[ 'divisible'] if get_asset_info else None message['_give_asset_divisible'] = give_asset_info[ 'divisible'] if give_asset_info else None if message['_category'] in [ 'order_matches', ] and message['_command'] == 'insert': forward_asset_info = config.mongo_db.tracked_assets.find_one( {'asset': message['forward_asset']}) backward_asset_info = config.mongo_db.tracked_assets.find_one( {'asset': message['backward_asset']}) message['_forward_asset_divisible'] = forward_asset_info[ 'divisible'] if forward_asset_info else None message['_backward_asset_divisible'] = backward_asset_info[ 'divisible'] if backward_asset_info else None if message['_category'] in [ 'orders', 'order_matches', ]: message['_btc_below_dust_limit'] = ( ('forward_asset' in message and message['forward_asset'] == config.BTC and message['forward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF) or ('backward_asset' in message and message['backward_asset'] == config.BTC and message['backward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF)) if message['_category'] in [ 'dividends', 'sends', ]: asset_info = config.mongo_db.tracked_assets.find_one( {'asset': message['asset']}) message['_divisible'] = asset_info['divisible'] if asset_info else None if message['_category'] in [ 'issuances', ]: message['_quantity_normalized'] = blockchain.normalize_quantity( message['quantity'], message['divisible']) return message
def compile_asset_market_info(): """Run through all assets and compose and store market ranking information.""" mongo_db = config.mongo_db if not config.state['caught_up']: logger.warn( "Not updating asset market info as counterblockd is not caught up." ) return False #grab the last block # we processed assets data off of last_block_assets_compiled = mongo_db.app_config.find_one( )['last_block_assets_compiled'] last_block_time_assets_compiled = database.get_block_time( last_block_assets_compiled) #logger.debug("Comping info for assets traded since block %i" % last_block_assets_compiled) current_block_index = config.state['my_latest_block'][ 'block_index'] #store now as it may change as we are compiling asset data :) current_block_time = database.get_block_time(current_block_index) if current_block_index == last_block_assets_compiled: #all caught up -- call again in 10 minutes return True mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives() all_traded_assets = list( set( list([config.BTC, config.XCP]) + list( mongo_db.trades.find({}, { 'quote_asset': 1, '_id': 0 }).distinct('quote_asset')))) ####################### #get a list of all assets with a trade within the last 24h (not necessarily just against XCP and BTC) # ^ this is important because compiled market info has a 24h vol parameter that designates total volume for the asset across ALL pairings start_dt_1d = datetime.datetime.utcnow() - datetime.timedelta(days=1) assets = list( set( list( mongo_db.trades.find({ 'block_time': { '$gte': start_dt_1d } }).distinct('quote_asset')) + list( mongo_db.trades.find({ 'block_time': { '$gte': start_dt_1d } }).distinct('base_asset')))) for asset in assets: market_info_24h = compile_24h_market_info(asset) mongo_db.asset_market_info.update({'asset': asset}, {"$set": market_info_24h}) #for all others (i.e. no trade in the last 24 hours), zero out the 24h trade data non_traded_assets = list(set(all_traded_assets) - set(assets)) mongo_db.asset_market_info.update({'asset': { '$in': non_traded_assets }}, { "$set": { '24h_summary': { 'vol': 0, 'count': 0 }, '24h_ohlc_in_{}'.format(config.XCP.lower()): {}, '24h_ohlc_in_{}'.format(config.BTC.lower()): {}, '24h_vol_price_change_in_{}'.format(config.XCP.lower()): None, '24h_vol_price_change_in_{}'.format(config.BTC.lower()): None, } }, multi=True) logger.info("Block: %s -- Calculated 24h stats for: %s" % (current_block_index, ', '.join(assets))) ####################### #get a list of all assets with a trade within the last 7d up against XCP and BTC start_dt_7d = datetime.datetime.utcnow() - datetime.timedelta(days=7) assets = list( set( list( mongo_db.trades.find({ 'block_time': { '$gte': start_dt_7d }, 'base_asset': { '$in': [config.XCP, config.BTC] } }).distinct('quote_asset')) + list( mongo_db.trades.find({ 'block_time': { '$gte': start_dt_7d } }).distinct('base_asset')))) for asset in assets: market_info_7d = compile_7d_market_info(asset) mongo_db.asset_market_info.update({'asset': asset}, {"$set": market_info_7d}) non_traded_assets = list(set(all_traded_assets) - set(assets)) mongo_db.asset_market_info.update({'asset': { '$in': non_traded_assets }}, { "$set": { '7d_history_in_{}'.format(config.XCP.lower()): [], '7d_history_in_{}'.format(config.BTC.lower()): [], } }, multi=True) logger.info("Block: %s -- Calculated 7d stats for: %s" % (current_block_index, ', '.join(assets))) ####################### #update summary market data for assets traded since last_block_assets_compiled #get assets that were traded since the last check with either BTC or XCP, and update their market summary data assets = list( set( list( mongo_db.trades.find({ 'block_index': { '$gt': last_block_assets_compiled }, 'base_asset': { '$in': [config.XCP, config.BTC] } }).distinct('quote_asset')) + list( mongo_db.trades.find({ 'block_index': { '$gt': last_block_assets_compiled } }).distinct('base_asset')))) #update our storage of the latest market info in mongo for asset in assets: logger.info("Block: %s -- Updating asset market info for %s ..." % (current_block_index, asset)) summary_info = compile_summary_market_info(asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price) mongo_db.asset_market_info.update({'asset': asset}, {"$set": summary_info}, upsert=True) ####################### #next, compile market cap historicals (and get the market price data that we can use to update assets with new trades) #NOTE: this algoritm still needs to be fleshed out some...I'm not convinced it's laid out/optimized like it should be #start by getting all trades from when we last compiled this data trades = mongo_db.trades.find({ 'block_index': { '$gt': last_block_assets_compiled } }).sort('block_index', pymongo.ASCENDING) trades_by_block = [ ] #tracks assets compiled per block, as we only want to analyze any given asset once per block trades_by_block_mapping = {} #organize trades by block for t in trades: if t['block_index'] in trades_by_block_mapping: assert trades_by_block_mapping[ t['block_index']]['block_index'] == t['block_index'] assert trades_by_block_mapping[ t['block_index']]['block_time'] == t['block_time'] trades_by_block_mapping[t['block_index']]['trades'].append(t) else: e = { 'block_index': t['block_index'], 'block_time': t['block_time'], 'trades': [ t, ] } trades_by_block.append(e) trades_by_block_mapping[t['block_index']] = e for t_block in trades_by_block: #reverse the tradelist per block, and ensure that we only process an asset that hasn't already been processed for this block # (as there could be multiple trades in a single block for any specific asset). we reverse the list because # we'd rather process a later trade for a given asset, as the market price for that will take into account # the earlier trades on that same block for that asset, and we don't want/need multiple cap points per block assets_in_block = {} mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives( end_dt=t_block['block_time']) for t in reversed(t_block['trades']): assets = [] if t['base_asset'] not in assets_in_block: assets.append(t['base_asset']) assets_in_block[t['base_asset']] = True if t['quote_asset'] not in assets_in_block: assets.append(t['quote_asset']) assets_in_block[t['quote_asset']] = True if not len(assets): continue for asset in assets: #recalculate the market cap for the asset this trade is for asset_info = get_asset_info(asset, at_dt=t['block_time']) (price_summary_in_xcp, price_summary_in_btc, price_in_xcp, price_in_btc, aggregated_price_in_xcp, aggregated_price_in_btc) = get_xcp_btc_price_info( asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price, with_last_trades=0, end_dt=t['block_time']) market_cap_in_xcp, market_cap_in_btc = calc_market_cap( asset_info, price_in_xcp, price_in_btc) #^ this will get price data from the block time of this trade back the standard number of days and trades # to determine our standard market price, relative (anchored) to the time of this trade for market_cap_as in (config.XCP, config.BTC): market_cap = market_cap_in_xcp if market_cap_as == config.XCP else market_cap_in_btc #if there is a previously stored market cap for this asset, add a new history point only if the two caps differ prev_market_cap_history = mongo_db.asset_marketcap_history.find( { 'market_cap_as': market_cap_as, 'asset': asset, 'block_index': { '$lt': t['block_index'] } }).sort('block_index', pymongo.DESCENDING).limit(1) prev_market_cap_history = list( prev_market_cap_history )[0] if prev_market_cap_history.count() == 1 else None if market_cap and (not prev_market_cap_history or prev_market_cap_history['market_cap'] != market_cap): mongo_db.asset_marketcap_history.insert({ 'block_index': t['block_index'], 'block_time': t['block_time'], 'asset': asset, 'market_cap': market_cap, 'market_cap_as': market_cap_as, }) logger.info( "Block %i -- Calculated market cap history point for %s as %s (mID: %s)" % (t['block_index'], asset, market_cap_as, t['message_index'])) mongo_db.app_config.update( {}, {'$set': { 'last_block_assets_compiled': current_block_index }}) return True
def compile_asset_market_info(): """Run through all assets and compose and store market ranking information.""" if not config.state['caught_up']: logger.warn("Not updating asset market info as counterblockd is not caught up.") return False #grab the last block # we processed assets data off of last_block_assets_compiled = config.mongo_db.app_config.find_one()['last_block_assets_compiled'] last_block_time_assets_compiled = database.get_block_time(last_block_assets_compiled) #logger.debug("Comping info for assets traded since block %i" % last_block_assets_compiled) current_block_index = config.state['my_latest_block']['block_index'] #store now as it may change as we are compiling asset data :) current_block_time = database.get_block_time(current_block_index) if current_block_index == last_block_assets_compiled: #all caught up -- call again in 10 minutes return True mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives() all_traded_assets = list(set(list([config.BTC, config.XCP]) + list(config.mongo_db.trades.find({}, {'quote_asset': 1, '_id': 0}).distinct('quote_asset')))) ####################### #get a list of all assets with a trade within the last 24h (not necessarily just against XCP and BTC) # ^ this is important because compiled market info has a 24h vol parameter that designates total volume for the asset across ALL pairings start_dt_1d = datetime.datetime.utcnow() - datetime.timedelta(days=1) assets = list(set( list(config.mongo_db.trades.find({'block_time': {'$gte': start_dt_1d}}).distinct('quote_asset')) + list(config.mongo_db.trades.find({'block_time': {'$gte': start_dt_1d}}).distinct('base_asset')) )) for asset in assets: market_info_24h = compile_24h_market_info(asset) config.mongo_db.asset_market_info.update({'asset': asset}, {"$set": market_info_24h}) #for all others (i.e. no trade in the last 24 hours), zero out the 24h trade data non_traded_assets = list(set(all_traded_assets) - set(assets)) config.mongo_db.asset_market_info.update( {'asset': {'$in': non_traded_assets}}, {"$set": { '24h_summary': {'vol': 0, 'count': 0}, '24h_ohlc_in_{}'.format(config.XCP.lower()): {}, '24h_ohlc_in_{}'.format(config.BTC.lower()): {}, '24h_vol_price_change_in_{}'.format(config.XCP.lower()): None, '24h_vol_price_change_in_{}'.format(config.BTC.lower()): None, }}, multi=True) logger.info("Block: %s -- Calculated 24h stats for: %s" % (current_block_index, ', '.join(assets))) ####################### #get a list of all assets with a trade within the last 7d up against XCP and BTC start_dt_7d = datetime.datetime.utcnow() - datetime.timedelta(days=7) assets = list(set( list(config.mongo_db.trades.find({'block_time': {'$gte': start_dt_7d}, 'base_asset': {'$in': [config.XCP, config.BTC]}}).distinct('quote_asset')) + list(config.mongo_db.trades.find({'block_time': {'$gte': start_dt_7d}}).distinct('base_asset')) )) for asset in assets: market_info_7d = compile_7d_market_info(asset) config.mongo_db.asset_market_info.update({'asset': asset}, {"$set": market_info_7d}) non_traded_assets = list(set(all_traded_assets) - set(assets)) config.mongo_db.asset_market_info.update( {'asset': {'$in': non_traded_assets}}, {"$set": { '7d_history_in_{}'.format(config.XCP.lower()): [], '7d_history_in_{}'.format(config.BTC.lower()): [], }}, multi=True) logger.info("Block: %s -- Calculated 7d stats for: %s" % (current_block_index, ', '.join(assets))) ####################### #update summary market data for assets traded since last_block_assets_compiled #get assets that were traded since the last check with either BTC or XCP, and update their market summary data assets = list(set( list(config.mongo_db.trades.find({'block_index': {'$gt': last_block_assets_compiled}, 'base_asset': {'$in': [config.XCP, config.BTC]}}).distinct('quote_asset')) + list(config.mongo_db.trades.find({'block_index': {'$gt': last_block_assets_compiled}}).distinct('base_asset')) )) #update our storage of the latest market info in mongo for asset in assets: logger.info("Block: %s -- Updating asset market info for %s ..." % (current_block_index, asset)) summary_info = compile_summary_market_info(asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price) config.mongo_db.asset_market_info.update( {'asset': asset}, {"$set": summary_info}, upsert=True) ####################### #next, compile market cap historicals (and get the market price data that we can use to update assets with new trades) #NOTE: this algoritm still needs to be fleshed out some...I'm not convinced it's laid out/optimized like it should be #start by getting all trades from when we last compiled this data trades = config.mongo_db.trades.find({'block_index': {'$gt': last_block_assets_compiled}}).sort('block_index', pymongo.ASCENDING) trades_by_block = [] #tracks assets compiled per block, as we only want to analyze any given asset once per block trades_by_block_mapping = {} #organize trades by block for t in trades: if t['block_index'] in trades_by_block_mapping: assert trades_by_block_mapping[t['block_index']]['block_index'] == t['block_index'] assert trades_by_block_mapping[t['block_index']]['block_time'] == t['block_time'] trades_by_block_mapping[t['block_index']]['trades'].append(t) else: e = {'block_index': t['block_index'], 'block_time': t['block_time'], 'trades': [t,]} trades_by_block.append(e) trades_by_block_mapping[t['block_index']] = e for t_block in trades_by_block: #reverse the tradelist per block, and ensure that we only process an asset that hasn't already been processed for this block # (as there could be multiple trades in a single block for any specific asset). we reverse the list because # we'd rather process a later trade for a given asset, as the market price for that will take into account # the earlier trades on that same block for that asset, and we don't want/need multiple cap points per block assets_in_block = {} mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives(end_dt=t_block['block_time']) for t in reversed(t_block['trades']): assets = [] if t['base_asset'] not in assets_in_block: assets.append(t['base_asset']) assets_in_block[t['base_asset']] = True if t['quote_asset'] not in assets_in_block: assets.append(t['quote_asset']) assets_in_block[t['quote_asset']] = True if not len(assets): continue for asset in assets: #recalculate the market cap for the asset this trade is for asset_info = get_asset_info(asset, at_dt=t['block_time']) (price_summary_in_xcp, price_summary_in_btc, price_in_xcp, price_in_btc, aggregated_price_in_xcp, aggregated_price_in_btc ) = get_xcp_btc_price_info(asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price, with_last_trades=0, end_dt=t['block_time']) market_cap_in_xcp, market_cap_in_btc = calc_market_cap(asset_info, price_in_xcp, price_in_btc) #^ this will get price data from the block time of this trade back the standard number of days and trades # to determine our standard market price, relative (anchored) to the time of this trade for market_cap_as in (config.XCP, config.BTC): market_cap = market_cap_in_xcp if market_cap_as == config.XCP else market_cap_in_btc #if there is a previously stored market cap for this asset, add a new history point only if the two caps differ prev_market_cap_history = config.mongo_db.asset_marketcap_history.find({'market_cap_as': market_cap_as, 'asset': asset, 'block_index': {'$lt': t['block_index']}}).sort('block_index', pymongo.DESCENDING).limit(1) prev_market_cap_history = list(prev_market_cap_history)[0] if prev_market_cap_history.count() == 1 else None if market_cap and (not prev_market_cap_history or prev_market_cap_history['market_cap'] != market_cap): config.mongo_db.asset_marketcap_history.insert({ 'block_index': t['block_index'], 'block_time': t['block_time'], 'asset': asset, 'market_cap': market_cap, 'market_cap_as': market_cap_as, }) logger.info("Block %i -- Calculated market cap history point for %s as %s (mID: %s)" % (t['block_index'], asset, market_cap_as, t['message_index'])) config.mongo_db.app_config.update({}, {'$set': {'last_block_assets_compiled': current_block_index}}) return True