def decorate_message_for_feed(msg, msg_data=None):
"""This function takes a message from counterpartyd's message feed and mutates it a bit to be suitable to be
sent through the counterblockd message feed to an end-client"""
if not msg_data:
msg_data = json.loads(msg['bindings'])
message = copy.deepcopy(msg_data)
message['_message_index'] = msg['message_index']
message['_command'] = msg['command']
message['_block_index'] = msg['block_index']
message['_block_time'] = database.get_block_time(msg['block_index'])
message['_category'] = msg['category']
message['_status'] = msg_data.get('status', 'valid')
message = decorate_message(message)
return message
def decorate_message_for_feed(msg, msg_data=None):
"""This function takes a message from counterpartyd's message feed and mutates it a bit to be suitable to be
sent through the counterblockd message feed to an end-client"""
if not msg_data:
msg_data = json.loads(msg['bindings'])
message = copy.deepcopy(msg_data)
message['_message_index'] = msg['message_index']
message['_command'] = msg['command']
message['_block_index'] = msg['block_index']
message['_block_time'] = database.get_block_time(msg['block_index'])
message['_category'] = msg['category']
message['_status'] = msg_data.get('status', 'valid')
message = decorate_message(message)
return message
def decorate_message(message, for_txn_history=False):
# insert custom fields in certain events...
# even invalid actions need these extra fields for proper reporting to the client (as the reporting message
# is produced via PendingActionViewModel.calcText) -- however make it able to deal with the queried data not existing in this case
assert '_category' in message
if for_txn_history:
message['_command'] = 'insert' # history data doesn't include this
block_index = message['block_index'] if 'block_index' in message else message['tx1_block_index']
message['_block_time'] = database.get_block_time(block_index)
message['_tx_index'] = message['tx_index'] if 'tx_index' in message else message.get('tx1_index', None)
if message['_category'] in ['bet_expirations', 'order_expirations', 'bet_match_expirations', 'order_match_expirations']:
message['_tx_index'] = 0 # add tx_index to all entries (so we can sort on it secondarily in history view), since these lack it
if message['_category'] in ['credits', 'debits']:
# find the last balance change on record
bal_change = config.mongo_db.balance_changes.find_one(
{'address': message['address'], 'asset': message['asset']},
sort=[("block_time", pymongo.DESCENDING)])
message['_quantity_normalized'] = abs(bal_change['quantity_normalized']) if bal_change else None
message['_balance'] = bal_change['new_balance'] if bal_change else None
message['_balance_normalized'] = bal_change['new_balance_normalized'] if bal_change else None
if message['_category'] in ['orders', ] and message['_command'] == 'insert':
get_asset_info = config.mongo_db.tracked_assets.find_one({'asset': message['get_asset']})
give_asset_info = config.mongo_db.tracked_assets.find_one({'asset': message['give_asset']})
message['_get_asset_divisible'] = get_asset_info['divisible'] if get_asset_info else None
message['_give_asset_divisible'] = give_asset_info['divisible'] if give_asset_info else None
if message['_category'] in ['order_matches', ] and message['_command'] == 'insert':
forward_asset_info = config.mongo_db.tracked_assets.find_one({'asset': message['forward_asset']})
backward_asset_info = config.mongo_db.tracked_assets.find_one({'asset': message['backward_asset']})
message['_forward_asset_divisible'] = forward_asset_info['divisible'] if forward_asset_info else None
message['_backward_asset_divisible'] = backward_asset_info['divisible'] if backward_asset_info else None
if message['_category'] in ['orders', 'order_matches', ]:
message['_btc_below_dust_limit'] = (
('forward_asset' in message and message['forward_asset'] == config.BTC and message['forward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF)
or ('backward_asset' in message and message['backward_asset'] == config.BTC and message['backward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF)
)
if message['_category'] in ['dividends', 'sends', ]:
asset_info = config.mongo_db.tracked_assets.find_one({'asset': message['asset']})
message['_divisible'] = asset_info['divisible'] if asset_info else None
if message['_category'] in ['issuances', ]:
message['_quantity_normalized'] = blockchain.normalize_quantity(message['quantity'], message['divisible'])
return message
def decorate_message(message, for_txn_history=False):
# insert custom fields in certain events...
# even invalid actions need these extra fields for proper reporting to the client (as the reporting message
# is produced via PendingActionViewModel.calcText) -- however make it able to deal with the queried data not existing in this case
assert '_category' in message
if for_txn_history:
message['_command'] = 'insert' # history data doesn't include this
block_index = message[
'block_index'] if 'block_index' in message else message[
'tx1_block_index']
message['_block_time'] = database.get_block_time(block_index)
message['_tx_index'] = message[
'tx_index'] if 'tx_index' in message else message.get(
'tx1_index', None)
if message['_category'] in [
'bet_expirations', 'order_expirations',
'bet_match_expirations', 'order_match_expirations'
]:
message[
'_tx_index'] = 0 # add tx_index to all entries (so we can sort on it secondarily in history view), since these lack it
if message['_category'] in ['credits', 'debits']:
# find the last balance change on record
bal_change = config.mongo_db.balance_changes.find_one(
{
'address': message['address'],
'asset': message['asset']
},
sort=[("block_time", pymongo.DESCENDING)])
message['_quantity_normalized'] = abs(
bal_change['quantity_normalized']) if bal_change else None
message['_balance'] = bal_change['new_balance'] if bal_change else None
message['_balance_normalized'] = bal_change[
'new_balance_normalized'] if bal_change else None
if message['_category'] in [
'orders',
] and message['_command'] == 'insert':
get_asset_info = config.mongo_db.tracked_assets.find_one(
{'asset': message['get_asset']})
give_asset_info = config.mongo_db.tracked_assets.find_one(
{'asset': message['give_asset']})
message['_get_asset_divisible'] = get_asset_info[
'divisible'] if get_asset_info else None
message['_give_asset_divisible'] = give_asset_info[
'divisible'] if give_asset_info else None
if message['_category'] in [
'order_matches',
] and message['_command'] == 'insert':
forward_asset_info = config.mongo_db.tracked_assets.find_one(
{'asset': message['forward_asset']})
backward_asset_info = config.mongo_db.tracked_assets.find_one(
{'asset': message['backward_asset']})
message['_forward_asset_divisible'] = forward_asset_info[
'divisible'] if forward_asset_info else None
message['_backward_asset_divisible'] = backward_asset_info[
'divisible'] if backward_asset_info else None
if message['_category'] in [
'orders',
'order_matches',
]:
message['_btc_below_dust_limit'] = (
('forward_asset' in message
and message['forward_asset'] == config.BTC and
message['forward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF)
or ('backward_asset' in message
and message['backward_asset'] == config.BTC
and message['backward_quantity'] <=
config.ORDER_BTC_DUST_LIMIT_CUTOFF))
if message['_category'] in [
'dividends',
'sends',
]:
asset_info = config.mongo_db.tracked_assets.find_one(
{'asset': message['asset']})
message['_divisible'] = asset_info['divisible'] if asset_info else None
if message['_category'] in [
'issuances',
]:
message['_quantity_normalized'] = blockchain.normalize_quantity(
message['quantity'], message['divisible'])
return message
def compile_asset_market_info():
"""Run through all assets and compose and store market ranking information."""
mongo_db = config.mongo_db
if not config.state['caught_up']:
logger.warn(
"Not updating asset market info as counterblockd is not caught up."
)
return False
#grab the last block # we processed assets data off of
last_block_assets_compiled = mongo_db.app_config.find_one(
)['last_block_assets_compiled']
last_block_time_assets_compiled = database.get_block_time(
last_block_assets_compiled)
#logger.debug("Comping info for assets traded since block %i" % last_block_assets_compiled)
current_block_index = config.state['my_latest_block'][
'block_index'] #store now as it may change as we are compiling asset data :)
current_block_time = database.get_block_time(current_block_index)
if current_block_index == last_block_assets_compiled:
#all caught up -- call again in 10 minutes
return True
mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives()
all_traded_assets = list(
set(
list([config.BTC, config.XCP]) + list(
mongo_db.trades.find({}, {
'quote_asset': 1,
'_id': 0
}).distinct('quote_asset'))))
#######################
#get a list of all assets with a trade within the last 24h (not necessarily just against XCP and BTC)
# ^ this is important because compiled market info has a 24h vol parameter that designates total volume for the asset across ALL pairings
start_dt_1d = datetime.datetime.utcnow() - datetime.timedelta(days=1)
assets = list(
set(
list(
mongo_db.trades.find({
'block_time': {
'$gte': start_dt_1d
}
}).distinct('quote_asset')) + list(
mongo_db.trades.find({
'block_time': {
'$gte': start_dt_1d
}
}).distinct('base_asset'))))
for asset in assets:
market_info_24h = compile_24h_market_info(asset)
mongo_db.asset_market_info.update({'asset': asset},
{"$set": market_info_24h})
#for all others (i.e. no trade in the last 24 hours), zero out the 24h trade data
non_traded_assets = list(set(all_traded_assets) - set(assets))
mongo_db.asset_market_info.update({'asset': {
'$in': non_traded_assets
}}, {
"$set": {
'24h_summary': {
'vol': 0,
'count': 0
},
'24h_ohlc_in_{}'.format(config.XCP.lower()): {},
'24h_ohlc_in_{}'.format(config.BTC.lower()): {},
'24h_vol_price_change_in_{}'.format(config.XCP.lower()): None,
'24h_vol_price_change_in_{}'.format(config.BTC.lower()): None,
}
},
multi=True)
logger.info("Block: %s -- Calculated 24h stats for: %s" %
(current_block_index, ', '.join(assets)))
#######################
#get a list of all assets with a trade within the last 7d up against XCP and BTC
start_dt_7d = datetime.datetime.utcnow() - datetime.timedelta(days=7)
assets = list(
set(
list(
mongo_db.trades.find({
'block_time': {
'$gte': start_dt_7d
},
'base_asset': {
'$in': [config.XCP, config.BTC]
}
}).distinct('quote_asset')) + list(
mongo_db.trades.find({
'block_time': {
'$gte': start_dt_7d
}
}).distinct('base_asset'))))
for asset in assets:
market_info_7d = compile_7d_market_info(asset)
mongo_db.asset_market_info.update({'asset': asset},
{"$set": market_info_7d})
non_traded_assets = list(set(all_traded_assets) - set(assets))
mongo_db.asset_market_info.update({'asset': {
'$in': non_traded_assets
}}, {
"$set": {
'7d_history_in_{}'.format(config.XCP.lower()): [],
'7d_history_in_{}'.format(config.BTC.lower()): [],
}
},
multi=True)
logger.info("Block: %s -- Calculated 7d stats for: %s" %
(current_block_index, ', '.join(assets)))
#######################
#update summary market data for assets traded since last_block_assets_compiled
#get assets that were traded since the last check with either BTC or XCP, and update their market summary data
assets = list(
set(
list(
mongo_db.trades.find({
'block_index': {
'$gt': last_block_assets_compiled
},
'base_asset': {
'$in': [config.XCP, config.BTC]
}
}).distinct('quote_asset')) + list(
mongo_db.trades.find({
'block_index': {
'$gt': last_block_assets_compiled
}
}).distinct('base_asset'))))
#update our storage of the latest market info in mongo
for asset in assets:
logger.info("Block: %s -- Updating asset market info for %s ..." %
(current_block_index, asset))
summary_info = compile_summary_market_info(asset, mps_xcp_btc,
xcp_btc_price,
btc_xcp_price)
mongo_db.asset_market_info.update({'asset': asset},
{"$set": summary_info},
upsert=True)
#######################
#next, compile market cap historicals (and get the market price data that we can use to update assets with new trades)
#NOTE: this algoritm still needs to be fleshed out some...I'm not convinced it's laid out/optimized like it should be
#start by getting all trades from when we last compiled this data
trades = mongo_db.trades.find({
'block_index': {
'$gt': last_block_assets_compiled
}
}).sort('block_index', pymongo.ASCENDING)
trades_by_block = [
] #tracks assets compiled per block, as we only want to analyze any given asset once per block
trades_by_block_mapping = {}
#organize trades by block
for t in trades:
if t['block_index'] in trades_by_block_mapping:
assert trades_by_block_mapping[
t['block_index']]['block_index'] == t['block_index']
assert trades_by_block_mapping[
t['block_index']]['block_time'] == t['block_time']
trades_by_block_mapping[t['block_index']]['trades'].append(t)
else:
e = {
'block_index': t['block_index'],
'block_time': t['block_time'],
'trades': [
t,
]
}
trades_by_block.append(e)
trades_by_block_mapping[t['block_index']] = e
for t_block in trades_by_block:
#reverse the tradelist per block, and ensure that we only process an asset that hasn't already been processed for this block
# (as there could be multiple trades in a single block for any specific asset). we reverse the list because
# we'd rather process a later trade for a given asset, as the market price for that will take into account
# the earlier trades on that same block for that asset, and we don't want/need multiple cap points per block
assets_in_block = {}
mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives(
end_dt=t_block['block_time'])
for t in reversed(t_block['trades']):
assets = []
if t['base_asset'] not in assets_in_block:
assets.append(t['base_asset'])
assets_in_block[t['base_asset']] = True
if t['quote_asset'] not in assets_in_block:
assets.append(t['quote_asset'])
assets_in_block[t['quote_asset']] = True
if not len(assets): continue
for asset in assets:
#recalculate the market cap for the asset this trade is for
asset_info = get_asset_info(asset, at_dt=t['block_time'])
(price_summary_in_xcp, price_summary_in_btc, price_in_xcp,
price_in_btc, aggregated_price_in_xcp,
aggregated_price_in_btc) = get_xcp_btc_price_info(
asset,
mps_xcp_btc,
xcp_btc_price,
btc_xcp_price,
with_last_trades=0,
end_dt=t['block_time'])
market_cap_in_xcp, market_cap_in_btc = calc_market_cap(
asset_info, price_in_xcp, price_in_btc)
#^ this will get price data from the block time of this trade back the standard number of days and trades
# to determine our standard market price, relative (anchored) to the time of this trade
for market_cap_as in (config.XCP, config.BTC):
market_cap = market_cap_in_xcp if market_cap_as == config.XCP else market_cap_in_btc
#if there is a previously stored market cap for this asset, add a new history point only if the two caps differ
prev_market_cap_history = mongo_db.asset_marketcap_history.find(
{
'market_cap_as': market_cap_as,
'asset': asset,
'block_index': {
'$lt': t['block_index']
}
}).sort('block_index', pymongo.DESCENDING).limit(1)
prev_market_cap_history = list(
prev_market_cap_history
)[0] if prev_market_cap_history.count() == 1 else None
if market_cap and (not prev_market_cap_history
or prev_market_cap_history['market_cap']
!= market_cap):
mongo_db.asset_marketcap_history.insert({
'block_index':
t['block_index'],
'block_time':
t['block_time'],
'asset':
asset,
'market_cap':
market_cap,
'market_cap_as':
market_cap_as,
})
logger.info(
"Block %i -- Calculated market cap history point for %s as %s (mID: %s)"
% (t['block_index'], asset, market_cap_as,
t['message_index']))
mongo_db.app_config.update(
{}, {'$set': {
'last_block_assets_compiled': current_block_index
}})
return True
def compile_asset_market_info():
"""Run through all assets and compose and store market ranking information."""
if not config.state['caught_up']:
logger.warn("Not updating asset market info as counterblockd is not caught up.")
return False
#grab the last block # we processed assets data off of
last_block_assets_compiled = config.mongo_db.app_config.find_one()['last_block_assets_compiled']
last_block_time_assets_compiled = database.get_block_time(last_block_assets_compiled)
#logger.debug("Comping info for assets traded since block %i" % last_block_assets_compiled)
current_block_index = config.state['my_latest_block']['block_index'] #store now as it may change as we are compiling asset data :)
current_block_time = database.get_block_time(current_block_index)
if current_block_index == last_block_assets_compiled:
#all caught up -- call again in 10 minutes
return True
mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives()
all_traded_assets = list(set(list([config.BTC, config.XCP]) + list(config.mongo_db.trades.find({}, {'quote_asset': 1, '_id': 0}).distinct('quote_asset'))))
#######################
#get a list of all assets with a trade within the last 24h (not necessarily just against XCP and BTC)
# ^ this is important because compiled market info has a 24h vol parameter that designates total volume for the asset across ALL pairings
start_dt_1d = datetime.datetime.utcnow() - datetime.timedelta(days=1)
assets = list(set(
list(config.mongo_db.trades.find({'block_time': {'$gte': start_dt_1d}}).distinct('quote_asset'))
+ list(config.mongo_db.trades.find({'block_time': {'$gte': start_dt_1d}}).distinct('base_asset'))
))
for asset in assets:
market_info_24h = compile_24h_market_info(asset)
config.mongo_db.asset_market_info.update({'asset': asset}, {"$set": market_info_24h})
#for all others (i.e. no trade in the last 24 hours), zero out the 24h trade data
non_traded_assets = list(set(all_traded_assets) - set(assets))
config.mongo_db.asset_market_info.update( {'asset': {'$in': non_traded_assets}}, {"$set": {
'24h_summary': {'vol': 0, 'count': 0},
'24h_ohlc_in_{}'.format(config.XCP.lower()): {},
'24h_ohlc_in_{}'.format(config.BTC.lower()): {},
'24h_vol_price_change_in_{}'.format(config.XCP.lower()): None,
'24h_vol_price_change_in_{}'.format(config.BTC.lower()): None,
}}, multi=True)
logger.info("Block: %s -- Calculated 24h stats for: %s" % (current_block_index, ', '.join(assets)))
#######################
#get a list of all assets with a trade within the last 7d up against XCP and BTC
start_dt_7d = datetime.datetime.utcnow() - datetime.timedelta(days=7)
assets = list(set(
list(config.mongo_db.trades.find({'block_time': {'$gte': start_dt_7d}, 'base_asset': {'$in': [config.XCP, config.BTC]}}).distinct('quote_asset'))
+ list(config.mongo_db.trades.find({'block_time': {'$gte': start_dt_7d}}).distinct('base_asset'))
))
for asset in assets:
market_info_7d = compile_7d_market_info(asset)
config.mongo_db.asset_market_info.update({'asset': asset}, {"$set": market_info_7d})
non_traded_assets = list(set(all_traded_assets) - set(assets))
config.mongo_db.asset_market_info.update( {'asset': {'$in': non_traded_assets}}, {"$set": {
'7d_history_in_{}'.format(config.XCP.lower()): [],
'7d_history_in_{}'.format(config.BTC.lower()): [],
}}, multi=True)
logger.info("Block: %s -- Calculated 7d stats for: %s" % (current_block_index, ', '.join(assets)))
#######################
#update summary market data for assets traded since last_block_assets_compiled
#get assets that were traded since the last check with either BTC or XCP, and update their market summary data
assets = list(set(
list(config.mongo_db.trades.find({'block_index': {'$gt': last_block_assets_compiled}, 'base_asset': {'$in': [config.XCP, config.BTC]}}).distinct('quote_asset'))
+ list(config.mongo_db.trades.find({'block_index': {'$gt': last_block_assets_compiled}}).distinct('base_asset'))
))
#update our storage of the latest market info in mongo
for asset in assets:
logger.info("Block: %s -- Updating asset market info for %s ..." % (current_block_index, asset))
summary_info = compile_summary_market_info(asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price)
config.mongo_db.asset_market_info.update( {'asset': asset}, {"$set": summary_info}, upsert=True)
#######################
#next, compile market cap historicals (and get the market price data that we can use to update assets with new trades)
#NOTE: this algoritm still needs to be fleshed out some...I'm not convinced it's laid out/optimized like it should be
#start by getting all trades from when we last compiled this data
trades = config.mongo_db.trades.find({'block_index': {'$gt': last_block_assets_compiled}}).sort('block_index', pymongo.ASCENDING)
trades_by_block = [] #tracks assets compiled per block, as we only want to analyze any given asset once per block
trades_by_block_mapping = {}
#organize trades by block
for t in trades:
if t['block_index'] in trades_by_block_mapping:
assert trades_by_block_mapping[t['block_index']]['block_index'] == t['block_index']
assert trades_by_block_mapping[t['block_index']]['block_time'] == t['block_time']
trades_by_block_mapping[t['block_index']]['trades'].append(t)
else:
e = {'block_index': t['block_index'], 'block_time': t['block_time'], 'trades': [t,]}
trades_by_block.append(e)
trades_by_block_mapping[t['block_index']] = e
for t_block in trades_by_block:
#reverse the tradelist per block, and ensure that we only process an asset that hasn't already been processed for this block
# (as there could be multiple trades in a single block for any specific asset). we reverse the list because
# we'd rather process a later trade for a given asset, as the market price for that will take into account
# the earlier trades on that same block for that asset, and we don't want/need multiple cap points per block
assets_in_block = {}
mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives(end_dt=t_block['block_time'])
for t in reversed(t_block['trades']):
assets = []
if t['base_asset'] not in assets_in_block:
assets.append(t['base_asset'])
assets_in_block[t['base_asset']] = True
if t['quote_asset'] not in assets_in_block:
assets.append(t['quote_asset'])
assets_in_block[t['quote_asset']] = True
if not len(assets): continue
for asset in assets:
#recalculate the market cap for the asset this trade is for
asset_info = get_asset_info(asset, at_dt=t['block_time'])
(price_summary_in_xcp, price_summary_in_btc, price_in_xcp, price_in_btc, aggregated_price_in_xcp, aggregated_price_in_btc
) = get_xcp_btc_price_info(asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price, with_last_trades=0, end_dt=t['block_time'])
market_cap_in_xcp, market_cap_in_btc = calc_market_cap(asset_info, price_in_xcp, price_in_btc)
#^ this will get price data from the block time of this trade back the standard number of days and trades
# to determine our standard market price, relative (anchored) to the time of this trade
for market_cap_as in (config.XCP, config.BTC):
market_cap = market_cap_in_xcp if market_cap_as == config.XCP else market_cap_in_btc
#if there is a previously stored market cap for this asset, add a new history point only if the two caps differ
prev_market_cap_history = config.mongo_db.asset_marketcap_history.find({'market_cap_as': market_cap_as, 'asset': asset,
'block_index': {'$lt': t['block_index']}}).sort('block_index', pymongo.DESCENDING).limit(1)
prev_market_cap_history = list(prev_market_cap_history)[0] if prev_market_cap_history.count() == 1 else None
if market_cap and (not prev_market_cap_history or prev_market_cap_history['market_cap'] != market_cap):
config.mongo_db.asset_marketcap_history.insert({
'block_index': t['block_index'],
'block_time': t['block_time'],
'asset': asset,
'market_cap': market_cap,
'market_cap_as': market_cap_as,
})
logger.info("Block %i -- Calculated market cap history point for %s as %s (mID: %s)" % (t['block_index'], asset, market_cap_as, t['message_index']))
config.mongo_db.app_config.update({}, {'$set': {'last_block_assets_compiled': current_block_index}})
return True
