def get_asset_info(asset, at_dt=None):
mongo_db = config.mongo_db
asset_info = mongo_db.tracked_assets.find_one({'asset': asset})
if asset not in (config.XCP, config.BTC) and at_dt and asset_info['_at_block_time'] > at_dt:
#get the asset info at or before the given at_dt datetime
for e in reversed(asset_info['_history']): #newest to oldest
if e['_at_block_time'] <= at_dt:
asset_info = e
break
else: #asset was created AFTER at_dt
asset_info = None
if asset_info is None: return None
assert asset_info['_at_block_time'] <= at_dt
#modify some of the properties of the returned asset_info for BTC and XCP
if asset == config.BTC:
if at_dt:
start_block_index, end_block_index = util.get_block_indexes_for_dates(end_dt=at_dt)
asset_info['total_issued'] = util_bitcoin.get_btc_supply(normalize=False, at_block_index=end_block_index)
asset_info['total_issued_normalized'] = util_bitcoin.normalize_quantity(asset_info['total_issued'])
else:
asset_info['total_issued'] = util_bitcoin.get_btc_supply(normalize=False)
asset_info['total_issued_normalized'] = util_bitcoin.normalize_quantity(asset_info['total_issued'])
elif asset == config.XCP:
#BUG: this does not take end_dt (if specified) into account. however, the deviation won't be too big
# as XCP doesn't deflate quickly at all, and shouldn't matter that much since there weren't any/much trades
# before the end of the burn period (which is what is involved with how we use at_dt with currently)
asset_info['total_issued'] = util.call_jsonrpc_api("get_xcp_supply", abort_on_error=True)['result']
asset_info['total_issued_normalized'] = util_bitcoin.normalize_quantity(asset_info['total_issued'])
if not asset_info:
raise Exception("Invalid asset: %s" % asset)
return asset_info
def get_asset_info(asset, at_dt=None):
mongo_db = config.mongo_db
asset_info = mongo_db.tracked_assets.find_one({'asset': asset})
if asset not in (config.XCP, config.BTC
) and at_dt and asset_info['_at_block_time'] > at_dt:
#get the asset info at or before the given at_dt datetime
for e in reversed(asset_info['_history']): #newest to oldest
if e['_at_block_time'] <= at_dt:
asset_info = e
break
else: #asset was created AFTER at_dt
asset_info = None
if asset_info is None: return None
assert asset_info['_at_block_time'] <= at_dt
#modify some of the properties of the returned asset_info for DOGE and XDP
if asset == config.BTC:
if at_dt:
start_block_index, end_block_index = util.get_block_indexes_for_dates(
end_dt=at_dt)
asset_info['total_issued'] = util_bitcoin.get_btc_supply(
normalize=False, at_block_index=end_block_index)
asset_info[
'total_issued_normalized'] = util_bitcoin.normalize_quantity(
asset_info['total_issued'])
else:
asset_info['total_issued'] = util_bitcoin.get_btc_supply(
normalize=False)
asset_info[
'total_issued_normalized'] = util_bitcoin.normalize_quantity(
asset_info['total_issued'])
elif asset == config.XCP:
#BUG: this does not take end_dt (if specified) into account. however, the deviation won't be too big
# as XDP doesn't deflate quickly at all, and shouldn't matter that much since there weren't any/much trades
# before the end of the burn period (which is what is involved with how we use at_dt with currently)
asset_info['total_issued'] = util.call_jsonrpc_api(
"get_xcp_supply", abort_on_error=True)['result']
asset_info[
'total_issued_normalized'] = util_bitcoin.normalize_quantity(
asset_info['total_issued'])
if not asset_info:
raise Exception("Invalid asset: %s" % asset)
return asset_info
def get_asset_info(asset, at_dt=None):
mongo_db = config.mongo_db
asset_info = mongo_db.tracked_assets.find_one({'asset': asset})
if asset not in (config.XCP, config.BTC
) and at_dt and asset_info['_at_block_time'] > at_dt:
#get the asset info at or before the given at_dt datetime
for e in reversed(asset_info['_history']): #newest to oldest
if e['_at_block_time'] <= at_dt:
asset_info = e
break
else: #asset was created AFTER at_dt
asset_info = None
if asset_info is None: return None
assert asset_info['_at_block_time'] <= at_dt
if asset == config.BTC:
if at_dt:
start_block_index, end_block_index = util.get_block_indexes_for_dates(
end_dt=at_dt)
asset_info['total_issued'] = util_bitcoin.get_btc_supply(
normalize=False, at_block_index=end_block_index)
asset_info[
'total_issued_normalized'] = util_bitcoin.normalize_quantity(
asset_info['total_issued'])
else:
asset_info['total_issued'] = util_bitcoin.get_btc_supply(
normalize=False)
asset_info[
'total_issued_normalized'] = util_bitcoin.normalize_quantity(
asset_info['total_issued'])
elif asset == config.XCP:
asset_info['total_issued'] = util.call_jsonrpc_api(
"get_xcp_supply", abort_on_error=True)['result']
asset_info[
'total_issued_normalized'] = util_bitcoin.normalize_quantity(
asset_info['total_issued'])
if not asset_info:
raise Exception("Invalid asset: %s" % asset)
return asset_info
def decorate_message(message, for_txn_history=False):
#insert custom fields in certain events...
#even invalid actions need these extra fields for proper reporting to the client (as the reporting message
# is produced via PendingActionViewModel.calcText) -- however make it able to deal with the queried data not existing in this case
assert '_category' in message
mongo_db = config.mongo_db
if for_txn_history:
message['_command'] = 'insert' #history data doesn't include this
block_index = message['block_index'] if 'block_index' in message else message['tx1_block_index']
message['_block_time'] = get_block_time(block_index)
message['_tx_index'] = message['tx_index'] if 'tx_index' in message else message.get('tx1_index', None)
if message['_category'] in ['bet_expirations', 'order_expirations', 'bet_match_expirations', 'order_match_expirations']:
message['_tx_index'] = 0 #add tx_index to all entries (so we can sort on it secondarily in history view), since these lack it
if message['_category'] in ['credits', 'debits']:
#find the last balance change on record
bal_change = mongo_db.balance_changes.find_one({ 'address': message['address'], 'asset': message['asset'] },
sort=[("block_time", pymongo.DESCENDING)])
message['_quantity_normalized'] = abs(bal_change['quantity_normalized']) if bal_change else None
message['_balance'] = bal_change['new_balance'] if bal_change else None
message['_balance_normalized'] = bal_change['new_balance_normalized'] if bal_change else None
if message['_category'] in ['orders',] and message['_command'] == 'insert':
get_asset_info = mongo_db.tracked_assets.find_one({'asset': message['get_asset']})
give_asset_info = mongo_db.tracked_assets.find_one({'asset': message['give_asset']})
message['_get_asset_divisible'] = get_asset_info['divisible'] if get_asset_info else None
message['_give_asset_divisible'] = give_asset_info['divisible'] if give_asset_info else None
if message['_category'] in ['order_matches',] and message['_command'] == 'insert':
forward_asset_info = mongo_db.tracked_assets.find_one({'asset': message['forward_asset']})
backward_asset_info = mongo_db.tracked_assets.find_one({'asset': message['backward_asset']})
message['_forward_asset_divisible'] = forward_asset_info['divisible'] if forward_asset_info else None
message['_backward_asset_divisible'] = backward_asset_info['divisible'] if backward_asset_info else None
if message['_category'] in ['orders', 'order_matches',]:
message['_btc_below_dust_limit'] = (
('forward_asset' in message and message['forward_asset'] == config.BTC and message['forward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF)
or ('backward_asset' in message and message['backward_asset'] == config.BTC and message['backward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF)
)
if message['_category'] in ['dividends', 'sends', 'callbacks']:
asset_info = mongo_db.tracked_assets.find_one({'asset': message['asset']})
message['_divisible'] = asset_info['divisible'] if asset_info else None
if message['_category'] in ['issuances',]:
message['_quantity_normalized'] = util_bitcoin.normalize_quantity(message['quantity'], message['divisible'])
return message
def decorate_message(message, for_txn_history=False):
#insert custom fields in certain events...
#even invalid actions need these extra fields for proper reporting to the client (as the reporting message
# is produced via PendingActionViewModel.calcText) -- however make it able to deal with the queried data not existing in this case
assert '_category' in message
mongo_db = config.mongo_db
if for_txn_history:
message['_command'] = 'insert' #history data doesn't include this
block_index = message['block_index'] if 'block_index' in message else message['tx1_block_index']
message['_block_time'] = get_block_time(block_index)
message['_tx_index'] = message['tx_index'] if 'tx_index' in message else message.get('tx1_index', None)
if message['_category'] in ['bet_expirations', 'order_expirations', 'bet_match_expirations', 'order_match_expirations']:
message['_tx_index'] = 0 #add tx_index to all entries (so we can sort on it secondarily in history view), since these lack it
if message['_category'] in ['credits', 'debits']:
#find the last balance change on record
bal_change = mongo_db.balance_changes.find_one({ 'address': message['address'], 'asset': message['asset'] },
sort=[("block_time", pymongo.DESCENDING)])
message['_quantity_normalized'] = abs(bal_change['quantity_normalized']) if bal_change else None
message['_balance'] = bal_change['new_balance'] if bal_change else None
message['_balance_normalized'] = bal_change['new_balance_normalized'] if bal_change else None
if message['_category'] in ['orders',] and message['_command'] == 'insert':
get_asset_info = mongo_db.tracked_assets.find_one({'asset': message['get_asset']})
give_asset_info = mongo_db.tracked_assets.find_one({'asset': message['give_asset']})
message['_get_asset_divisible'] = get_asset_info['divisible'] if get_asset_info else None
message['_give_asset_divisible'] = give_asset_info['divisible'] if give_asset_info else None
if message['_category'] in ['order_matches',] and message['_command'] == 'insert':
forward_asset_info = mongo_db.tracked_assets.find_one({'asset': message['forward_asset']})
backward_asset_info = mongo_db.tracked_assets.find_one({'asset': message['backward_asset']})
message['_forward_asset_divisible'] = forward_asset_info['divisible'] if forward_asset_info else None
message['_backward_asset_divisible'] = backward_asset_info['divisible'] if backward_asset_info else None
if message['_category'] in ['orders', 'order_matches',]:
message['_btc_below_dust_limit'] = (
('forward_asset' in message and message['forward_asset'] == config.BTC and message['forward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF)
or ('backward_asset' in message and message['backward_asset'] == config.BTC and message['backward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF)
)
if message['_category'] in ['dividends', 'sends', 'callbacks']:
asset_info = mongo_db.tracked_assets.find_one({'asset': message['asset']})
message['_divisible'] = asset_info['divisible'] if asset_info else None
if message['_category'] in ['issuances',]:
message['_quantity_normalized'] = util_bitcoin.normalize_quantity(message['quantity'], message['divisible'])
return message
def compile_asset_pair_market_info():
"""Compiles the pair-level statistics that show on the View Prices page of counterwallet, for instance"""
#loop through all open orders, and compile a listing of pairs, with a count of open orders for each pair
mongo_db = config.mongo_db
end_dt = datetime.datetime.utcnow()
start_dt = end_dt - datetime.timedelta(days=1)
start_block_index, end_block_index = util.get_block_indexes_for_dates(start_dt=start_dt, end_dt=end_dt)
open_orders = util.call_jsonrpc_api("get_orders",
{ 'filters': [
{'field': 'give_remaining', 'op': '>', 'value': 0},
{'field': 'get_remaining', 'op': '>', 'value': 0},
{'field': 'fee_required_remaining', 'op': '>=', 'value': 0},
{'field': 'fee_provided_remaining', 'op': '>=', 'value': 0},
],
'status': 'open',
'show_expired': False,
}, abort_on_error=True)['result']
pair_data = {}
asset_info = {}
def get_price(base_quantity_normalized, quote_quantity_normalized):
return float(D(quote_quantity_normalized / base_quantity_normalized ))
#COMPOSE order depth, lowest ask, and highest bid column data
for o in open_orders:
(base_asset, quote_asset) = util.assets_to_asset_pair(o['give_asset'], o['get_asset'])
pair = '%s/%s' % (base_asset, quote_asset)
base_asset_info = asset_info.get(base_asset, mongo_db.tracked_assets.find_one({ 'asset': base_asset }))
if base_asset not in asset_info: asset_info[base_asset] = base_asset_info
quote_asset_info = asset_info.get(quote_asset, mongo_db.tracked_assets.find_one({ 'asset': quote_asset }))
if quote_asset not in asset_info: asset_info[quote_asset] = quote_asset_info
pair_data.setdefault(pair, {'open_orders_count': 0, 'lowest_ask': None, 'highest_bid': None,
'completed_trades_count': 0, 'vol_base': 0, 'vol_quote': 0})
#^ highest ask = open order selling base, highest bid = open order buying base
#^ we also initialize completed_trades_count, vol_base, vol_quote because every pair inited here may
# not have cooresponding data out of the trades_data_by_pair aggregation below
pair_data[pair]['open_orders_count'] += 1
base_quantity_normalized = util_bitcoin.normalize_quantity(o['give_quantity'] if base_asset == o['give_asset'] else o['get_quantity'], base_asset_info['divisible'])
quote_quantity_normalized = util_bitcoin.normalize_quantity(o['give_quantity'] if quote_asset == o['give_asset'] else o['get_quantity'], quote_asset_info['divisible'])
order_price = get_price(base_quantity_normalized, quote_quantity_normalized)
if base_asset == o['give_asset']: #selling base
if pair_data[pair]['lowest_ask'] is None or order_price < pair_data[pair]['lowest_ask']:
pair_data[pair]['lowest_ask'] = order_price
elif base_asset == o['get_asset']: #buying base
if pair_data[pair]['highest_bid'] is None or order_price > pair_data[pair]['highest_bid']:
pair_data[pair]['highest_bid'] = order_price
#COMPOSE volume data (in XCP and BTC), and % change data
#loop through all trade volume over the past 24h, and match that to the open orders
trades_data_by_pair = mongo_db.trades.aggregate([
{"$match": {
"block_time": {"$gte": start_dt, "$lte": end_dt } }
},
{"$project": {
"base_asset": 1,
"quote_asset": 1,
"base_quantity_normalized": 1, #to derive base volume
"quote_quantity_normalized": 1 #to derive quote volume
}},
{"$group": {
"_id": {"base_asset": "$base_asset", "quote_asset": "$quote_asset"},
"vol_base": {"$sum": "$base_quantity_normalized"},
"vol_quote": {"$sum": "$quote_quantity_normalized"},
"count": {"$sum": 1},
}}
])
trades_data_by_pair = [] if not trades_data_by_pair['ok'] else trades_data_by_pair['result']
for e in trades_data_by_pair:
pair = '%s/%s' % (e['_id']['base_asset'], e['_id']['quote_asset'])
pair_data.setdefault(pair, {'open_orders_count': 0, 'lowest_ask': None, 'highest_bid': None})
#^ initialize an empty pair in the event there are no open orders for that pair, but there ARE completed trades for it
pair_data[pair]['completed_trades_count'] = e['count']
pair_data[pair]['vol_base'] = e['vol_base']
pair_data[pair]['vol_quote'] = e['vol_quote']
#compose price data, relative to BTC and XCP
mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives()
for pair, e in pair_data.iteritems():
base_asset, quote_asset = pair.split('/')
_24h_vol_in_btc = None
_24h_vol_in_xcp = None
#derive asset price data, expressed in BTC and XCP, for the given volumes
if base_asset == config.XCP:
_24h_vol_in_xcp = e['vol_base']
_24h_vol_in_btc = util_bitcoin.round_out(e['vol_base'] * xcp_btc_price) if xcp_btc_price else 0
elif base_asset == config.BTC:
_24h_vol_in_xcp = util_bitcoin.round_out(e['vol_base'] * btc_xcp_price) if btc_xcp_price else 0
_24h_vol_in_btc = e['vol_base']
else: #base is not XCP or BTC
price_summary_in_xcp, price_summary_in_btc, price_in_xcp, price_in_btc, aggregated_price_in_xcp, aggregated_price_in_btc = \
get_xcp_btc_price_info(base_asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price, with_last_trades=0, start_dt=start_dt, end_dt=end_dt)
if price_in_xcp:
_24h_vol_in_xcp = util_bitcoin.round_out(e['vol_base'] * price_in_xcp)
if price_in_btc:
_24h_vol_in_btc = util_bitcoin.round_out(e['vol_base'] * price_in_btc)
if _24h_vol_in_xcp is None or _24h_vol_in_btc is None:
#the base asset didn't have price data against BTC or XCP, or both...try against the quote asset instead
price_summary_in_xcp, price_summary_in_btc, price_in_xcp, price_in_btc, aggregated_price_in_xcp, aggregated_price_in_btc = \
get_xcp_btc_price_info(quote_asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price, with_last_trades=0, start_dt=start_dt, end_dt=end_dt)
if _24h_vol_in_xcp is None and price_in_xcp:
_24h_vol_in_xcp = util_bitcoin.round_out(e['vol_quote'] * price_in_xcp)
if _24h_vol_in_btc is None and price_in_btc:
_24h_vol_in_btc = util_bitcoin.round_out(e['vol_quote'] * price_in_btc)
pair_data[pair]['24h_vol_in_{}'.format(config.XCP.lower())] = _24h_vol_in_xcp #might still be None
pair_data[pair]['24h_vol_in_{}'.format(config.BTC.lower())] = _24h_vol_in_btc #might still be None
#get % change stats -- start by getting the first trade directly before the 24h period starts
prev_trade = mongo_db.trades.find({
"base_asset": base_asset,
"quote_asset": quote_asset,
"block_time": {'$lt': start_dt}}).sort('block_time', pymongo.DESCENDING).limit(1)
latest_trade = mongo_db.trades.find({
"base_asset": base_asset,
"quote_asset": quote_asset}).sort('block_time', pymongo.DESCENDING).limit(1)
if not prev_trade.count(): #no previous trade before this 24hr period
pair_data[pair]['24h_pct_change'] = None
else:
prev_trade = prev_trade[0]
latest_trade = latest_trade[0]
prev_trade_price = get_price(prev_trade['base_quantity_normalized'], prev_trade['quote_quantity_normalized'])
latest_trade_price = get_price(latest_trade['base_quantity_normalized'], latest_trade['quote_quantity_normalized'])
pair_data[pair]['24h_pct_change'] = ((latest_trade_price - prev_trade_price) / prev_trade_price) * 100
pair_data[pair]['last_updated'] = end_dt
#print "PRODUCED", pair, pair_data[pair]
mongo_db.asset_pair_market_info.update( {'base_asset': base_asset, 'quote_asset': quote_asset}, {"$set": pair_data[pair]}, upsert=True)
#remove any old pairs that were not just updated
mongo_db.asset_pair_market_info.remove({'last_updated': {'$lt': end_dt}})
logging.info("Recomposed 24h trade statistics for %i asset pairs: %s" % (len(pair_data), ', '.join(pair_data.keys())))
def parse_issuance(db, message, cur_block_index, cur_block):
if message['status'] != 'valid':
return
def modify_extended_asset_info(asset, description):
"""adds an asset to asset_extended_info collection if the description is a valid json link. or, if the link
is not a valid json link, will remove the asset entry from the table if it exists"""
if util.is_valid_url(description,
suffix='.json',
allow_no_protocol=True):
db.asset_extended_info.update(
{'asset': asset},
{
'$set': {
'info_url': description,
'info_status': 'needfetch',
'fetch_info_retry':
0, # retry ASSET_MAX_RETRY times to fetch info from info_url
'info_data': {},
'errors': []
}
},
upsert=True)
#^ valid info_status settings: needfetch, valid, invalid, error
#additional fields will be added later in events, once the asset info is pulled
else:
db.asset_extended_info.remove({'asset': asset})
#remove any saved asset image data
imagePath = os.path.join(config.DATA_DIR,
config.SUBDIR_ASSET_IMAGES,
asset + '.png')
if os.path.exists(imagePath):
os.remove(imagePath)
tracked_asset = db.tracked_assets.find_one({'asset': message['asset']}, {
'_id': 0,
'_history': 0
})
#^ pulls the tracked asset without the _id and history fields. This may be None
if message['locked']: #lock asset
assert tracked_asset is not None
db.tracked_assets.update({'asset': message['asset']}, {
"$set": {
'_at_block': cur_block_index,
'_at_block_time': cur_block['block_time_obj'],
'_change_type': 'locked',
'locked': True,
},
"$push": {
'_history': tracked_asset
}
},
upsert=False)
logging.info("Locking asset %s" % (message['asset'], ))
elif message['transfer']: #transfer asset
assert tracked_asset is not None
db.tracked_assets.update({'asset': message['asset']}, {
"$set": {
'_at_block': cur_block_index,
'_at_block_time': cur_block['block_time_obj'],
'_change_type': 'transferred',
'owner': message['issuer'],
},
"$push": {
'_history': tracked_asset
}
},
upsert=False)
logging.info("Transferring asset %s to address %s" %
(message['asset'], message['issuer']))
elif message[
'quantity'] == 0 and tracked_asset is not None: #change description
db.tracked_assets.update({'asset': message['asset']}, {
"$set": {
'_at_block': cur_block_index,
'_at_block_time': cur_block['block_time_obj'],
'_change_type': 'changed_description',
'description': message['description'],
},
"$push": {
'_history': tracked_asset
}
},
upsert=False)
modify_extended_asset_info(message['asset'], message['description'])
logging.info("Changing description for asset %s to '%s'" %
(message['asset'], message['description']))
else: #issue new asset or issue addition qty of an asset
if not tracked_asset: #new issuance
tracked_asset = {
'_change_type':
'created',
'_at_block':
cur_block_index, #the block ID this asset is current for
'_at_block_time':
cur_block['block_time_obj'],
#^ NOTE: (if there are multiple asset tracked changes updates in a single block for the same
# asset, the last one with _at_block == that block id in the history array is the
# final version for that asset at that block
'asset':
message['asset'],
'owner':
message['issuer'],
'description':
message['description'],
'divisible':
message['divisible'],
'locked':
False,
'total_issued':
message['quantity'],
'total_issued_normalized':
util_bitcoin.normalize_quantity(message['quantity'],
message['divisible']),
'_history': [] #to allow for block rollbacks
}
db.tracked_assets.insert(tracked_asset)
logging.info("Tracking new asset: %s" % message['asset'])
modify_extended_asset_info(message['asset'],
message['description'])
else: #issuing additional of existing asset
assert tracked_asset is not None
db.tracked_assets.update({'asset': message['asset']}, {
"$set": {
'_at_block': cur_block_index,
'_at_block_time': cur_block['block_time_obj'],
'_change_type': 'issued_more',
},
"$inc": {
'total_issued':
message['quantity'],
'total_issued_normalized':
util_bitcoin.normalize_quantity(message['quantity'],
message['divisible'])
},
"$push": {
'_history': tracked_asset
}
},
upsert=False)
logging.info("Adding additional %s quantity for asset %s" %
(util_bitcoin.normalize_quantity(
message['quantity'],
message['divisible']), message['asset']))
return True
def process_cpd_blockfeed(zmq_publisher_eventfeed):
LATEST_BLOCK_INIT = {
'block_index': config.BLOCK_FIRST,
'block_time': None,
'block_hash': None
}
mongo_db = config.mongo_db
blocks_to_insert = []
def blow_away_db():
"""boom! blow away all applicable collections in mongo"""
mongo_db.processed_blocks.drop()
mongo_db.tracked_assets.drop()
mongo_db.trades.drop()
mongo_db.balance_changes.drop()
mongo_db.asset_market_info.drop()
mongo_db.asset_marketcap_history.drop()
mongo_db.pair_market_info.drop()
mongo_db.btc_open_orders.drop()
mongo_db.asset_extended_info.drop()
mongo_db.transaction_stats.drop()
mongo_db.feeds.drop()
mongo_db.wallet_stats.drop()
#create/update default app_config object
mongo_db.app_config.update(
{},
{
'db_version':
config.DB_VERSION, #counterblockd database version
'running_testnet': config.TESTNET,
'counterpartyd_db_version_major': None,
'counterpartyd_db_version_minor': None,
'counterpartyd_running_testnet': None,
'last_block_assets_compiled': config.
BLOCK_FIRST, #for asset data compilation in events.py (resets on reparse as well)
},
upsert=True)
app_config = mongo_db.app_config.find()[0]
#DO NOT DELETE preferences and chat_handles and chat_history
#create XCP and BTC assets in tracked_assets
for asset in [config.XCP, config.BTC]:
base_asset = {
'asset': asset,
'owner': None,
'divisible': True,
'locked': False,
'total_issued': None,
'_at_block':
config.BLOCK_FIRST, #the block ID this asset is current for
'_history': [] #to allow for block rollbacks
}
mongo_db.tracked_assets.insert(base_asset)
#reinitialize some internal counters
config.CURRENT_BLOCK_INDEX = 0
config.LAST_MESSAGE_INDEX = -1
return app_config
def prune_my_stale_blocks(max_block_index):
"""called if there are any records for blocks higher than this in the database? If so, they were impartially created
and we should get rid of them
NOTE: after calling this function, you should always trigger a "continue" statement to reiterate the processing loop
(which will get a new last_processed_block from counterpartyd and resume as appropriate)
"""
logging.warn("Pruning to block %i ..." % (max_block_index))
mongo_db.processed_blocks.remove(
{"block_index": {
"$gt": max_block_index
}})
mongo_db.balance_changes.remove(
{"block_index": {
"$gt": max_block_index
}})
mongo_db.trades.remove({"block_index": {"$gt": max_block_index}})
mongo_db.asset_marketcap_history.remove(
{"block_index": {
"$gt": max_block_index
}})
mongo_db.transaction_stats.remove(
{"block_index": {
"$gt": max_block_index
}})
#to roll back the state of the tracked asset, dive into the history object for each asset that has
# been updated on or after the block that we are pruning back to
assets_to_prune = mongo_db.tracked_assets.find(
{'_at_block': {
"$gt": max_block_index
}})
for asset in assets_to_prune:
logging.info(
"Pruning asset %s (last modified @ block %i, pruning to state at block %i)"
% (asset['asset'], asset['_at_block'], max_block_index))
prev_ver = None
while len(asset['_history']):
prev_ver = asset['_history'].pop()
if prev_ver['_at_block'] <= max_block_index:
break
if prev_ver:
if prev_ver['_at_block'] > max_block_index:
#even the first history version is newer than max_block_index.
#in this case, just remove the asset tracking record itself
mongo_db.tracked_assets.remove({'asset': asset['asset']})
else:
#if here, we were able to find a previous version that was saved at or before max_block_index
# (which should be prev_ver ... restore asset's values to its values
prev_ver['_id'] = asset['_id']
prev_ver['_history'] = asset['_history']
mongo_db.tracked_assets.save(prev_ver)
config.LAST_MESSAGE_INDEX = -1
config.CAUGHT_UP = False
util.blockinfo_cache.clear()
latest_block = mongo_db.processed_blocks.find_one(
{"block_index": max_block_index}) or LATEST_BLOCK_INIT
return latest_block
def publish_mempool_tx():
"""fetch new tx from mempool"""
tx_hashes = []
mempool_txs = mongo_db.mempool.find(fields={'tx_hash': True})
for mempool_tx in mempool_txs:
tx_hashes.append(str(mempool_tx['tx_hash']))
params = None
if len(tx_hashes) > 0:
params = {
'filters': [{
'field': 'tx_hash',
'op': 'NOT IN',
'value': tx_hashes
}, {
'field':
'category',
'op':
'IN',
'value': [
'sends', 'btcpays', 'issuances', 'dividends',
'callbacks'
]
}],
'filterop':
'AND'
}
new_txs = util.call_jsonrpc_api("get_mempool",
params,
abort_on_error=True)
for new_tx in new_txs['result']:
tx = {
'tx_hash': new_tx['tx_hash'],
'command': new_tx['command'],
'category': new_tx['category'],
'bindings': new_tx['bindings'],
'timestamp': new_tx['timestamp'],
'viewed_in_block': config.CURRENT_BLOCK_INDEX
}
mongo_db.mempool.insert(tx)
del (tx['_id'])
tx['_category'] = tx['category']
tx['_message_index'] = 'mempool'
logging.debug("Spotted mempool tx: %s" % tx)
zmq_publisher_eventfeed.send_json(tx)
def clean_mempool_tx():
"""clean mempool transactions older than MAX_REORG_NUM_BLOCKS blocks"""
mongo_db.mempool.remove({
"viewed_in_block": {
"$lt": config.CURRENT_BLOCK_INDEX - config.MAX_REORG_NUM_BLOCKS
}
})
config.CURRENT_BLOCK_INDEX = 0 #initialize (last processed block index -- i.e. currently active block)
config.LAST_MESSAGE_INDEX = -1 #initialize (last processed message index)
config.BLOCKCHAIN_SERVICE_LAST_BLOCK = 0 #simply for printing/alerting purposes
config.CAUGHT_UP_STARTED_EVENTS = False
#^ set after we are caught up and start up the recurring events that depend on us being caught up with the blockchain
#grab our stored preferences, and rebuild the database if necessary
app_config = mongo_db.app_config.find()
assert app_config.count() in [0, 1]
if (app_config.count() == 0 or config.REPARSE_FORCED
or app_config[0]['db_version'] != config.DB_VERSION
or app_config[0]['running_testnet'] != config.TESTNET):
if app_config.count():
logging.warn(
"counterblockd database version UPDATED (from %i to %i) or testnet setting changed (from %s to %s), or REINIT forced (%s). REBUILDING FROM SCRATCH ..."
% (app_config[0]['db_version'], config.DB_VERSION,
app_config[0]['running_testnet'], config.TESTNET,
config.REPARSE_FORCED))
else:
logging.warn(
"counterblockd database app_config collection doesn't exist. BUILDING FROM SCRATCH..."
)
app_config = blow_away_db()
my_latest_block = LATEST_BLOCK_INIT
else:
app_config = app_config[0]
#get the last processed block out of mongo
my_latest_block = mongo_db.processed_blocks.find_one(
sort=[("block_index", pymongo.DESCENDING)]) or LATEST_BLOCK_INIT
#remove any data we have for blocks higher than this (would happen if counterblockd or mongo died
# or errored out while processing a block)
my_latest_block = prune_my_stale_blocks(my_latest_block['block_index'])
#avoid contacting counterpartyd (on reparse, to speed up)
autopilot = False
autopilot_runner = 0
#start polling counterpartyd for new blocks
while True:
if not autopilot or autopilot_runner == 0:
try:
running_info = util.call_jsonrpc_api("get_running_info",
abort_on_error=True)
if 'result' not in running_info:
raise AssertionError("Could not contact counterpartyd")
running_info = running_info['result']
except Exception, e:
logging.warn(
str(e) + " -- Waiting 3 seconds before trying again...")
time.sleep(3)
continue
if running_info[
'last_message_index'] == -1: #last_message_index not set yet (due to no messages in counterpartyd DB yet)
logging.warn(
"No last_message_index returned. Waiting until counterpartyd has messages..."
)
time.sleep(10)
continue
#wipe our state data if necessary, if counterpartyd has moved on to a new DB version
wipeState = False
updatePrefs = False
if app_config['counterpartyd_db_version_major'] is None \
or app_config['counterpartyd_db_version_minor'] is None \
or app_config['counterpartyd_running_testnet'] is None:
updatePrefs = True
elif running_info['version_major'] != app_config[
'counterpartyd_db_version_major']:
logging.warn(
"counterpartyd MAJOR DB version change (we built from %s, counterpartyd is at %s). Wiping our state data."
% (app_config['counterpartyd_db_version_major'],
running_info['version_major']))
wipeState = True
updatePrefs = True
elif running_info['version_minor'] != app_config[
'counterpartyd_db_version_minor']:
logging.warn(
"counterpartyd MINOR DB version change (we built from %s.%s, counterpartyd is at %s.%s). Wiping our state data."
%
(app_config['counterpartyd_db_version_major'],
app_config['counterpartyd_db_version_minor'],
running_info['version_major'], running_info['version_minor']))
wipeState = True
updatePrefs = True
elif running_info.get(
'running_testnet',
False) != app_config['counterpartyd_running_testnet']:
logging.warn(
"counterpartyd testnet setting change (from %s to %s). Wiping our state data."
% (app_config['counterpartyd_running_testnet'],
running_info['running_testnet']))
wipeState = True
updatePrefs = True
if wipeState:
app_config = blow_away_db()
if updatePrefs:
app_config['counterpartyd_db_version_major'] = running_info[
'version_major']
app_config['counterpartyd_db_version_minor'] = running_info[
'version_minor']
app_config['counterpartyd_running_testnet'] = running_info[
'running_testnet']
mongo_db.app_config.update({}, app_config)
#reset my latest block record
my_latest_block = LATEST_BLOCK_INIT
config.CAUGHT_UP = False #You've Come a Long Way, Baby
#work up to what block counterpartyd is at
last_processed_block = running_info['last_block']
if last_processed_block['block_index'] is None:
logging.warn(
"counterpartyd has no last processed block (probably is reparsing). Waiting 3 seconds before trying again..."
)
time.sleep(3)
continue
if my_latest_block['block_index'] < last_processed_block['block_index']:
#need to catch up
config.CAUGHT_UP = False
if last_processed_block['block_index'] - my_latest_block[
'block_index'] > 500: #we are safely far from the tip, switch to bulk-everything
autopilot = True
if autopilot_runner == 0:
autopilot_runner = 500
autopilot_runner -= 1
else:
autopilot = False
cur_block_index = my_latest_block['block_index'] + 1
try:
cur_block = util.get_block_info_cached(
cur_block_index,
min(
200, last_processed_block['block_index'] -
my_latest_block['block_index']))
block_data = cur_block['_messages']
except Exception, e:
logging.warn(
str(e) + " Waiting 3 seconds before trying again...")
time.sleep(3)
continue
cur_block['block_time_obj'] = datetime.datetime.utcfromtimestamp(
cur_block['block_time'])
cur_block['block_time_str'] = cur_block[
'block_time_obj'].isoformat()
# clean api cache
if last_processed_block[
'block_index'] - cur_block_index <= config.MAX_REORG_NUM_BLOCKS: #only when we are near the tip
util.clean_block_cache(cur_block_index)
#parse out response (list of txns, ordered as they appeared in the block)
for msg in block_data:
msg_data = json.loads(msg['bindings'])
if msg['message_index'] != config.LAST_MESSAGE_INDEX + 1 and config.LAST_MESSAGE_INDEX != -1:
logging.error(
"BUG: MESSAGE RECEIVED NOT WHAT WE EXPECTED. EXPECTED: %s, GOT: %s: %s (ALL MSGS IN get_messages PAYLOAD: %s)..."
% (config.LAST_MESSAGE_INDEX + 1, msg['message_index'],
msg, [m['message_index'] for m in block_data]))
# we are likely cojones deep in desync, enforcing deep reorg
my_latest_block = prune_my_stale_blocks(
cur_block_index - config.MAX_FORCED_REORG_NUM_BLOCKS)
break
#sys.exit(1) #FOR NOW
#BUG: sometimes counterpartyd seems to return OLD messages out of the message feed. deal with those
#TODO unreachable now, delete?
if msg['message_index'] <= config.LAST_MESSAGE_INDEX:
logging.warn("BUG: IGNORED old RAW message %s: %s ..." %
(msg['message_index'], msg))
continue
logging.info("Received message %s: %s ..." %
(msg['message_index'], msg))
#don't process invalid messages, but do forward them along to clients
status = msg_data.get('status', 'valid').lower()
if status.startswith('invalid'):
#(but don't forward along while we're catching up)
if last_processed_block['block_index'] - my_latest_block[
'block_index'] < config.MAX_REORG_NUM_BLOCKS:
event = util.decorate_message_for_feed(
msg, msg_data=msg_data)
zmq_publisher_eventfeed.send_json(event)
config.LAST_MESSAGE_INDEX = msg['message_index']
continue
#track message types, for compiling of statistics
if msg['command'] == 'insert' \
and msg['category'] not in ["debits", "credits", "order_matches", "bet_matches",
"order_expirations", "bet_expirations", "order_match_expirations", "bet_match_expirations",
"rps_matches", "rps_expirations", "rps_match_expirations", "bet_match_resolutions"]:
mongo_db.transaction_stats.insert({
'block_index':
cur_block_index,
'block_time':
cur_block['block_time_obj'],
'message_index':
msg['message_index'],
'category':
msg['category']
})
#HANDLE REORGS
if msg['command'] == 'reorg':
logging.warn("Blockchain reorginization at block %s" %
msg_data['block_index'])
#prune back to and including the specified message_index
my_latest_block = prune_my_stale_blocks(
msg_data['block_index'] - 1)
config.CURRENT_BLOCK_INDEX = msg_data['block_index'] - 1
#for the current last_message_index (which could have gone down after the reorg), query counterpartyd
running_info = util.call_jsonrpc_api(
"get_running_info", abort_on_error=True)['result']
config.LAST_MESSAGE_INDEX = running_info[
'last_message_index']
#send out the message to listening clients (but don't forward along while we're catching up)
if last_processed_block['block_index'] - my_latest_block[
'block_index'] < config.MAX_REORG_NUM_BLOCKS:
msg_data[
'_last_message_index'] = config.LAST_MESSAGE_INDEX
event = util.decorate_message_for_feed(
msg, msg_data=msg_data)
zmq_publisher_eventfeed.send_json(event)
break #break out of inner loop
#track assets
if msg['category'] == 'issuances':
assets.parse_issuance(mongo_db, msg_data, cur_block_index,
cur_block)
#track balance changes for each address
bal_change = None
if msg['category'] in [
'credits',
'debits',
]:
actionName = 'credit' if msg[
'category'] == 'credits' else 'debit'
address = msg_data['address']
asset_info = mongo_db.tracked_assets.find_one(
{'asset': msg_data['asset']})
if asset_info is None:
logging.warn(
"Credit/debit of %s where asset ('%s') does not exist. Ignoring..."
% (msg_data['quantity'], msg_data['asset']))
config.LAST_MESSAGE_INDEX = msg['message_index']
continue
quantity = msg_data['quantity'] if msg[
'category'] == 'credits' else -msg_data['quantity']
quantity_normalized = util_bitcoin.normalize_quantity(
quantity, asset_info['divisible'])
#look up the previous balance to go off of
last_bal_change = mongo_db.balance_changes.find_one(
{
'address': address,
'asset': asset_info['asset']
},
sort=[("block_index", pymongo.DESCENDING),
("_id", pymongo.DESCENDING)])
if last_bal_change \
and last_bal_change['block_index'] == cur_block_index:
#modify this record, as we want at most one entry per block index for each (address, asset) pair
last_bal_change['quantity'] += quantity
last_bal_change[
'quantity_normalized'] += quantity_normalized
last_bal_change['new_balance'] += quantity
last_bal_change[
'new_balance_normalized'] += quantity_normalized
mongo_db.balance_changes.save(last_bal_change)
logging.info(
"Procesed %s bal change (UPDATED) from tx %s :: %s"
% (actionName, msg['message_index'],
last_bal_change))
bal_change = last_bal_change
else: #new balance change record for this block
bal_change = {
'address':
address,
'asset':
asset_info['asset'],
'block_index':
cur_block_index,
'block_time':
cur_block['block_time_obj'],
'quantity':
quantity,
'quantity_normalized':
quantity_normalized,
'new_balance':
last_bal_change['new_balance'] +
quantity if last_bal_change else quantity,
'new_balance_normalized':
last_bal_change['new_balance_normalized'] +
quantity_normalized
if last_bal_change else quantity_normalized,
}
mongo_db.balance_changes.insert(bal_change)
logging.info(
"Procesed %s bal change from tx %s :: %s" %
(actionName, msg['message_index'], bal_change))
#book trades
if (msg['category'] == 'order_matches' and
((
msg['command'] == 'update'
and msg_data['status'] == 'completed'
) #for a trade with BTC involved, but that is settled (completed)
or ('forward_asset' in msg_data
and msg_data['forward_asset'] != config.BTC
and msg_data['backward_asset'] != config.BTC))
): #or for a trade without BTC on either end
if msg['command'] == 'update' and msg_data[
'status'] == 'completed':
#an order is being updated to a completed status (i.e. a BTCpay has completed)
tx0_hash, tx1_hash = msg_data[
'order_match_id'][:64], msg_data['order_match_id'][
64:]
#get the order_match this btcpay settles
order_match = util.call_jsonrpc_api(
"get_order_matches", {
'filters': [{
'field': 'tx0_hash',
'op': '==',
'value': tx0_hash
}, {
'field': 'tx1_hash',
'op': '==',
'value': tx1_hash
}]
},
abort_on_error=True)['result'][0]
else:
assert msg_data[
'status'] == 'completed' #should not enter a pending state for non BTC matches
order_match = msg_data
forward_asset_info = mongo_db.tracked_assets.find_one(
{'asset': order_match['forward_asset']})
backward_asset_info = mongo_db.tracked_assets.find_one(
{'asset': order_match['backward_asset']})
assert forward_asset_info and backward_asset_info
base_asset, quote_asset = util.assets_to_asset_pair(
order_match['forward_asset'],
order_match['backward_asset'])
#don't create trade records from order matches with BTC that are under the dust limit
if (order_match['forward_asset'] == config.BTC and order_match['forward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF) \
or (order_match['backward_asset'] == config.BTC and order_match['backward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF):
logging.debug(
"Order match %s ignored due to %s under dust limit."
% (order_match['tx0_hash'] +
order_match['tx1_hash'], config.BTC))
config.LAST_MESSAGE_INDEX = msg['message_index']
continue
#take divisible trade quantities to floating point
forward_quantity = util_bitcoin.normalize_quantity(
order_match['forward_quantity'],
forward_asset_info['divisible'])
backward_quantity = util_bitcoin.normalize_quantity(
order_match['backward_quantity'],
backward_asset_info['divisible'])
#compose trade
trade = {
'block_index':
cur_block_index,
'block_time':
cur_block['block_time_obj'],
'message_index':
msg['message_index'], #secondary temporaral ordering off of when
'order_match_id':
order_match['tx0_hash'] + order_match['tx1_hash'],
'order_match_tx0_index':
order_match['tx0_index'],
'order_match_tx1_index':
order_match['tx1_index'],
'order_match_tx0_address':
order_match['tx0_address'],
'order_match_tx1_address':
order_match['tx1_address'],
'base_asset':
base_asset,
'quote_asset':
quote_asset,
'base_quantity':
order_match['forward_quantity']
if order_match['forward_asset'] == base_asset else
order_match['backward_quantity'],
'quote_quantity':
order_match['backward_quantity']
if order_match['forward_asset'] == base_asset else
order_match['forward_quantity'],
'base_quantity_normalized':
forward_quantity if order_match['forward_asset']
== base_asset else backward_quantity,
'quote_quantity_normalized':
backward_quantity if order_match['forward_asset']
== base_asset else forward_quantity,
}
trade['unit_price'] = float(
(D(trade['quote_quantity_normalized']) /
D(trade['base_quantity_normalized'])).quantize(
D('.00000000'), rounding=decimal.ROUND_HALF_EVEN))
trade['unit_price_inverse'] = float(
(D(trade['base_quantity_normalized']) /
D(trade['quote_quantity_normalized'])).quantize(
D('.00000000'), rounding=decimal.ROUND_HALF_EVEN))
mongo_db.trades.insert(trade)
logging.info("Procesed Trade from tx %s :: %s" %
(msg['message_index'], trade))
#broadcast
if msg['category'] == 'broadcasts':
betting.parse_broadcast(mongo_db, msg_data)
#if we're catching up beyond MAX_REORG_NUM_BLOCKS blocks out, make sure not to send out any socket.io
# events, as to not flood on a resync (as we may give a 525 to kick the logged in clients out, but we
# can't guarantee that the socket.io connection will always be severed as well??)
if last_processed_block['block_index'] - my_latest_block[
'block_index'] < config.MAX_REORG_NUM_BLOCKS:
#send out the message to listening clients
event = util.decorate_message_for_feed(msg,
msg_data=msg_data)
zmq_publisher_eventfeed.send_json(event)
#this is the last processed message index
config.LAST_MESSAGE_INDEX = msg['message_index']
else:
#block successfully processed, track this in our DB
new_block = {
'block_index': cur_block_index,
'block_time': cur_block['block_time_obj'],
'block_hash': cur_block['block_hash'],
}
blocks_to_insert.append(new_block)
if last_processed_block[
'block_index'] - cur_block_index > 1000: #reparsing, do bulk inserts
if len(blocks_to_insert) >= 1000:
mongo_db.processed_blocks.insert(blocks_to_insert)
blocks_to_insert[:] = []
else:
mongo_db.processed_blocks.insert(blocks_to_insert)
blocks_to_insert[:] = []
my_latest_block = new_block
config.CURRENT_BLOCK_INDEX = cur_block_index
#get the current blockchain service block
if config.BLOCKCHAIN_SERVICE_LAST_BLOCK == 0 or config.BLOCKCHAIN_SERVICE_LAST_BLOCK - config.CURRENT_BLOCK_INDEX < config.MAX_REORG_NUM_BLOCKS:
#update as CURRENT_BLOCK_INDEX catches up with BLOCKCHAIN_SERVICE_LAST_BLOCK and/or surpasses it (i.e. if blockchain service gets behind for some reason)
try:
block_height_response = blockchain.getinfo()
except:
block_height_response = None
config.BLOCKCHAIN_SERVICE_LAST_BLOCK = block_height_response[
'info']['blocks'] if block_height_response else 0
logging.info(
"Block: %i (message_index height=%s) (blockchain latest block=%s)"
% (config.CURRENT_BLOCK_INDEX, config.LAST_MESSAGE_INDEX
if config.LAST_MESSAGE_INDEX != -1 else '???',
config.BLOCKCHAIN_SERVICE_LAST_BLOCK
if config.BLOCKCHAIN_SERVICE_LAST_BLOCK else '???'))
if last_processed_block[
'block_index'] - cur_block_index < config.MAX_REORG_NUM_BLOCKS: #only when we are near the tip
clean_mempool_tx()
def process_cpd_blockfeed():
LATEST_BLOCK_INIT = {
'block_index': config.BLOCK_FIRST,
'block_time': None,
'block_hash': None
}
mongo_db = config.mongo_db
def blow_away_db():
mongo_db.processed_blocks.drop()
mongo_db.tracked_assets.drop()
mongo_db.trades.drop()
mongo_db.balance_changes.drop()
mongo_db.asset_market_info.drop()
mongo_db.asset_marketcap_history.drop()
mongo_db.pair_market_info.drop()
mongo_db.btc_open_orders.drop()
mongo_db.asset_extended_info.drop()
mongo_db.transaction_stats.drop()
mongo_db.feeds.drop()
mongo_db.wallet_stats.drop()
mongo_db.wallet_messages.drop()
mongo_db.app_config.update(
{},
{
'db_version': config.DB_VERSION,
'running_testnet': config.TESTNET,
'counterpartyd_db_version_major': None,
'counterpartyd_db_version_minor': None,
'counterpartyd_running_testnet': None,
'last_block_assets_compiled': config.
BLOCK_FIRST, #for asset data compilation in events.py (resets on reparse as well)
},
upsert=True)
app_config = mongo_db.app_config.find()[0]
for asset in [config.XCP, config.BTC]:
base_asset = {
'asset': asset,
'owner': None,
'divisible': True,
'locked': False,
'total_issued': None,
'_at_block':
config.BLOCK_FIRST, #the block ID this asset is current for
'_history': [] #to allow for block rollbacks
}
mongo_db.tracked_assets.insert(base_asset)
mongo_db.wallet_messages.insert({
'_id': 0,
'when': calendar.timegm(time.gmtime()),
'message': None,
})
#reinitialize some internal counters
config.CURRENT_BLOCK_INDEX = 0
config.LAST_MESSAGE_INDEX = -1
config.cw_last_message_seq = 0
return app_config
def prune_my_stale_blocks(max_block_index):
assert isinstance(max_block_index, int)
if max_block_index <= config.BLOCK_FIRST:
max_block_index = config.BLOCK_FIRST + 1
if not mongo_db.processed_blocks.find_one(
{"block_index": max_block_index}):
raise Exception(
"Can't roll back to specified block index: %i doesn't exist in database"
% max_block_index)
logging.warn("Pruning to block %i ..." % (max_block_index))
mongo_db.processed_blocks.remove(
{"block_index": {
"$gt": max_block_index
}})
mongo_db.balance_changes.remove(
{"block_index": {
"$gt": max_block_index
}})
mongo_db.trades.remove({"block_index": {"$gt": max_block_index}})
mongo_db.asset_marketcap_history.remove(
{"block_index": {
"$gt": max_block_index
}})
mongo_db.transaction_stats.remove(
{"block_index": {
"$gt": max_block_index
}})
#to roll back the state of the tracked asset, dive into the history object for each asset that has
# been updated on or after the block that we are pruning back to
assets_to_prune = mongo_db.tracked_assets.find(
{'_at_block': {
"$gt": max_block_index
}})
for asset in assets_to_prune:
logging.info(
"Pruning asset %s (last modified @ block %i, pruning to state at block %i)"
% (asset['asset'], asset['_at_block'], max_block_index))
prev_ver = None
while len(asset['_history']):
prev_ver = asset['_history'].pop()
if prev_ver['_at_block'] <= max_block_index:
break
if prev_ver:
if prev_ver['_at_block'] > max_block_index:
#even the first history version is newer than max_block_index.
#in this case, just remove the asset tracking record itself
mongo_db.tracked_assets.remove({'asset': asset['asset']})
else:
#if here, we were able to find a previous version that was saved at or before max_block_index
# (which should be prev_ver ... restore asset's values to its values
prev_ver['_id'] = asset['_id']
prev_ver['_history'] = asset['_history']
mongo_db.tracked_assets.save(prev_ver)
config.LAST_MESSAGE_INDEX = -1
config.CAUGHT_UP = False
latest_block = mongo_db.processed_blocks.find_one(
{"block_index": max_block_index})
return latest_block
def publish_mempool_tx():
"""fetch new tx from mempool"""
tx_hashes = []
mempool_txs = mongo_db.mempool.find(projection={'tx_hash': True})
for mempool_tx in mempool_txs:
tx_hashes.append(str(mempool_tx['tx_hash']))
params = None
if len(tx_hashes) > 0:
params = {
'filters': [{
'field': 'tx_hash',
'op': 'NOT IN',
'value': tx_hashes
}, {
'field':
'category',
'op':
'IN',
'value': ['sends', 'btcpays', 'issuances', 'dividends']
}],
'filterop':
'AND'
}
new_txs = util.call_jsonrpc_api("get_mempool",
params,
abort_on_error=True)
for new_tx in new_txs['result']:
tx = {
'tx_hash': new_tx['tx_hash'],
'command': new_tx['command'],
'category': new_tx['category'],
'bindings': new_tx['bindings'],
'timestamp': new_tx['timestamp'],
'viewed_in_block': config.CURRENT_BLOCK_INDEX
}
mongo_db.mempool.insert(tx)
del (tx['_id'])
tx['_category'] = tx['category']
tx['_message_index'] = 'mempool'
logging.debug("Spotted mempool tx: %s" % tx)
util.store_wallet_message(tx,
json.loads(tx['bindings']),
decorate=False)
def clean_mempool_tx():
"""clean mempool transactions older than MAX_REORG_NUM_BLOCKS blocks"""
mongo_db.mempool.remove({
"viewed_in_block": {
"$lt": config.CURRENT_BLOCK_INDEX - config.MAX_REORG_NUM_BLOCKS
}
})
config.CURRENT_BLOCK_INDEX = 0
config.LAST_MESSAGE_INDEX = -1
config.BLOCKCHAIN_SERVICE_LAST_BLOCK = 0 #simply for printing/alerting purposes
config.CAUGHT_UP_STARTED_EVENTS = False
#^ set after we are caught up and start up the recurring events that depend on us being caught up with the blockchain
#grab our stored preferences, and rebuild the database if necessary
app_config = mongo_db.app_config.find()
assert app_config.count() in [0, 1]
if (app_config.count() == 0 or config.REPARSE_FORCED
or app_config[0]['db_version'] != config.DB_VERSION
or app_config[0]['running_testnet'] != config.TESTNET):
if app_config.count():
logging.warn(
"energyblockd database version UPDATED (from %i to %i) or testnet setting changed (from %s to %s), or REINIT forced (%s). REBUILDING FROM SCRATCH ..."
% (app_config[0]['db_version'], config.DB_VERSION,
app_config[0]['running_testnet'], config.TESTNET,
config.REPARSE_FORCED))
else:
logging.warn(
"energyblockd database app_config collection doesn't exist. BUILDING FROM SCRATCH..."
)
app_config = blow_away_db()
my_latest_block = LATEST_BLOCK_INIT
else:
app_config = app_config[0]
#get the last processed block out of mongo
my_latest_block = mongo_db.processed_blocks.find_one(
sort=[("block_index", pymongo.DESCENDING)])
if my_latest_block:
my_latest_block = prune_my_stale_blocks(
my_latest_block['block_index'])
else:
config.CURRENT_BLOCK_INDEX = LATEST_BLOCK_INIT
while True:
try:
running_info = util.call_jsonrpc_api("get_running_info",
abort_on_error=True)
if 'result' not in running_info:
raise AssertionError("Could not contact energypartyd")
running_info = running_info['result']
except Exception as e:
logging.warn(
str(e) + " -- Waiting 30 seconds before trying again...")
time.sleep(30)
continue
if running_info['last_message_index'] == -1:
logging.warn(
"No last_message_index returned. Waiting until energypartyd has messages..."
)
time.sleep(30)
continue
wipeState = False
updatePrefs = False
if app_config['counterpartyd_db_version_major'] is None \
or app_config['counterpartyd_db_version_minor'] is None \
or app_config['counterpartyd_running_testnet'] is None:
updatePrefs = True
elif running_info['version_major'] != app_config[
'counterpartyd_db_version_major']:
logging.warn(
"energypartyd MAJOR DB version change (we built from %s, energypartyd is at %s). Wiping our state data."
% (app_config['counterpartyd_db_version_major'],
running_info['version_major']))
wipeState = True
updatePrefs = True
elif running_info['version_minor'] != app_config[
'counterpartyd_db_version_minor']:
logging.warn(
"energypartyd MINOR DB version change (we built from %s.%s, energypartyd is at %s.%s). Wiping our state data."
%
(app_config['counterpartyd_db_version_major'],
app_config['counterpartyd_db_version_minor'],
running_info['version_major'], running_info['version_minor']))
wipeState = True
updatePrefs = True
elif running_info.get(
'running_testnet',
False) != app_config['counterpartyd_running_testnet']:
logging.warn(
"energypartyd testnet setting change (from %s to %s). Wiping our state data."
% (app_config['counterpartyd_running_testnet'],
running_info['running_testnet']))
wipeState = True
updatePrefs = True
if wipeState:
app_config = blow_away_db()
if updatePrefs:
app_config['counterpartyd_db_version_major'] = running_info[
'version_major']
app_config['counterpartyd_db_version_minor'] = running_info[
'version_minor']
app_config['counterpartyd_running_testnet'] = running_info[
'running_testnet']
mongo_db.app_config.update({}, app_config)
#reset my latest block record
my_latest_block = LATEST_BLOCK_INIT
config.CAUGHT_UP = False #You've Come a Long Way, Baby
last_processed_block = running_info['last_block']
if last_processed_block['block_index'] is None:
logging.warn(
"energypartyd has no last processed block (probably is reparsing). Waiting 5 seconds before trying again..."
)
time.sleep(5)
continue
if my_latest_block['block_index'] < last_processed_block['block_index']:
#need to catch up
config.CAUGHT_UP = False
cur_block_index = my_latest_block['block_index'] + 1
#get the blocktime for the next block we have to process
try:
cur_block = util.call_jsonrpc_api(
"get_block_info", {'block_index': cur_block_index},
abort_on_error=True)['result']
except Exception as e:
logging.warn(
str(e) + " Waiting 5 seconds before trying again...")
time.sleep(5)
continue
cur_block['block_time_obj'] = datetime.datetime.utcfromtimestamp(
cur_block['block_time'])
cur_block['block_time_str'] = cur_block[
'block_time_obj'].isoformat()
try:
block_data = util.call_jsonrpc_api(
"get_messages", {'block_index': cur_block_index},
abort_on_error=True)['result']
except Exception as e:
logging.warn(
str(e) + " Waiting 15 seconds before trying again...")
time.sleep(15)
continue
#parse out response (list of txns, ordered as they appeared in the block)
for msg in block_data:
msg_data = json.loads(msg['bindings'])
if msg['message_index'] != config.LAST_MESSAGE_INDEX + 1 and config.LAST_MESSAGE_INDEX != -1:
logging.error(
"BUG: MESSAGE RECEIVED NOT WHAT WE EXPECTED. EXPECTED: %s, GOT: %s: %s (ALL MSGS IN get_messages PAYLOAD: %s)..."
% (config.LAST_MESSAGE_INDEX + 1, msg['message_index'],
msg, [m['message_index'] for m in block_data]))
my_latest_block = prune_my_stale_blocks(
cur_block_index - config.MAX_FORCED_REORG_NUM_BLOCKS)
break
#sys.exit(1) #FOR NOW
if msg['message_index'] <= config.LAST_MESSAGE_INDEX:
logging.warn("BUG: IGNORED old RAW message %s: %s ..." %
(msg['message_index'], msg))
continue
logging.info("Received message %s: %s ..." %
(msg['message_index'], msg))
status = msg_data.get('status', 'valid').lower()
if status.startswith('invalid'):
if last_processed_block['block_index'] - my_latest_block[
'block_index'] < config.MAX_REORG_NUM_BLOCKS:
util.store_wallet_message(msg, msg_data)
config.LAST_MESSAGE_INDEX = msg['message_index']
continue
#track message types, for compiling of statistics
if msg['command'] == 'insert' \
and msg['category'] not in ["debits", "credits", "order_matches", "bet_matches",
"order_expirations", "bet_expirations", "order_match_expirations", "bet_match_expirations", "bet_match_resolutions"]:
try:
mongo_db.transaction_stats.insert({
'block_index':
cur_block_index,
'block_time':
cur_block['block_time_obj'],
'category':
msg['category']
})
except pymongo.errors.DuplicateKeyError as e:
logging.exception(e)
#HANDLE REORGS
if msg['command'] == 'reorg':
logging.warn("Blockchain reorginization at block %s" %
msg_data['block_index'])
#prune back to and including the specified message_index
my_latest_block = prune_my_stale_blocks(
msg_data['block_index'] - 1)
config.CURRENT_BLOCK_INDEX = msg_data['block_index'] - 1
running_info = util.call_jsonrpc_api(
"get_running_info", abort_on_error=True)['result']
config.LAST_MESSAGE_INDEX = running_info[
'last_message_index']
if last_processed_block['block_index'] - my_latest_block[
'block_index'] < config.MAX_REORG_NUM_BLOCKS:
msg_data[
'_last_message_index'] = config.LAST_MESSAGE_INDEX
util.store_wallet_message(msg, msg_data)
event = util.decorate_message_for_feed(
msg, msg_data=msg_data)
break #break out of inner loop
#track assets
if msg['category'] == 'issuances':
assets.parse_issuance(mongo_db, msg_data, cur_block_index,
cur_block)
#track balance changes for each address
bal_change = None
if msg['category'] in [
'credits',
'debits',
]:
actionName = 'credit' if msg[
'category'] == 'credits' else 'debit'
address = msg_data['address']
asset_info = mongo_db.tracked_assets.find_one(
{'asset': msg_data['asset']})
if asset_info is None:
logging.warn(
"Credit/debit of %s where asset ('%s') does not exist. Ignoring..."
% (msg_data['quantity'], msg_data['asset']))
continue
quantity = msg_data['quantity'] if msg[
'category'] == 'credits' else -msg_data['quantity']
quantity_normalized = util_bitcoin.normalize_quantity(
quantity, asset_info['divisible'])
#look up the previous balance to go off of
last_bal_change = mongo_db.balance_changes.find_one(
{
'address': address,
'asset': asset_info['asset']
},
sort=[("block_index", pymongo.DESCENDING),
("_id", pymongo.DESCENDING)])
if last_bal_change \
and last_bal_change['block_index'] == cur_block_index:
last_bal_change['quantity'] += quantity
last_bal_change[
'quantity_normalized'] += quantity_normalized
last_bal_change['new_balance'] += quantity
last_bal_change[
'new_balance_normalized'] += quantity_normalized
mongo_db.balance_changes.save(last_bal_change)
logging.info(
"Procesed %s bal change (UPDATED) from tx %s :: %s"
% (actionName, msg['message_index'],
last_bal_change))
bal_change = last_bal_change
else: #new balance change record for this block
bal_change = {
'address':
address,
'asset':
asset_info['asset'],
'block_index':
cur_block_index,
'block_time':
cur_block['block_time_obj'],
'quantity':
quantity,
'quantity_normalized':
quantity_normalized,
'new_balance':
last_bal_change['new_balance'] +
quantity if last_bal_change else quantity,
'new_balance_normalized':
last_bal_change['new_balance_normalized'] +
quantity_normalized
if last_bal_change else quantity_normalized,
}
mongo_db.balance_changes.insert(bal_change)
logging.info(
"Procesed %s bal change from tx %s :: %s" %
(actionName, msg['message_index'], bal_change))
#book trades
if (msg['category'] == 'order_matches'
and ((msg['command'] == 'update'
and msg_data['status'] == 'completed') or
('forward_asset' in msg_data
and msg_data['forward_asset'] != config.BTC
and msg_data['backward_asset'] != config.BTC))):
if msg['command'] == 'update' and msg_data[
'status'] == 'completed':
tx0_hash, tx1_hash = msg_data[
'order_match_id'][:64], msg_data['order_match_id'][
64:]
order_match = util.call_jsonrpc_api(
"get_order_matches", {
'filters': [{
'field': 'tx0_hash',
'op': '==',
'value': tx0_hash
}, {
'field': 'tx1_hash',
'op': '==',
'value': tx1_hash
}]
},
abort_on_error=False)['result'][0]
else:
assert msg_data['status'] == 'completed'
order_match = msg_data
forward_asset_info = mongo_db.tracked_assets.find_one(
{'asset': order_match['forward_asset']})
backward_asset_info = mongo_db.tracked_assets.find_one(
{'asset': order_match['backward_asset']})
assert forward_asset_info and backward_asset_info
base_asset, quote_asset = util.assets_to_asset_pair(
order_match['forward_asset'],
order_match['backward_asset'])
if (order_match['forward_asset'] == config.BTC and order_match['forward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF) \
or (order_match['backward_asset'] == config.BTC and order_match['backward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF):
logging.debug(
"Order match %s ignored due to %s under dust limit."
% (order_match['tx0_hash'] +
order_match['tx1_hash'], config.BTC))
continue
#take divisible trade quantities to floating point
forward_quantity = util_bitcoin.normalize_quantity(
order_match['forward_quantity'],
forward_asset_info['divisible'])
backward_quantity = util_bitcoin.normalize_quantity(
order_match['backward_quantity'],
backward_asset_info['divisible'])
#compose trade
trade = {
'block_index':
cur_block_index,
'block_time':
cur_block['block_time_obj'],
'message_index':
msg['message_index'], #secondary temporaral ordering off of when
'order_match_id':
order_match['tx0_hash'] + '_' +
order_match['tx1_hash'],
'order_match_tx0_index':
order_match['tx0_index'],
'order_match_tx1_index':
order_match['tx1_index'],
'order_match_tx0_address':
order_match['tx0_address'],
'order_match_tx1_address':
order_match['tx1_address'],
'base_asset':
base_asset,
'quote_asset':
quote_asset,
'base_quantity':
order_match['forward_quantity']
if order_match['forward_asset'] == base_asset else
order_match['backward_quantity'],
'quote_quantity':
order_match['backward_quantity']
if order_match['forward_asset'] == base_asset else
order_match['forward_quantity'],
'base_quantity_normalized':
forward_quantity if order_match['forward_asset']
== base_asset else backward_quantity,
'quote_quantity_normalized':
backward_quantity if order_match['forward_asset']
== base_asset else forward_quantity,
}
d = D(trade['quote_quantity_normalized']) / D(
trade['base_quantity_normalized'])
d = d.quantize(EIGHT_PLACES,
rounding=decimal.ROUND_HALF_EVEN,
context=decimal.Context(prec=20))
trade['unit_price'] = float(d)
d = D(trade['base_quantity_normalized']) / D(
trade['quote_quantity_normalized'])
d = d.quantize(EIGHT_PLACES,
rounding=decimal.ROUND_HALF_EVEN,
context=decimal.Context(prec=20))
trade['unit_price_inverse'] = float(d)
mongo_db.trades.insert(trade)
logging.info("Procesed Trade from tx %s :: %s" %
(msg['message_index'], trade))
#broadcast
if msg['category'] == 'broadcasts':
betting.parse_broadcast(mongo_db, msg_data)
if last_processed_block['block_index'] - my_latest_block[
'block_index'] < config.MAX_REORG_NUM_BLOCKS:
#send out the message to listening clients
util.store_wallet_message(msg, msg_data)
#this is the last processed message index
config.LAST_MESSAGE_INDEX = msg['message_index']
new_block = {
'block_index': cur_block_index,
'block_time': cur_block['block_time_obj'],
'block_hash': cur_block['block_hash'],
}
mongo_db.processed_blocks.insert(new_block)
my_latest_block = new_block
config.CURRENT_BLOCK_INDEX = cur_block_index
if config.BLOCKCHAIN_SERVICE_LAST_BLOCK == 0 or config.BLOCKCHAIN_SERVICE_LAST_BLOCK - config.CURRENT_BLOCK_INDEX < config.MAX_REORG_NUM_BLOCKS:
try:
block_height_response = blockchain.getinfo()
except:
block_height_response = None
config.BLOCKCHAIN_SERVICE_LAST_BLOCK = block_height_response[
'info']['blocks'] if block_height_response else 0
logging.info(
"Block: %i (message_index height=%s) (blockchain latest block=%s)"
% (config.CURRENT_BLOCK_INDEX, config.LAST_MESSAGE_INDEX
if config.LAST_MESSAGE_INDEX != -1 else '???',
config.BLOCKCHAIN_SERVICE_LAST_BLOCK
if config.BLOCKCHAIN_SERVICE_LAST_BLOCK else '???'))
clean_mempool_tx()
elif my_latest_block['block_index'] > last_processed_block[
'block_index']:
logging.error(
"Very odd: Ahead of energypartyd with block indexes! Pruning back %s blocks to be safe."
% config.MAX_REORG_NUM_BLOCKS)
my_latest_block = prune_my_stale_blocks(
last_processed_block['block_index'] -
config.MAX_REORG_NUM_BLOCKS)
else:
config.CAUGHT_UP = running_info['db_caught_up']
if config.LAST_MESSAGE_INDEX == -1 or config.CURRENT_BLOCK_INDEX == 0:
if config.LAST_MESSAGE_INDEX == -1:
config.LAST_MESSAGE_INDEX = running_info[
'last_message_index']
if config.CURRENT_BLOCK_INDEX == 0:
config.CURRENT_BLOCK_INDEX = running_info['last_block'][
'block_index']
logging.info(
"Detected blocks caught up on startup. Setting last message idx to %s, current block index to %s ..."
% (config.LAST_MESSAGE_INDEX, config.CURRENT_BLOCK_INDEX))
if config.CAUGHT_UP and not config.CAUGHT_UP_STARTED_EVENTS:
logging.debug(
"Starting event timer: compile_asset_pair_market_info")
gevent.spawn(events.compile_asset_pair_market_info)
logging.debug(
"Starting event timer: compile_asset_market_info")
gevent.spawn(events.compile_asset_market_info)
logging.debug(
"Starting event timer: compile_extended_asset_info")
gevent.spawn(events.compile_extended_asset_info)
logging.debug(
"Starting event timer: compile_extended_feed_info")
gevent.spawn(events.compile_extended_feed_info)
config.CAUGHT_UP_STARTED_EVENTS = True
publish_mempool_tx()
time.sleep(30)
#track assets
if msg['category'] == 'issuances':
assets.parse_issuance(mongo_db, msg_data, cur_block_index, cur_block)
#track balance changes for each address
bal_change = None
if msg['category'] in ['credits', 'debits',]:
actionName = 'credit' if msg['category'] == 'credits' else 'debit'
address = msg_data['address']
asset_info = mongo_db.tracked_assets.find_one({ 'asset': msg_data['asset'] })
if asset_info is None:
logging.warn("Credit/debit of %s where asset ('%s') does not exist. Ignoring..." % (msg_data['quantity'], msg_data['asset']))
continue
quantity = msg_data['quantity'] if msg['category'] == 'credits' else -msg_data['quantity']
quantity_normalized = util_bitcoin.normalize_quantity(quantity, asset_info['divisible'])
#look up the previous balance to go off of
last_bal_change = mongo_db.balance_changes.find_one({
'address': address,
'asset': asset_info['asset']
}, sort=[("block_index", pymongo.DESCENDING), ("_id", pymongo.DESCENDING)])
if last_bal_change \
and last_bal_change['block_index'] == cur_block_index:
#modify this record, as we want at most one entry per block index for each (address, asset) pair
last_bal_change['quantity'] += quantity
last_bal_change['quantity_normalized'] += quantity_normalized
last_bal_change['new_balance'] += quantity
last_bal_change['new_balance_normalized'] += quantity_normalized
mongo_db.balance_changes.save(last_bal_change)
def compile_asset_pair_market_info():
"""Compiles the pair-level statistics that show on the View Prices page of counterwallet, for instance"""
#loop through all open orders, and compile a listing of pairs, with a count of open orders for each pair
mongo_db = config.mongo_db
end_dt = datetime.datetime.utcnow()
start_dt = end_dt - datetime.timedelta(days=1)
start_block_index, end_block_index = util.get_block_indexes_for_dates(
start_dt=start_dt, end_dt=end_dt)
open_orders = util.call_jsonrpc_api("get_orders", {
'filters': [
{
'field': 'give_remaining',
'op': '>',
'value': 0
},
{
'field': 'get_remaining',
'op': '>',
'value': 0
},
{
'field': 'fee_required_remaining',
'op': '>=',
'value': 0
},
{
'field': 'fee_provided_remaining',
'op': '>=',
'value': 0
},
],
'status':
'open',
'show_expired':
False,
},
abort_on_error=True)['result']
pair_data = {}
asset_info = {}
def get_price(base_quantity_normalized, quote_quantity_normalized):
return float(D(quote_quantity_normalized / base_quantity_normalized))
#COMPOSE order depth, lowest ask, and highest bid column data
for o in open_orders:
(base_asset,
quote_asset) = util.assets_to_asset_pair(o['give_asset'],
o['get_asset'])
pair = '%s/%s' % (base_asset, quote_asset)
base_asset_info = asset_info.get(
base_asset, mongo_db.tracked_assets.find_one({'asset':
base_asset}))
if base_asset not in asset_info:
asset_info[base_asset] = base_asset_info
quote_asset_info = asset_info.get(
quote_asset,
mongo_db.tracked_assets.find_one({'asset': quote_asset}))
if quote_asset not in asset_info:
asset_info[quote_asset] = quote_asset_info
pair_data.setdefault(
pair, {
'open_orders_count': 0,
'lowest_ask': None,
'highest_bid': None,
'completed_trades_count': 0,
'vol_base': 0,
'vol_quote': 0
})
#^ highest ask = open order selling base, highest bid = open order buying base
#^ we also initialize completed_trades_count, vol_base, vol_quote because every pair inited here may
# not have cooresponding data out of the trades_data_by_pair aggregation below
pair_data[pair]['open_orders_count'] += 1
base_quantity_normalized = util_bitcoin.normalize_quantity(
o['give_quantity'] if base_asset == o['give_asset'] else
o['get_quantity'], base_asset_info['divisible'])
quote_quantity_normalized = util_bitcoin.normalize_quantity(
o['give_quantity'] if quote_asset == o['give_asset'] else
o['get_quantity'], quote_asset_info['divisible'])
order_price = get_price(base_quantity_normalized,
quote_quantity_normalized)
if base_asset == o['give_asset']: #selling base
if pair_data[pair]['lowest_ask'] is None or order_price < pair_data[
pair]['lowest_ask']:
pair_data[pair]['lowest_ask'] = order_price
elif base_asset == o['get_asset']: #buying base
if pair_data[pair][
'highest_bid'] is None or order_price > pair_data[pair][
'highest_bid']:
pair_data[pair]['highest_bid'] = order_price
#COMPOSE volume data (in XDP and DOGE), and % change data
#loop through all trade volume over the past 24h, and match that to the open orders
trades_data_by_pair = mongo_db.trades.aggregate([
{
"$match": {
"block_time": {
"$gte": start_dt,
"$lte": end_dt
}
}
},
{
"$project": {
"base_asset": 1,
"quote_asset": 1,
"base_quantity_normalized": 1, #to derive base volume
"quote_quantity_normalized": 1 #to derive quote volume
}
},
{
"$group": {
"_id": {
"base_asset": "$base_asset",
"quote_asset": "$quote_asset"
},
"vol_base": {
"$sum": "$base_quantity_normalized"
},
"vol_quote": {
"$sum": "$quote_quantity_normalized"
},
"count": {
"$sum": 1
},
}
}
])
trades_data_by_pair = [] if not trades_data_by_pair[
'ok'] else trades_data_by_pair['result']
for e in trades_data_by_pair:
pair = '%s/%s' % (e['_id']['base_asset'], e['_id']['quote_asset'])
pair_data.setdefault(pair, {
'open_orders_count': 0,
'lowest_ask': None,
'highest_bid': None
})
#^ initialize an empty pair in the event there are no open orders for that pair, but there ARE completed trades for it
pair_data[pair]['completed_trades_count'] = e['count']
pair_data[pair]['vol_base'] = e['vol_base']
pair_data[pair]['vol_quote'] = e['vol_quote']
#compose price data, relative to DOGE and XDP
mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives()
for pair, e in pair_data.iteritems():
base_asset, quote_asset = pair.split('/')
_24h_vol_in_btc = None
_24h_vol_in_xcp = None
#derive asset price data, expressed in DOGE and XDP, for the given volumes
if base_asset == config.XCP:
_24h_vol_in_xcp = e['vol_base']
_24h_vol_in_btc = util_bitcoin.round_out(
e['vol_base'] * xcp_btc_price) if xcp_btc_price else 0
elif base_asset == config.BTC:
_24h_vol_in_xcp = util_bitcoin.round_out(
e['vol_base'] * btc_xcp_price) if btc_xcp_price else 0
_24h_vol_in_btc = e['vol_base']
else: #base is not XDP or DOGE
price_summary_in_xcp, price_summary_in_btc, price_in_xcp, price_in_btc, aggregated_price_in_xcp, aggregated_price_in_btc = \
get_xcp_btc_price_info(base_asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price, with_last_trades=0, start_dt=start_dt, end_dt=end_dt)
if price_in_xcp:
_24h_vol_in_xcp = util_bitcoin.round_out(e['vol_base'] *
price_in_xcp)
if price_in_btc:
_24h_vol_in_btc = util_bitcoin.round_out(e['vol_base'] *
price_in_btc)
if _24h_vol_in_xcp is None or _24h_vol_in_btc is None:
#the base asset didn't have price data against DOGE or XDP, or both...try against the quote asset instead
price_summary_in_xcp, price_summary_in_btc, price_in_xcp, price_in_btc, aggregated_price_in_xcp, aggregated_price_in_btc = \
get_xcp_btc_price_info(quote_asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price, with_last_trades=0, start_dt=start_dt, end_dt=end_dt)
if _24h_vol_in_xcp is None and price_in_xcp:
_24h_vol_in_xcp = util_bitcoin.round_out(e['vol_quote'] *
price_in_xcp)
if _24h_vol_in_btc is None and price_in_btc:
_24h_vol_in_btc = util_bitcoin.round_out(e['vol_quote'] *
price_in_btc)
pair_data[pair]['24h_vol_in_{}'.format(
config.XCP.lower())] = _24h_vol_in_xcp #might still be None
pair_data[pair]['24h_vol_in_{}'.format(
config.BTC.lower())] = _24h_vol_in_btc #might still be None
#get % change stats -- start by getting the first trade directly before the 24h period starts
prev_trade = mongo_db.trades.find({
"base_asset": base_asset,
"quote_asset": quote_asset,
"block_time": {
'$lt': start_dt
}
}).sort('block_time', pymongo.DESCENDING).limit(1)
latest_trade = mongo_db.trades.find({
"base_asset": base_asset,
"quote_asset": quote_asset
}).sort('block_time', pymongo.DESCENDING).limit(1)
if not prev_trade.count(): #no previous trade before this 24hr period
pair_data[pair]['24h_pct_change'] = None
else:
prev_trade = prev_trade[0]
latest_trade = latest_trade[0]
prev_trade_price = get_price(
prev_trade['base_quantity_normalized'],
prev_trade['quote_quantity_normalized'])
latest_trade_price = get_price(
latest_trade['base_quantity_normalized'],
latest_trade['quote_quantity_normalized'])
pair_data[pair]['24h_pct_change'] = (
(latest_trade_price - prev_trade_price) /
prev_trade_price) * 100
pair_data[pair]['last_updated'] = end_dt
#print "PRODUCED", pair, pair_data[pair]
mongo_db.asset_pair_market_info.update(
{
'base_asset': base_asset,
'quote_asset': quote_asset
}, {"$set": pair_data[pair]},
upsert=True)
#remove any old pairs that were not just updated
mongo_db.asset_pair_market_info.remove({'last_updated': {'$lt': end_dt}})
logging.info("Recomposed 24h trade statistics for %i asset pairs: %s" %
(len(pair_data), ', '.join(pair_data.keys())))
def parse_issuance(db, message, cur_block_index, cur_block):
def modify_extended_asset_info(asset, description):
"""adds an asset to asset_extended_info collection if the description is a valid json link. or, if the link
is not a valid json link, will remove the asset entry from the table if it exists"""
if util.is_valid_url(description, suffix='.json', allow_no_protocol=True):
db.asset_extended_info.update({'asset': asset},
{'$set': {
'info_url': description,
'info_status': 'needfetch',
'fetch_info_retry': 0, # retry ASSET_MAX_RETRY times to fetch info from info_url
'info_data': {},
'errors': []
}}, upsert=True)
#^ valid info_status settings: needfetch, valid, invalid, error
#additional fields will be added later in events, once the asset info is pulled
else:
db.asset_extended_info.remove({ 'asset': asset })
#remove any saved asset image data
imagePath = os.path.join(config.DATA_DIR, config.SUBDIR_ASSET_IMAGES, asset + '.png')
if os.path.exists(imagePath):
os.remove(imagePath)
tracked_asset = db.tracked_assets.find_one(
{'asset': message['asset']}, {'_id': 0, '_history': 0})
#^ pulls the tracked asset without the _id and history fields. This may be None
if message['locked']: #lock asset
assert tracked_asset is not None
db.tracked_assets.update(
{'asset': message['asset']},
{"$set": {
'_at_block': cur_block_index,
'_at_block_time': cur_block['block_time_obj'],
'_change_type': 'locked',
'locked': True,
},
"$push": {'_history': tracked_asset } }, upsert=False)
logging.info("Locking asset %s" % (message['asset'],))
elif message['transfer']: #transfer asset
assert tracked_asset is not None
db.tracked_assets.update(
{'asset': message['asset']},
{"$set": {
'_at_block': cur_block_index,
'_at_block_time': cur_block['block_time_obj'],
'_change_type': 'transferred',
'owner': message['issuer'],
},
"$push": {'_history': tracked_asset } }, upsert=False)
logging.info("Transferring asset %s to address %s" % (message['asset'], message['issuer']))
elif message['quantity'] == 0 and tracked_asset is not None: #change description
db.tracked_assets.update(
{'asset': message['asset']},
{"$set": {
'_at_block': cur_block_index,
'_at_block_time': cur_block['block_time_obj'],
'_change_type': 'changed_description',
'description': message['description'],
},
"$push": {'_history': tracked_asset } }, upsert=False)
modify_extended_asset_info(message['asset'], message['description'])
logging.info("Changing description for asset %s to '%s'" % (message['asset'], message['description']))
else: #issue new asset or issue addition qty of an asset
if not tracked_asset: #new issuance
tracked_asset = {
'_change_type': 'created',
'_at_block': cur_block_index, #the block ID this asset is current for
'_at_block_time': cur_block['block_time_obj'],
#^ NOTE: (if there are multiple asset tracked changes updates in a single block for the same
# asset, the last one with _at_block == that block id in the history array is the
# final version for that asset at that block
'asset': message['asset'],
'owner': message['issuer'],
'description': message['description'],
'divisible': message['divisible'],
'locked': False,
'total_issued': message['quantity'],
'total_issued_normalized': util_bitcoin.normalize_quantity(message['quantity'], message['divisible']),
'_history': [] #to allow for block rollbacks
}
db.tracked_assets.insert(tracked_asset)
logging.info("Tracking new asset: %s" % message['asset'])
modify_extended_asset_info(message['asset'], message['description'])
else: #issuing additional of existing asset
assert tracked_asset is not None
db.tracked_assets.update(
{'asset': message['asset']},
{"$set": {
'_at_block': cur_block_index,
'_at_block_time': cur_block['block_time_obj'],
'_change_type': 'issued_more',
},
"$inc": {
'total_issued': message['quantity'],
'total_issued_normalized': util_bitcoin.normalize_quantity(message['quantity'], message['divisible'])
},
"$push": {'_history': tracked_asset} }, upsert=False)
logging.info("Adding additional %s quantity for asset %s" % (
util_bitcoin.normalize_quantity(message['quantity'], message['divisible']), message['asset']))
return True
assets.parse_issuance(mongo_db, msg_data, cur_block_index, cur_block)
# track balance changes for each address
bal_change = None
if msg["category"] in ["credits", "debits"]:
actionName = "credit" if msg["category"] == "credits" else "debit"
address = msg_data["address"]
asset_info = mongo_db.tracked_assets.find_one({"asset": msg_data["asset"]})
if asset_info is None:
logging.warn(
"Credit/debit of %s where asset ('%s') does not exist. Ignoring..."
% (msg_data["quantity"], msg_data["asset"])
)
continue
quantity = msg_data["quantity"] if msg["category"] == "credits" else -msg_data["quantity"]
quantity_normalized = util_bitcoin.normalize_quantity(quantity, asset_info["divisible"])
# look up the previous balance to go off of
last_bal_change = mongo_db.balance_changes.find_one(
{"address": address, "asset": asset_info["asset"]},
sort=[("block_index", pymongo.DESCENDING), ("_id", pymongo.DESCENDING)],
)
if last_bal_change and last_bal_change["block_index"] == cur_block_index:
# modify this record, as we want at most one entry per block index for each (address, asset) pair
last_bal_change["quantity"] += quantity
last_bal_change["quantity_normalized"] += quantity_normalized
last_bal_change["new_balance"] += quantity
last_bal_change["new_balance_normalized"] += quantity_normalized
mongo_db.balance_changes.save(last_bal_change)
logging.info(
def decorate_message(message, for_txn_history=False):
assert '_category' in message
mongo_db = config.mongo_db
if for_txn_history:
message['_command'] = 'insert' #history data doesn't include this
block_index = message[
'block_index'] if 'block_index' in message else message[
'tx1_block_index']
message['_block_time'] = get_block_time(block_index)
message['_tx_index'] = message[
'tx_index'] if 'tx_index' in message else message.get(
'tx1_index', None)
if message['_category'] in [
'bet_expirations', 'order_expirations',
'bet_match_expirations', 'order_match_expirations'
]:
message[
'_tx_index'] = 0 #add tx_index to all entries (so we can sort on it secondarily in history view), since these lack it
if message['_category'] in ['credits', 'debits']:
#find the last balance change on record
bal_change = mongo_db.balance_changes.find_one(
{
'address': message['address'],
'asset': message['asset']
},
sort=[("block_time", pymongo.DESCENDING)])
message['_quantity_normalized'] = abs(
bal_change['quantity_normalized']) if bal_change else None
message['_balance'] = bal_change['new_balance'] if bal_change else None
message['_balance_normalized'] = bal_change[
'new_balance_normalized'] if bal_change else None
if message['_category'] in [
'orders',
] and message['_command'] == 'insert':
get_asset_info = mongo_db.tracked_assets.find_one(
{'asset': message['get_asset']})
give_asset_info = mongo_db.tracked_assets.find_one(
{'asset': message['give_asset']})
message['_get_asset_divisible'] = get_asset_info[
'divisible'] if get_asset_info else None
message['_give_asset_divisible'] = give_asset_info[
'divisible'] if give_asset_info else None
if message['_category'] in [
'order_matches',
] and message['_command'] == 'insert':
forward_asset_info = mongo_db.tracked_assets.find_one(
{'asset': message['forward_asset']})
backward_asset_info = mongo_db.tracked_assets.find_one(
{'asset': message['backward_asset']})
message['_forward_asset_divisible'] = forward_asset_info[
'divisible'] if forward_asset_info else None
message['_backward_asset_divisible'] = backward_asset_info[
'divisible'] if backward_asset_info else None
if message['_category'] in [
'orders',
'order_matches',
]:
message['_btc_below_dust_limit'] = (
('forward_asset' in message
and message['forward_asset'] == config.BTC and
message['forward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF)
or ('backward_asset' in message
and message['backward_asset'] == config.BTC
and message['backward_quantity'] <=
config.ORDER_BTC_DUST_LIMIT_CUTOFF))
if message['_category'] in ['dividends', 'sends']:
asset_info = mongo_db.tracked_assets.find_one(
{'asset': message['asset']})
message['_divisible'] = asset_info['divisible'] if asset_info else None
if message['_category'] in [
'issuances',
]:
message['_quantity_normalized'] = util_bitcoin.normalize_quantity(
message['quantity'], message['divisible'])
return message
def process_cpd_blockfeed(zmq_publisher_eventfeed):
LATEST_BLOCK_INIT = {'block_index': config.BLOCK_FIRST, 'block_time': None, 'block_hash': None}
mongo_db = config.mongo_db
blocks_to_insert = []
def blow_away_db():
"""boom! blow away all applicable collections in mongo"""
mongo_db.processed_blocks.drop()
mongo_db.tracked_assets.drop()
mongo_db.trades.drop()
mongo_db.balance_changes.drop()
mongo_db.asset_market_info.drop()
mongo_db.asset_marketcap_history.drop()
mongo_db.pair_market_info.drop()
mongo_db.btc_open_orders.drop()
mongo_db.asset_extended_info.drop()
mongo_db.transaction_stats.drop()
mongo_db.feeds.drop()
mongo_db.wallet_stats.drop()
#create/update default app_config object
mongo_db.app_config.update({}, {
'db_version': config.DB_VERSION, #counterblockd database version
'running_testnet': config.TESTNET,
'clearinghoused_db_version_major': None,
'clearinghoused_db_version_minor': None,
'clearinghoused_running_testnet': None,
'last_block_assets_compiled': config.BLOCK_FIRST, #for asset data compilation in events.py (resets on reparse as well)
}, upsert=True)
app_config = mongo_db.app_config.find()[0]
#DO NOT DELETE preferences and chat_handles and chat_history
#create XCP and BTC assets in tracked_assets
for asset in [config.XCP, config.BTC]:
base_asset = {
'asset': asset,
'owner': None,
'divisible': True,
'locked': False,
'total_issued': None,
'_at_block': config.BLOCK_FIRST, #the block ID this asset is current for
'_history': [] #to allow for block rollbacks
}
mongo_db.tracked_assets.insert(base_asset)
#reinitialize some internal counters
config.CURRENT_BLOCK_INDEX = 0
config.LAST_MESSAGE_INDEX = -1
return app_config
def prune_my_stale_blocks(max_block_index):
"""called if there are any records for blocks higher than this in the database? If so, they were impartially created
and we should get rid of them
NOTE: after calling this function, you should always trigger a "continue" statement to reiterate the processing loop
(which will get a new last_processed_block from clearinghoused and resume as appropriate)
"""
logging.warn("Pruning to block %i ..." % (max_block_index))
mongo_db.processed_blocks.remove({"block_index": {"$gt": max_block_index}})
mongo_db.balance_changes.remove({"block_index": {"$gt": max_block_index}})
mongo_db.trades.remove({"block_index": {"$gt": max_block_index}})
mongo_db.asset_marketcap_history.remove({"block_index": {"$gt": max_block_index}})
mongo_db.transaction_stats.remove({"block_index": {"$gt": max_block_index}})
#to roll back the state of the tracked asset, dive into the history object for each asset that has
# been updated on or after the block that we are pruning back to
assets_to_prune = mongo_db.tracked_assets.find({'_at_block': {"$gt": max_block_index}})
for asset in assets_to_prune:
logging.info("Pruning asset %s (last modified @ block %i, pruning to state at block %i)" % (
asset['asset'], asset['_at_block'], max_block_index))
prev_ver = None
while len(asset['_history']):
prev_ver = asset['_history'].pop()
if prev_ver['_at_block'] <= max_block_index:
break
if prev_ver:
if prev_ver['_at_block'] > max_block_index:
#even the first history version is newer than max_block_index.
#in this case, just remove the asset tracking record itself
mongo_db.tracked_assets.remove({'asset': asset['asset']})
else:
#if here, we were able to find a previous version that was saved at or before max_block_index
# (which should be prev_ver ... restore asset's values to its values
prev_ver['_id'] = asset['_id']
prev_ver['_history'] = asset['_history']
mongo_db.tracked_assets.save(prev_ver)
config.LAST_MESSAGE_INDEX = -1
config.CAUGHT_UP = False
util.blockinfo_cache.clear()
latest_block = mongo_db.processed_blocks.find_one({"block_index": max_block_index}) or LATEST_BLOCK_INIT
return latest_block
def publish_mempool_tx():
"""fetch new tx from mempool"""
tx_hashes = []
mempool_txs = mongo_db.mempool.find(fields={'tx_hash': True})
for mempool_tx in mempool_txs:
tx_hashes.append(str(mempool_tx['tx_hash']))
params = None
if len(tx_hashes) > 0:
params = {
'filters': [
{'field':'tx_hash', 'op': 'NOT IN', 'value': tx_hashes},
{'field':'category', 'op': 'IN', 'value': ['sends', 'btcpays', 'issuances', 'dividends', 'callbacks']}
],
'filterop': 'AND'
}
new_txs = util.call_jsonrpc_api("get_mempool", params, abort_on_error=True)
for new_tx in new_txs['result']:
tx = {
'tx_hash': new_tx['tx_hash'],
'command': new_tx['command'],
'category': new_tx['category'],
'bindings': new_tx['bindings'],
'timestamp': new_tx['timestamp'],
'viewed_in_block': config.CURRENT_BLOCK_INDEX
}
mongo_db.mempool.insert(tx)
del(tx['_id'])
tx['_category'] = tx['category']
tx['_message_index'] = 'mempool'
logging.debug("Spotted mempool tx: %s" % tx)
zmq_publisher_eventfeed.send_json(tx)
def clean_mempool_tx():
"""clean mempool transactions older than MAX_REORG_NUM_BLOCKS blocks"""
mongo_db.mempool.remove({"viewed_in_block": {"$lt": config.CURRENT_BLOCK_INDEX - config.MAX_REORG_NUM_BLOCKS}})
config.CURRENT_BLOCK_INDEX = 0 #initialize (last processed block index -- i.e. currently active block)
config.LAST_MESSAGE_INDEX = -1 #initialize (last processed message index)
config.BLOCKCHAIN_SERVICE_LAST_BLOCK = 0 #simply for printing/alerting purposes
config.CAUGHT_UP_STARTED_EVENTS = False
#^ set after we are caught up and start up the recurring events that depend on us being caught up with the blockchain
#grab our stored preferences, and rebuild the database if necessary
app_config = mongo_db.app_config.find()
assert app_config.count() in [0, 1]
if ( app_config.count() == 0
or config.REPARSE_FORCED
or app_config[0]['db_version'] != config.DB_VERSION
or app_config[0]['running_testnet'] != config.TESTNET):
if app_config.count():
logging.warn("clearblockd database version UPDATED (from %i to %i) or testnet setting changed (from %s to %s), or REINIT forced (%s). REBUILDING FROM SCRATCH ..." % (
app_config[0]['db_version'], config.DB_VERSION, app_config[0]['running_testnet'], config.TESTNET, config.REPARSE_FORCED))
else:
logging.warn("clearblockd database app_config collection doesn't exist. BUILDING FROM SCRATCH...")
app_config = blow_away_db()
my_latest_block = LATEST_BLOCK_INIT
else:
app_config = app_config[0]
#get the last processed block out of mongo
my_latest_block = mongo_db.processed_blocks.find_one(sort=[("block_index", pymongo.DESCENDING)]) or LATEST_BLOCK_INIT
#remove any data we have for blocks higher than this (would happen if counterblockd or mongo died
# or errored out while processing a block)
my_latest_block = prune_my_stale_blocks(my_latest_block['block_index'])
#avoid contacting clearinghoused (on reparse, to speed up)
autopilot = False
autopilot_runner = 0
#start polling clearinghoused for new blocks
while True:
if not autopilot or autopilot_runner == 0:
try:
running_info = util.call_jsonrpc_api("get_running_info", abort_on_error=True)
if 'result' not in running_info:
raise AssertionError("Could not contact clearinghoused")
running_info = running_info['result']
except Exception, e:
logging.warn(str(e) + " -- Waiting 3 seconds before trying again...")
time.sleep(3)
continue
if running_info['last_message_index'] == -1: #last_message_index not set yet (due to no messages in clearinghoused DB yet)
logging.warn("No last_message_index returned. Waiting until clearinghoused has messages...")
time.sleep(10)
continue
#wipe our state data if necessary, if clearinghoused has moved on to a new DB version
wipeState = False
updatePrefs = False
if app_config['clearinghoused_db_version_major'] is None \
or app_config['clearinghoused_db_version_minor'] is None \
or app_config['clearinghoused_running_testnet'] is None:
updatePrefs = True
elif running_info['version_major'] != app_config['clearinghoused_db_version_major']:
logging.warn("clearinghoused MAJOR DB version change (we built from %s, clearinghoused is at %s). Wiping our state data." % (
app_config['clearinghoused_db_version_major'], running_info['version_major']))
wipeState = True
updatePrefs = True
elif running_info['version_minor'] != app_config['clearinghoused_db_version_minor']:
logging.warn("clearinghoused MINOR DB version change (we built from %s.%s, clearinghoused is at %s.%s). Wiping our state data." % (
app_config['clearinghoused_db_version_major'], app_config['clearinghoused_db_version_minor'],
running_info['version_major'], running_info['version_minor']))
wipeState = True
updatePrefs = True
elif running_info.get('running_testnet', False) != app_config['clearinghoused_running_testnet']:
logging.warn("clearinghoused testnet setting change (from %s to %s). Wiping our state data." % (
app_config['clearinghoused_running_testnet'], running_info['running_testnet']))
wipeState = True
updatePrefs = True
if wipeState:
app_config = blow_away_db()
if updatePrefs:
app_config['clearinghoused_db_version_major'] = running_info['version_major']
app_config['clearinghoused_db_version_minor'] = running_info['version_minor']
app_config['clearinghoused_running_testnet'] = running_info['running_testnet']
mongo_db.app_config.update({}, app_config)
#reset my latest block record
my_latest_block = LATEST_BLOCK_INIT
config.CAUGHT_UP = False #You've Come a Long Way, Baby
#work up to what block clearinghoused is at
last_processed_block = running_info['last_block']
if last_processed_block['block_index'] is None:
logging.warn("clearinghoused has no last processed block (probably is reparsing). Waiting 3 seconds before trying again...")
time.sleep(3)
continue
if my_latest_block['block_index'] < last_processed_block['block_index']:
#need to catch up
config.CAUGHT_UP = False
if last_processed_block['block_index'] - my_latest_block['block_index'] > 500: #we are safely far from the tip, switch to bulk-everything
autopilot = True
if autopilot_runner == 0:
autopilot_runner = 500
autopilot_runner -= 1
else:
autopilot = False
cur_block_index = my_latest_block['block_index'] + 1
try:
cur_block = util.get_block_info_cached(cur_block_index, min(200, last_processed_block['block_index'] - my_latest_block['block_index']))
block_data = cur_block['_messages']
except Exception, e:
logging.warn(str(e) + " Waiting 3 seconds before trying again...")
time.sleep(3)
continue
cur_block['block_time_obj'] = datetime.datetime.utcfromtimestamp(cur_block['block_time'])
cur_block['block_time_str'] = cur_block['block_time_obj'].isoformat()
# clean api cache
if last_processed_block['block_index'] - cur_block_index <= config.MAX_REORG_NUM_BLOCKS: #only when we are near the tip
util.clean_block_cache(cur_block_index)
#parse out response (list of txns, ordered as they appeared in the block)
for msg in block_data:
msg_data = json.loads(msg['bindings'])
if msg['message_index'] != config.LAST_MESSAGE_INDEX + 1 and config.LAST_MESSAGE_INDEX != -1:
logging.error("BUG: MESSAGE RECEIVED NOT WHAT WE EXPECTED. EXPECTED: %s, GOT: %s: %s (ALL MSGS IN get_messages PAYLOAD: %s)..." % (
config.LAST_MESSAGE_INDEX + 1, msg['message_index'], msg, [m['message_index'] for m in block_data]))
# we are likely cojones deep in desync, enforcing deep reorg
my_latest_block = prune_my_stale_blocks(cur_block_index - config.MAX_FORCED_REORG_NUM_BLOCKS)
break
#sys.exit(1) #FOR NOW
#BUG: sometimes clearinghoused seems to return OLD messages out of the message feed. deal with those
#TODO unreachable now, delete?
if msg['message_index'] <= config.LAST_MESSAGE_INDEX:
logging.warn("BUG: IGNORED old RAW message %s: %s ..." % (msg['message_index'], msg))
continue
logging.info("Received message %s: %s ..." % (msg['message_index'], msg))
#don't process invalid messages, but do forward them along to clients
status = msg_data.get('status', 'valid').lower()
if status.startswith('invalid'):
#(but don't forward along while we're catching up)
if last_processed_block['block_index'] - my_latest_block['block_index'] < config.MAX_REORG_NUM_BLOCKS:
event = util.decorate_message_for_feed(msg, msg_data=msg_data)
zmq_publisher_eventfeed.send_json(event)
config.LAST_MESSAGE_INDEX = msg['message_index']
continue
#track message types, for compiling of statistics
if msg['command'] == 'insert' \
and msg['category'] not in ["debits", "credits", "order_matches", "bet_matches",
"order_expirations", "bet_expirations", "order_match_expirations", "bet_match_expirations",
"rps_matches", "rps_expirations", "rps_match_expirations", "bet_match_resolutions"]:
mongo_db.transaction_stats.insert({
'block_index': cur_block_index,
'block_time': cur_block['block_time_obj'],
'message_index': msg['message_index'],
'category': msg['category']
})
#HANDLE REORGS
if msg['command'] == 'reorg':
logging.warn("Blockchain reorginization at block %s" % msg_data['block_index'])
#prune back to and including the specified message_index
my_latest_block = prune_my_stale_blocks(msg_data['block_index'] - 1)
config.CURRENT_BLOCK_INDEX = msg_data['block_index'] - 1
#for the current last_message_index (which could have gone down after the reorg), query clearinghoused
running_info = util.call_jsonrpc_api("get_running_info", abort_on_error=True)['result']
config.LAST_MESSAGE_INDEX = running_info['last_message_index']
#send out the message to listening clients (but don't forward along while we're catching up)
if last_processed_block['block_index'] - my_latest_block['block_index'] < config.MAX_REORG_NUM_BLOCKS:
msg_data['_last_message_index'] = config.LAST_MESSAGE_INDEX
event = util.decorate_message_for_feed(msg, msg_data=msg_data)
zmq_publisher_eventfeed.send_json(event)
break #break out of inner loop
#track assets
if msg['category'] == 'issuances':
assets.parse_issuance(mongo_db, msg_data, cur_block_index, cur_block)
#track balance changes for each address
bal_change = None
if msg['category'] in ['credits', 'debits',]:
actionName = 'credit' if msg['category'] == 'credits' else 'debit'
address = msg_data['address']
asset_info = mongo_db.tracked_assets.find_one({ 'asset': msg_data['asset'] })
if asset_info is None:
logging.warn("Credit/debit of %s where asset ('%s') does not exist. Ignoring..." % (msg_data['quantity'], msg_data['asset']))
continue
quantity = msg_data['quantity'] if msg['category'] == 'credits' else -msg_data['quantity']
quantity_normalized = util_bitcoin.normalize_quantity(quantity, asset_info['divisible'])
#look up the previous balance to go off of
last_bal_change = mongo_db.balance_changes.find_one({
'address': address,
'asset': asset_info['asset']
}, sort=[("block_index", pymongo.DESCENDING), ("_id", pymongo.DESCENDING)])
if last_bal_change \
and last_bal_change['block_index'] == cur_block_index:
#modify this record, as we want at most one entry per block index for each (address, asset) pair
last_bal_change['quantity'] += quantity
last_bal_change['quantity_normalized'] += quantity_normalized
last_bal_change['new_balance'] += quantity
last_bal_change['new_balance_normalized'] += quantity_normalized
mongo_db.balance_changes.save(last_bal_change)
logging.info("Procesed %s bal change (UPDATED) from tx %s :: %s" % (actionName, msg['message_index'], last_bal_change))
bal_change = last_bal_change
else: #new balance change record for this block
bal_change = {
'address': address,
'asset': asset_info['asset'],
'block_index': cur_block_index,
'block_time': cur_block['block_time_obj'],
'quantity': quantity,
'quantity_normalized': quantity_normalized,
'new_balance': last_bal_change['new_balance'] + quantity if last_bal_change else quantity,
'new_balance_normalized': last_bal_change['new_balance_normalized'] + quantity_normalized if last_bal_change else quantity_normalized,
}
mongo_db.balance_changes.insert(bal_change)
logging.info("Procesed %s bal change from tx %s :: %s" % (actionName, msg['message_index'], bal_change))
#book trades
if (msg['category'] == 'order_matches'
and ((msg['command'] == 'update' and msg_data['status'] == 'completed') #for a trade with BTC involved, but that is settled (completed)
or ('forward_asset' in msg_data and msg_data['forward_asset'] != config.BTC and msg_data['backward_asset'] != config.BTC))): #or for a trade without BTC on either end
if msg['command'] == 'update' and msg_data['status'] == 'completed':
#an order is being updated to a completed status (i.e. a BTCpay has completed)
tx0_hash, tx1_hash = msg_data['order_match_id'][:64], msg_data['order_match_id'][64:]
#get the order_match this btcpay settles
order_match = util.call_jsonrpc_api("get_order_matches",
{'filters': [
{'field': 'tx0_hash', 'op': '==', 'value': tx0_hash},
{'field': 'tx1_hash', 'op': '==', 'value': tx1_hash}]
}, abort_on_error=True)['result'][0]
else:
assert msg_data['status'] == 'completed' #should not enter a pending state for non BTC matches
order_match = msg_data
forward_asset_info = mongo_db.tracked_assets.find_one({'asset': order_match['forward_asset']})
backward_asset_info = mongo_db.tracked_assets.find_one({'asset': order_match['backward_asset']})
assert forward_asset_info and backward_asset_info
base_asset, quote_asset = util.assets_to_asset_pair(order_match['forward_asset'], order_match['backward_asset'])
#don't create trade records from order matches with BTC that are under the dust limit
if (order_match['forward_asset'] == config.BTC and order_match['forward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF) \
or (order_match['backward_asset'] == config.BTC and order_match['backward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF):
logging.debug("Order match %s ignored due to %s under dust limit." % (order_match['tx0_hash'] + order_match['tx1_hash'], config.BTC))
continue
#take divisible trade quantities to floating point
forward_quantity = util_bitcoin.normalize_quantity(order_match['forward_quantity'], forward_asset_info['divisible'])
backward_quantity = util_bitcoin.normalize_quantity(order_match['backward_quantity'], backward_asset_info['divisible'])
#compose trade
trade = {
'block_index': cur_block_index,
'block_time': cur_block['block_time_obj'],
'message_index': msg['message_index'], #secondary temporaral ordering off of when
'order_match_id': order_match['tx0_hash'] + order_match['tx1_hash'],
'order_match_tx0_index': order_match['tx0_index'],
'order_match_tx1_index': order_match['tx1_index'],
'order_match_tx0_address': order_match['tx0_address'],
'order_match_tx1_address': order_match['tx1_address'],
'base_asset': base_asset,
'quote_asset': quote_asset,
'base_quantity': order_match['forward_quantity'] if order_match['forward_asset'] == base_asset else order_match['backward_quantity'],
'quote_quantity': order_match['backward_quantity'] if order_match['forward_asset'] == base_asset else order_match['forward_quantity'],
'base_quantity_normalized': forward_quantity if order_match['forward_asset'] == base_asset else backward_quantity,
'quote_quantity_normalized': backward_quantity if order_match['forward_asset'] == base_asset else forward_quantity,
}
trade['unit_price'] = float(
( D(trade['quote_quantity_normalized']) / D(trade['base_quantity_normalized']) ).quantize(
D('.00000000'), rounding=decimal.ROUND_HALF_EVEN))
trade['unit_price_inverse'] = float(
( D(trade['base_quantity_normalized']) / D(trade['quote_quantity_normalized']) ).quantize(
D('.00000000'), rounding=decimal.ROUND_HALF_EVEN))
mongo_db.trades.insert(trade)
logging.info("Procesed Trade from tx %s :: %s" % (msg['message_index'], trade))
#broadcast
if msg['category'] == 'broadcasts':
betting.parse_broadcast(mongo_db, msg_data)
#if we're catching up beyond MAX_REORG_NUM_BLOCKS blocks out, make sure not to send out any socket.io
# events, as to not flood on a resync (as we may give a 525 to kick the logged in clients out, but we
# can't guarantee that the socket.io connection will always be severed as well??)
if last_processed_block['block_index'] - my_latest_block['block_index'] < config.MAX_REORG_NUM_BLOCKS:
#send out the message to listening clients
event = util.decorate_message_for_feed(msg, msg_data=msg_data)
zmq_publisher_eventfeed.send_json(event)
#this is the last processed message index
config.LAST_MESSAGE_INDEX = msg['message_index']
else:
#block successfully processed, track this in our DB
new_block = {
'block_index': cur_block_index,
'block_time': cur_block['block_time_obj'],
'block_hash': cur_block['block_hash'],
}
blocks_to_insert.append(new_block)
if last_processed_block['block_index'] - cur_block_index > 1000: #reparsing, do bulk inserts
if len(blocks_to_insert) >= 1000:
mongo_db.processed_blocks.insert(blocks_to_insert)
blocks_to_insert[:] = []
else:
mongo_db.processed_blocks.insert(blocks_to_insert)
blocks_to_insert[:] = []
my_latest_block = new_block
config.CURRENT_BLOCK_INDEX = cur_block_index
#get the current blockchain service block
if config.BLOCKCHAIN_SERVICE_LAST_BLOCK == 0 or config.BLOCKCHAIN_SERVICE_LAST_BLOCK - config.CURRENT_BLOCK_INDEX < config.MAX_REORG_NUM_BLOCKS:
#update as CURRENT_BLOCK_INDEX catches up with BLOCKCHAIN_SERVICE_LAST_BLOCK and/or surpasses it (i.e. if blockchain service gets behind for some reason)
try:
block_height_response = blockchain.getinfo()
except:
block_height_response = None
config.BLOCKCHAIN_SERVICE_LAST_BLOCK = block_height_response['info']['blocks'] if block_height_response else 0
logging.info("Block: %i (message_index height=%s) (blockchain latest block=%s)" % (config.CURRENT_BLOCK_INDEX,
config.LAST_MESSAGE_INDEX if config.LAST_MESSAGE_INDEX != -1 else '???',
config.BLOCKCHAIN_SERVICE_LAST_BLOCK if config.BLOCKCHAIN_SERVICE_LAST_BLOCK else '???'))
if last_processed_block['block_index'] - cur_block_index < config.MAX_REORG_NUM_BLOCKS: #only when we are near the tip
clean_mempool_tx()
