def parse_trade_book(msg, msg_data):
#book trades
if (msg['category'] == 'order_matches'
and ((msg['command'] == 'update' and msg_data['status'] == 'completed') #for a trade with BTC involved, but that is settled (completed)
or ('forward_asset' in msg_data and msg_data['forward_asset'] != config.BTC and msg_data['backward_asset'] != config.BTC))): #or for a trade without BTC on either end
if msg['command'] == 'update' and msg_data['status'] == 'completed':
#an order is being updated to a completed status (i.e. a BTCpay has completed)
tx0_hash, tx1_hash = msg_data['order_match_id'][:64], msg_data['order_match_id'][65:]
#get the order_match this btcpay settles
order_match = util.jsonrpc_api("get_order_matches",
{'filters': [
{'field': 'tx0_hash', 'op': '==', 'value': tx0_hash},
{'field': 'tx1_hash', 'op': '==', 'value': tx1_hash}]
}, abort_on_error=False)['result'][0]
else:
assert msg_data['status'] == 'completed' #should not enter a pending state for non BTC matches
order_match = msg_data
forward_asset_info = config.mongo_db.tracked_assets.find_one({'asset': order_match['forward_asset']})
backward_asset_info = config.mongo_db.tracked_assets.find_one({'asset': order_match['backward_asset']})
assert forward_asset_info and backward_asset_info
base_asset, quote_asset = util.assets_to_asset_pair(order_match['forward_asset'], order_match['backward_asset'])
#don't create trade records from order matches with BTC that are under the dust limit
if (order_match['forward_asset'] == config.BTC and order_match['forward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF) \
or (order_match['backward_asset'] == config.BTC and order_match['backward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF):
logger.debug("Order match %s ignored due to %s under dust limit." % (order_match['tx0_hash'] + order_match['tx1_hash'], config.BTC))
return 'ABORT_THIS_MESSAGE_PROCESSING'
#take divisible trade quantities to floating point
forward_quantity = blockchain.normalize_quantity(order_match['forward_quantity'], forward_asset_info['divisible'])
backward_quantity = blockchain.normalize_quantity(order_match['backward_quantity'], backward_asset_info['divisible'])
#compose trade
trade = {
'block_index': config.state['cur_block']['block_index'],
'block_time': config.state['cur_block']['block_time_obj'],
'message_index': msg['message_index'], #secondary temporaral ordering off of when
'order_match_id': order_match['tx0_hash'] + '_' + order_match['tx1_hash'],
'order_match_tx0_index': order_match['tx0_index'],
'order_match_tx1_index': order_match['tx1_index'],
'order_match_tx0_address': order_match['tx0_address'],
'order_match_tx1_address': order_match['tx1_address'],
'base_asset': base_asset,
'quote_asset': quote_asset,
'base_quantity': order_match['forward_quantity'] if order_match['forward_asset'] == base_asset else order_match['backward_quantity'],
'quote_quantity': order_match['backward_quantity'] if order_match['forward_asset'] == base_asset else order_match['forward_quantity'],
'base_quantity_normalized': forward_quantity if order_match['forward_asset'] == base_asset else backward_quantity,
'quote_quantity_normalized': backward_quantity if order_match['forward_asset'] == base_asset else forward_quantity,
}
trade['unit_price'] = float(
( D(trade['quote_quantity_normalized']) / D(trade['base_quantity_normalized']) ).quantize(
D('.00000000'), rounding=decimal.ROUND_HALF_EVEN))
trade['unit_price_inverse'] = float(
( D(trade['base_quantity_normalized']) / D(trade['quote_quantity_normalized']) ).quantize(
D('.00000000'), rounding=decimal.ROUND_HALF_EVEN))
config.mongo_db.trades.insert(trade)
logger.info("Procesed Trade from tx %s :: %s" % (msg['message_index'], trade))
def get_asset_info(asset, at_dt=None):
asset_info = config.mongo_db.tracked_assets.find_one({'asset': asset})
if asset not in (config.XCP, config.BTC) and at_dt and asset_info['_at_block_time'] > at_dt:
#get the asset info at or before the given at_dt datetime
for e in reversed(asset_info['_history']): #newest to oldest
if e['_at_block_time'] <= at_dt:
asset_info = e
break
else: #asset was created AFTER at_dt
asset_info = None
if asset_info is None: return None
assert asset_info['_at_block_time'] <= at_dt
#modify some of the properties of the returned asset_info for BTC and XCP
if asset == config.BTC:
if at_dt:
start_block_index, end_block_index = database.get_block_indexes_for_dates(end_dt=at_dt)
asset_info['total_issued'] = blockchain.get_btc_supply(normalize=False, at_block_index=end_block_index)
asset_info['total_issued_normalized'] = blockchain.normalize_quantity(asset_info['total_issued'])
else:
asset_info['total_issued'] = blockchain.get_btc_supply(normalize=False)
asset_info['total_issued_normalized'] = blockchain.normalize_quantity(asset_info['total_issued'])
elif asset == config.XCP:
#BUG: this does not take end_dt (if specified) into account. however, the deviation won't be too big
# as XCP doesn't deflate quickly at all, and shouldn't matter that much since there weren't any/much trades
# before the end of the burn period (which is what is involved with how we use at_dt with currently)
asset_info['total_issued'] = util.call_jsonrpc_api("get_supply", {'asset': 'XCP'}, abort_on_error=True)['result']
asset_info['total_issued_normalized'] = blockchain.normalize_quantity(asset_info['total_issued'])
if not asset_info:
raise Exception("Invalid asset: %s" % asset)
return asset_info
def get_normalized_balances(addresses):
"""
This call augments counterparty's get_balances with a normalized_quantity field. It also will include any owned
assets for an address, even if their balance is zero.
NOTE: Does not retrieve BTC balance. Use get_address_info for that.
"""
if not isinstance(addresses, list):
raise Exception("addresses must be a list of addresses, even if it just contains one address")
if not len(addresses):
raise Exception("Invalid address list supplied")
filters = []
for address in addresses:
filters.append({'field': 'address', 'op': '==', 'value': address})
mappings = {}
result = util.call_jsonrpc_api(
"get_balances",
{'filters': filters, 'filterop': 'or'}, abort_on_error=True)['result']
isowner = {}
owned_assets = config.mongo_db.tracked_assets.find(
{'$or': [{'owner': a} for a in addresses]}, {'_history': 0, '_id': 0})
for o in owned_assets:
isowner[o['owner'] + o['asset']] = o
data = []
for d in result:
if not d['quantity'] and ((d['address'] + d['asset']) not in isowner):
continue # don't include balances with a zero asset value
asset_info = config.mongo_db.tracked_assets.find_one({'asset': d['asset']})
divisible = True # XCP and BTC
if asset_info and 'divisible' in asset_info:
divisible = asset_info['divisible']
d['normalized_quantity'] = blockchain.normalize_quantity(d['quantity'], divisible)
d['owner'] = (d['address'] + d['asset']) in isowner
try:
d['asset_longname'] = asset_info['asset_longname']
except TypeError as e:
d['asset_longname'] = d['asset']
mappings[d['address'] + d['asset']] = d
data.append(d)
# include any owned assets for each address, even if their balance is zero
for key in isowner:
if key not in mappings:
o = isowner[key]
data.append({
'address': o['owner'],
'asset': o['asset'],
'quantity': 0,
'normalized_quantity': 0,
'owner': True,
})
return data
def make_book(orders, isBidBook):
book = {}
for o in orders:
if o['give_asset'] == base_asset:
if base_asset == config.BTC and o['give_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF:
continue #filter dust orders, if necessary
give_quantity = blockchain.normalize_quantity(o['give_quantity'], base_asset_info['divisible'])
get_quantity = blockchain.normalize_quantity(o['get_quantity'], quote_asset_info['divisible'])
unit_price = float(( D(get_quantity) / D(give_quantity) ))
remaining = blockchain.normalize_quantity(o['give_remaining'], base_asset_info['divisible'])
else:
if quote_asset == config.BTC and o['give_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF:
continue #filter dust orders, if necessary
give_quantity = blockchain.normalize_quantity(o['give_quantity'], quote_asset_info['divisible'])
get_quantity = blockchain.normalize_quantity(o['get_quantity'], base_asset_info['divisible'])
unit_price = float(( D(give_quantity) / D(get_quantity) ))
remaining = blockchain.normalize_quantity(o['get_remaining'], base_asset_info['divisible'])
id = "%s_%s_%s" % (base_asset, quote_asset, unit_price)
#^ key = {base}_{bid}_{unit_price}, values ref entries in book
book.setdefault(id, {'unit_price': unit_price, 'quantity': 0, 'count': 0})
book[id]['quantity'] += remaining #base quantity outstanding
book[id]['count'] += 1 #num orders at this price level
book = sorted(book.itervalues(), key=operator.itemgetter('unit_price'), reverse=isBidBook)
#^ convert to list and sort -- bid book = descending, ask book = ascending
return book
def get_normalized_balances(addresses):
"""
This call augments counterparty's get_balances with a normalized_quantity field. It also will include any owned
assets for an address, even if their balance is zero.
NOTE: Does not retrieve BTC balance. Use get_address_info for that.
"""
if not isinstance(addresses, list):
raise Exception("addresses must be a list of addresses, even if it just contains one address")
if not len(addresses):
raise Exception("Invalid address list supplied")
filters = []
for address in addresses:
filters.append({'field': 'address', 'op': '==', 'value': address})
mappings = {}
result = util.call_jsonrpc_api(
"get_balances",
{'filters': filters, 'filterop': 'or'}, abort_on_error=True)['result']
isowner = {}
owned_assets = config.mongo_db.tracked_assets.find(
{'$or': [{'owner': a} for a in addresses]}, {'_history': 0, '_id': 0})
for o in owned_assets:
isowner[o['owner'] + o['asset']] = o
data = []
for d in result:
if not d['quantity'] and ((d['address'] + d['asset']) not in isowner):
continue # don't include balances with a zero asset value
asset_info = config.mongo_db.tracked_assets.find_one({'asset': d['asset']})
divisible = True # XCP and BTC
if asset_info and 'divisible' in asset_info:
divisible = asset_info['divisible']
d['normalized_quantity'] = blockchain.normalize_quantity(d['quantity'], divisible)
d['owner'] = (d['address'] + d['asset']) in isowner
mappings[d['address'] + d['asset']] = d
data.append(d)
# include any owned assets for each address, even if their balance is zero
for key in isowner:
if key not in mappings:
o = isowner[key]
data.append({
'address': o['owner'],
'asset': o['asset'],
'quantity': 0,
'normalized_quantity': 0,
'owner': True,
})
return data
def parse_balance_change(msg, msg_data):
# track balance changes for each address
bal_change = None
if msg['category'] in ['credits', 'debits', ]:
actionName = 'credit' if msg['category'] == 'credits' else 'debit'
address = msg_data['address']
asset_info = config.mongo_db.tracked_assets.find_one({'asset': msg_data['asset']})
if asset_info is None:
logger.warn("Credit/debit of %s where asset ('%s') does not exist. Ignoring..." % (msg_data['quantity'], msg_data['asset']))
return 'ABORT_THIS_MESSAGE_PROCESSING'
quantity = msg_data['quantity'] if msg['category'] == 'credits' else -msg_data['quantity']
quantity_normalized = blockchain.normalize_quantity(quantity, asset_info['divisible'])
# look up the previous balance to go off of
last_bal_change = config.mongo_db.balance_changes.find_one({
'address': address,
'asset': asset_info['asset']
}, sort=[("block_index", pymongo.DESCENDING), ("_id", pymongo.DESCENDING)])
if last_bal_change \
and last_bal_change['block_index'] == config.state['cur_block']['block_index']:
# modify this record, as we want at most one entry per block index for each (address, asset) pair
last_bal_change['quantity'] += quantity
last_bal_change['quantity_normalized'] += quantity_normalized
last_bal_change['new_balance'] += quantity
last_bal_change['new_balance_normalized'] += quantity_normalized
config.mongo_db.balance_changes.save(last_bal_change)
logger.info("%s (UPDATED) %s %s %s %s (new bal: %s, msgID: %s)" % (
actionName.capitalize(), ('%f' % last_bal_change['quantity_normalized']).rstrip('0').rstrip('.'), last_bal_change['asset'],
'from' if actionName == 'debit' else 'to',
last_bal_change['address'], ('%f' % last_bal_change['new_balance_normalized']).rstrip('0').rstrip('.'), msg['message_index'],))
bal_change = last_bal_change
else: # new balance change record for this block
bal_change = {
'address': address,
'asset': asset_info['asset'],
'asset_longname': asset_info['asset_longname'],
'block_index': config.state['cur_block']['block_index'],
'block_time': config.state['cur_block']['block_time_obj'],
'quantity': quantity,
'quantity_normalized': quantity_normalized,
'new_balance': last_bal_change['new_balance'] + quantity if last_bal_change else quantity,
'new_balance_normalized': last_bal_change['new_balance_normalized'] + quantity_normalized if last_bal_change else quantity_normalized,
}
config.mongo_db.balance_changes.insert(bal_change)
logger.info("%s %s %s %s %s (new bal: %s, msgID: %s)" % (
actionName.capitalize(), ('%f' % bal_change['quantity_normalized']).rstrip('0').rstrip('.'), bal_change['asset'],
'from' if actionName == 'debit' else 'to',
bal_change['address'], ('%f' % bal_change['new_balance_normalized']).rstrip('0').rstrip('.'), msg['message_index'],))
def parse_balance_change(msg, msg_data):
#track balance changes for each address
bal_change = None
if msg['category'] in ['credits', 'debits',]:
actionName = 'credit' if msg['category'] == 'credits' else 'debit'
address = msg_data['address']
asset_info = config.mongo_db.tracked_assets.find_one({ 'asset': msg_data['asset'] })
if asset_info is None:
logger.warn("Credit/debit of %s where asset ('%s') does not exist. Ignoring..." % (msg_data['quantity'], msg_data['asset']))
return 'ABORT_THIS_MESSAGE_PROCESSING'
quantity = msg_data['quantity'] if msg['category'] == 'credits' else -msg_data['quantity']
quantity_normalized = blockchain.normalize_quantity(quantity, asset_info['divisible'])
#look up the previous balance to go off of
last_bal_change = config.mongo_db.balance_changes.find_one({
'address': address,
'asset': asset_info['asset']
}, sort=[("block_index", pymongo.DESCENDING), ("_id", pymongo.DESCENDING)])
if last_bal_change \
and last_bal_change['block_index'] == config.state['cur_block']['block_index']:
#modify this record, as we want at most one entry per block index for each (address, asset) pair
last_bal_change['quantity'] += quantity
last_bal_change['quantity_normalized'] += quantity_normalized
last_bal_change['new_balance'] += quantity
last_bal_change['new_balance_normalized'] += quantity_normalized
config.mongo_db.balance_changes.save(last_bal_change)
logger.info("%s (UPDATED) %s %s %s %s (new bal: %s, msgID: %s)" % (
actionName.capitalize(), ('%f' % last_bal_change['quantity_normalized']).rstrip('0').rstrip('.'), last_bal_change['asset'],
'from' if actionName == 'debit' else 'to',
last_bal_change['address'], ('%f' % last_bal_change['new_balance_normalized']).rstrip('0').rstrip('.'), msg['message_index'],))
bal_change = last_bal_change
else: #new balance change record for this block
bal_change = {
'address': address,
'asset': asset_info['asset'],
'block_index': config.state['cur_block']['block_index'],
'block_time': config.state['cur_block']['block_time_obj'],
'quantity': quantity,
'quantity_normalized': quantity_normalized,
'new_balance': last_bal_change['new_balance'] + quantity if last_bal_change else quantity,
'new_balance_normalized': last_bal_change['new_balance_normalized'] + quantity_normalized if last_bal_change else quantity_normalized,
}
config.mongo_db.balance_changes.insert(bal_change)
logger.info("%s %s %s %s %s (new bal: %s, msgID: %s)" % (
actionName.capitalize(), ('%f' % bal_change['quantity_normalized']).rstrip('0').rstrip('.'), bal_change['asset'],
'from' if actionName == 'debit' else 'to',
bal_change['address'], ('%f' % bal_change['new_balance_normalized']).rstrip('0').rstrip('.'), msg['message_index'],))
def decorate_message(message, for_txn_history=False):
# insert custom fields in certain events...
# even invalid actions need these extra fields for proper reporting to the client (as the reporting message
# is produced via PendingActionViewModel.calcText) -- however make it able to deal with the queried data not existing in this case
assert '_category' in message
if for_txn_history:
message['_command'] = 'insert' # history data doesn't include this
block_index = message['block_index'] if 'block_index' in message else message['tx1_block_index']
message['_block_time'] = database.get_block_time(block_index)
message['_tx_index'] = message['tx_index'] if 'tx_index' in message else message.get('tx1_index', None)
if message['_category'] in ['bet_expirations', 'order_expirations', 'bet_match_expirations', 'order_match_expirations']:
message['_tx_index'] = 0 # add tx_index to all entries (so we can sort on it secondarily in history view), since these lack it
if message['_category'] in ['credits', 'debits']:
# find the last balance change on record
bal_change = config.mongo_db.balance_changes.find_one(
{'address': message['address'], 'asset': message['asset']},
sort=[("block_time", pymongo.DESCENDING)])
message['_quantity_normalized'] = abs(bal_change['quantity_normalized']) if bal_change else None
message['_balance'] = bal_change['new_balance'] if bal_change else None
message['_balance_normalized'] = bal_change['new_balance_normalized'] if bal_change else None
if message['_category'] in ['orders', ] and message['_command'] == 'insert':
get_asset_info = config.mongo_db.tracked_assets.find_one({'asset': message['get_asset']})
give_asset_info = config.mongo_db.tracked_assets.find_one({'asset': message['give_asset']})
message['_get_asset_divisible'] = get_asset_info['divisible'] if get_asset_info else None
message['_give_asset_divisible'] = give_asset_info['divisible'] if give_asset_info else None
if message['_category'] in ['order_matches', ] and message['_command'] == 'insert':
forward_asset_info = config.mongo_db.tracked_assets.find_one({'asset': message['forward_asset']})
backward_asset_info = config.mongo_db.tracked_assets.find_one({'asset': message['backward_asset']})
message['_forward_asset_divisible'] = forward_asset_info['divisible'] if forward_asset_info else None
message['_backward_asset_divisible'] = backward_asset_info['divisible'] if backward_asset_info else None
if message['_category'] in ['orders', 'order_matches', ]:
message['_btc_below_dust_limit'] = (
('forward_asset' in message and message['forward_asset'] == config.BTC and message['forward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF)
or ('backward_asset' in message and message['backward_asset'] == config.BTC and message['backward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF)
)
if message['_category'] in ['dividends', 'sends', ]:
asset_info = config.mongo_db.tracked_assets.find_one({'asset': message['asset']})
message['_divisible'] = asset_info['divisible'] if asset_info else None
if message['_category'] in ['issuances', ]:
message['_quantity_normalized'] = blockchain.normalize_quantity(message['quantity'], message['divisible'])
return message
def parse_issuance(msg, msg_data):
if msg['category'] != 'issuances':
return
if msg_data['status'] != 'valid':
return
cur_block_index = config.state['cur_block']['block_index']
cur_block = config.state['cur_block']
def modify_extended_asset_info(asset, description):
"""adds an asset to asset_extended_info collection if the description is a valid json link. or, if the link
is not a valid json link, will remove the asset entry from the table if it exists"""
if util.is_valid_url(description,
suffix='.json',
allow_no_protocol=True):
config.mongo_db.asset_extended_info.update(
{'asset': asset},
{
'$set': {
'info_url': description,
'info_status': 'needfetch',
'fetch_info_retry':
0, # retry ASSET_MAX_RETRY times to fetch info from info_url
'info_data': {},
'errors': []
}
},
upsert=True)
#^ valid info_status settings: needfetch, valid, invalid, error
# additional fields will be added later in events, once the asset info is pulled
else:
config.mongo_db.asset_extended_info.remove({'asset': asset})
# remove any saved asset image data
imagePath = os.path.join(config.data_dir,
config.SUBDIR_ASSET_IMAGES,
asset + '.png')
if os.path.exists(imagePath):
os.remove(imagePath)
tracked_asset = config.mongo_db.tracked_assets.find_one(
{'asset': msg_data['asset']}, {
'_id': 0,
'_history': 0
})
#^ pulls the tracked asset without the _id and history fields. This may be None
if msg_data['locked']: # lock asset
assert tracked_asset is not None
config.mongo_db.tracked_assets.update({'asset': msg_data['asset']}, {
"$set": {
'_at_block': cur_block_index,
'_at_block_time': cur_block['block_time_obj'],
'_change_type': 'locked',
'locked': True,
},
"$push": {
'_history': tracked_asset
}
},
upsert=False)
logger.info("Locking asset %s" % (msg_data['asset'], ))
elif msg_data['transfer']: # transfer asset
assert tracked_asset is not None
config.mongo_db.tracked_assets.update({'asset': msg_data['asset']}, {
"$set": {
'_at_block': cur_block_index,
'_at_block_time': cur_block['block_time_obj'],
'_change_type': 'transferred',
'owner': msg_data['issuer'],
},
"$push": {
'_history': tracked_asset
}
},
upsert=False)
logger.info("Transferring asset %s to address %s" %
(msg_data['asset'], msg_data['issuer']))
elif msg_data[
'quantity'] == 0 and tracked_asset is not None: # change description
config.mongo_db.tracked_assets.update({'asset': msg_data['asset']}, {
"$set": {
'_at_block': cur_block_index,
'_at_block_time': cur_block['block_time_obj'],
'_change_type': 'changed_description',
'description': msg_data['description'],
},
"$push": {
'_history': tracked_asset
}
},
upsert=False)
modify_extended_asset_info(msg_data['asset'], msg_data['description'])
logger.info("Changing description for asset %s to '%s'" %
(msg_data['asset'], msg_data['description']))
else: # issue new asset or issue addition qty of an asset
if not tracked_asset: # new issuance
tracked_asset = {
'_change_type':
'created',
'_at_block':
cur_block_index, # the block ID this asset is current for
'_at_block_time':
cur_block['block_time_obj'],
#^ NOTE: (if there are multiple asset tracked changes updates in a single block for the same
# asset, the last one with _at_block == that block id in the history array is the
# final version for that asset at that block
'asset':
msg_data['asset'],
'owner':
msg_data['issuer'],
'description':
msg_data['description'],
'divisible':
msg_data['divisible'],
'locked':
False,
'total_issued':
int(msg_data['quantity']),
'total_issued_normalized':
blockchain.normalize_quantity(msg_data['quantity'],
msg_data['divisible']),
'_history': [] # to allow for block rollbacks
}
config.mongo_db.tracked_assets.insert(tracked_asset)
logger.info("Tracking new asset: %s" % msg_data['asset'])
modify_extended_asset_info(msg_data['asset'],
msg_data['description'])
else: # issuing additional of existing asset
assert tracked_asset is not None
config.mongo_db.tracked_assets.update(
{'asset': msg_data['asset']}, {
"$set": {
'_at_block': cur_block_index,
'_at_block_time': cur_block['block_time_obj'],
'_change_type': 'issued_more',
},
"$inc": {
'total_issued':
msg_data['quantity'],
'total_issued_normalized':
blockchain.normalize_quantity(msg_data['quantity'],
msg_data['divisible'])
},
"$push": {
'_history': tracked_asset
}
},
upsert=False)
logger.info("Adding additional %s quantity for asset %s" %
(blockchain.normalize_quantity(
msg_data['quantity'],
msg_data['divisible']), msg_data['asset']))
return True
def decorate_message(message, for_txn_history=False):
# insert custom fields in certain events...
# even invalid actions need these extra fields for proper reporting to the client (as the reporting message
# is produced via PendingActionViewModel.calcText) -- however make it able to deal with the queried data not existing in this case
assert '_category' in message
if for_txn_history:
message['_command'] = 'insert' # history data doesn't include this
block_index = message[
'block_index'] if 'block_index' in message else message[
'tx1_block_index']
message['_block_time'] = database.get_block_time(block_index)
message['_tx_index'] = message[
'tx_index'] if 'tx_index' in message else message.get(
'tx1_index', None)
if message['_category'] in [
'bet_expirations', 'order_expirations',
'bet_match_expirations', 'order_match_expirations'
]:
message[
'_tx_index'] = 0 # add tx_index to all entries (so we can sort on it secondarily in history view), since these lack it
if message['_category'] in ['credits', 'debits']:
# find the last balance change on record
bal_change = config.mongo_db.balance_changes.find_one(
{
'address': message['address'],
'asset': message['asset']
},
sort=[("block_time", pymongo.DESCENDING)])
message['_quantity_normalized'] = abs(
bal_change['quantity_normalized']) if bal_change else None
message['_balance'] = bal_change['new_balance'] if bal_change else None
message['_balance_normalized'] = bal_change[
'new_balance_normalized'] if bal_change else None
if message['_category'] in [
'orders',
] and message['_command'] == 'insert':
get_asset_info = config.mongo_db.tracked_assets.find_one(
{'asset': message['get_asset']})
give_asset_info = config.mongo_db.tracked_assets.find_one(
{'asset': message['give_asset']})
message['_get_asset_divisible'] = get_asset_info[
'divisible'] if get_asset_info else None
message['_give_asset_divisible'] = give_asset_info[
'divisible'] if give_asset_info else None
if message['_category'] in [
'order_matches',
] and message['_command'] == 'insert':
forward_asset_info = config.mongo_db.tracked_assets.find_one(
{'asset': message['forward_asset']})
backward_asset_info = config.mongo_db.tracked_assets.find_one(
{'asset': message['backward_asset']})
message['_forward_asset_divisible'] = forward_asset_info[
'divisible'] if forward_asset_info else None
message['_backward_asset_divisible'] = backward_asset_info[
'divisible'] if backward_asset_info else None
if message['_category'] in [
'orders',
'order_matches',
]:
message['_btc_below_dust_limit'] = (
('forward_asset' in message
and message['forward_asset'] == config.BTC and
message['forward_quantity'] <= config.ORDER_BTC_DUST_LIMIT_CUTOFF)
or ('backward_asset' in message
and message['backward_asset'] == config.BTC
and message['backward_quantity'] <=
config.ORDER_BTC_DUST_LIMIT_CUTOFF))
if message['_category'] in [
'dividends',
'sends',
]:
asset_info = config.mongo_db.tracked_assets.find_one(
{'asset': message['asset']})
message['_divisible'] = asset_info['divisible'] if asset_info else None
if message['_category'] in [
'issuances',
]:
message['_quantity_normalized'] = blockchain.normalize_quantity(
message['quantity'], message['divisible'])
return message
def compile_asset_pair_market_info():
"""Compiles the pair-level statistics that show on the View Prices page of counterwallet, for instance"""
#loop through all open orders, and compile a listing of pairs, with a count of open orders for each pair
mongo_db = config.mongo_db
end_dt = datetime.datetime.utcnow()
start_dt = end_dt - datetime.timedelta(days=1)
start_block_index, end_block_index = database.get_block_indexes_for_dates(
start_dt=start_dt, end_dt=end_dt)
open_orders = util.call_jsonrpc_api("get_orders", {
'filters': [
{
'field': 'give_remaining',
'op': '>',
'value': 0
},
{
'field': 'get_remaining',
'op': '>',
'value': 0
},
{
'field': 'fee_required_remaining',
'op': '>=',
'value': 0
},
{
'field': 'fee_provided_remaining',
'op': '>=',
'value': 0
},
],
'status':
'open',
'show_expired':
False,
},
abort_on_error=True)['result']
pair_data = {}
asset_info = {}
def get_price(base_quantity_normalized, quote_quantity_normalized):
return float(D(quote_quantity_normalized / base_quantity_normalized))
#COMPOSE order depth, lowest ask, and highest bid column data
for o in open_orders:
(base_asset,
quote_asset) = util.assets_to_asset_pair(o['give_asset'],
o['get_asset'])
pair = '%s/%s' % (base_asset, quote_asset)
base_asset_info = asset_info.get(
base_asset, mongo_db.tracked_assets.find_one({'asset':
base_asset}))
if base_asset not in asset_info:
asset_info[base_asset] = base_asset_info
quote_asset_info = asset_info.get(
quote_asset,
mongo_db.tracked_assets.find_one({'asset': quote_asset}))
if quote_asset not in asset_info:
asset_info[quote_asset] = quote_asset_info
pair_data.setdefault(
pair, {
'open_orders_count': 0,
'lowest_ask': None,
'highest_bid': None,
'completed_trades_count': 0,
'vol_base': 0,
'vol_quote': 0
})
#^ highest ask = open order selling base, highest bid = open order buying base
#^ we also initialize completed_trades_count, vol_base, vol_quote because every pair inited here may
# not have cooresponding data out of the trades_data_by_pair aggregation below
pair_data[pair]['open_orders_count'] += 1
base_quantity_normalized = blockchain.normalize_quantity(
o['give_quantity'] if base_asset == o['give_asset'] else
o['get_quantity'], base_asset_info['divisible'])
quote_quantity_normalized = blockchain.normalize_quantity(
o['give_quantity'] if quote_asset == o['give_asset'] else
o['get_quantity'], quote_asset_info['divisible'])
order_price = get_price(base_quantity_normalized,
quote_quantity_normalized)
if base_asset == o['give_asset']: #selling base
if pair_data[pair]['lowest_ask'] is None or order_price < pair_data[
pair]['lowest_ask']:
pair_data[pair]['lowest_ask'] = order_price
elif base_asset == o['get_asset']: #buying base
if pair_data[pair][
'highest_bid'] is None or order_price > pair_data[pair][
'highest_bid']:
pair_data[pair]['highest_bid'] = order_price
#COMPOSE volume data (in XCP and BTC), and % change data
#loop through all trade volume over the past 24h, and match that to the open orders
trades_data_by_pair = mongo_db.trades.aggregate([
{
"$match": {
"block_time": {
"$gte": start_dt,
"$lte": end_dt
}
}
},
{
"$project": {
"base_asset": 1,
"quote_asset": 1,
"base_quantity_normalized": 1, #to derive base volume
"quote_quantity_normalized": 1 #to derive quote volume
}
},
{
"$group": {
"_id": {
"base_asset": "$base_asset",
"quote_asset": "$quote_asset"
},
"vol_base": {
"$sum": "$base_quantity_normalized"
},
"vol_quote": {
"$sum": "$quote_quantity_normalized"
},
"count": {
"$sum": 1
},
}
}
])
trades_data_by_pair = [] if not trades_data_by_pair[
'ok'] else trades_data_by_pair['result']
for e in trades_data_by_pair:
pair = '%s/%s' % (e['_id']['base_asset'], e['_id']['quote_asset'])
pair_data.setdefault(pair, {
'open_orders_count': 0,
'lowest_ask': None,
'highest_bid': None
})
#^ initialize an empty pair in the event there are no open orders for that pair, but there ARE completed trades for it
pair_data[pair]['completed_trades_count'] = e['count']
pair_data[pair]['vol_base'] = e['vol_base']
pair_data[pair]['vol_quote'] = e['vol_quote']
#compose price data, relative to BTC and XCP
mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives()
for pair, e in pair_data.iteritems():
base_asset, quote_asset = pair.split('/')
_24h_vol_in_btc = None
_24h_vol_in_xcp = None
#derive asset price data, expressed in BTC and XCP, for the given volumes
if base_asset == config.XCP:
_24h_vol_in_xcp = e['vol_base']
_24h_vol_in_btc = blockchain.round_out(
e['vol_base'] * xcp_btc_price) if xcp_btc_price else 0
elif base_asset == config.BTC:
_24h_vol_in_xcp = blockchain.round_out(
e['vol_base'] * btc_xcp_price) if btc_xcp_price else 0
_24h_vol_in_btc = e['vol_base']
else: #base is not XCP or BTC
price_summary_in_xcp, price_summary_in_btc, price_in_xcp, price_in_btc, aggregated_price_in_xcp, aggregated_price_in_btc = \
get_xcp_btc_price_info(base_asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price, with_last_trades=0, start_dt=start_dt, end_dt=end_dt)
if price_in_xcp:
_24h_vol_in_xcp = blockchain.round_out(e['vol_base'] *
price_in_xcp)
if price_in_btc:
_24h_vol_in_btc = blockchain.round_out(e['vol_base'] *
price_in_btc)
if _24h_vol_in_xcp is None or _24h_vol_in_btc is None:
#the base asset didn't have price data against BTC or XCP, or both...try against the quote asset instead
price_summary_in_xcp, price_summary_in_btc, price_in_xcp, price_in_btc, aggregated_price_in_xcp, aggregated_price_in_btc = \
get_xcp_btc_price_info(quote_asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price, with_last_trades=0, start_dt=start_dt, end_dt=end_dt)
if _24h_vol_in_xcp is None and price_in_xcp:
_24h_vol_in_xcp = blockchain.round_out(e['vol_quote'] *
price_in_xcp)
if _24h_vol_in_btc is None and price_in_btc:
_24h_vol_in_btc = blockchain.round_out(e['vol_quote'] *
price_in_btc)
pair_data[pair]['24h_vol_in_{}'.format(
config.XCP.lower())] = _24h_vol_in_xcp #might still be None
pair_data[pair]['24h_vol_in_{}'.format(
config.BTC.lower())] = _24h_vol_in_btc #might still be None
#get % change stats -- start by getting the first trade directly before the 24h period starts
prev_trade = mongo_db.trades.find({
"base_asset": base_asset,
"quote_asset": quote_asset,
"block_time": {
'$lt': start_dt
}
}).sort('block_time', pymongo.DESCENDING).limit(1)
latest_trade = mongo_db.trades.find({
"base_asset": base_asset,
"quote_asset": quote_asset
}).sort('block_time', pymongo.DESCENDING).limit(1)
if not prev_trade.count(): #no previous trade before this 24hr period
pair_data[pair]['24h_pct_change'] = None
else:
prev_trade = prev_trade[0]
latest_trade = latest_trade[0]
prev_trade_price = get_price(
prev_trade['base_quantity_normalized'],
prev_trade['quote_quantity_normalized'])
latest_trade_price = get_price(
latest_trade['base_quantity_normalized'],
latest_trade['quote_quantity_normalized'])
pair_data[pair]['24h_pct_change'] = (
(latest_trade_price - prev_trade_price) /
prev_trade_price) * 100
pair_data[pair]['last_updated'] = end_dt
#print "PRODUCED", pair, pair_data[pair]
mongo_db.asset_pair_market_info.update(
{
'base_asset': base_asset,
'quote_asset': quote_asset
}, {"$set": pair_data[pair]},
upsert=True)
#remove any old pairs that were not just updated
mongo_db.asset_pair_market_info.remove({'last_updated': {'$lt': end_dt}})
logger.info("Recomposed 24h trade statistics for %i asset pairs: %s" %
(len(pair_data), ', '.join(pair_data.keys())))
def compile_asset_pair_market_info():
"""Compiles the pair-level statistics that show on the View Prices page of counterwallet, for instance"""
#loop through all open orders, and compile a listing of pairs, with a count of open orders for each pair
end_dt = datetime.datetime.utcnow()
start_dt = end_dt - datetime.timedelta(days=1)
start_block_index, end_block_index = database.get_block_indexes_for_dates(start_dt=start_dt, end_dt=end_dt)
open_orders = util.call_jsonrpc_api("get_orders",
{ 'filters': [
{'field': 'give_remaining', 'op': '>', 'value': 0},
{'field': 'get_remaining', 'op': '>', 'value': 0},
{'field': 'fee_required_remaining', 'op': '>=', 'value': 0},
{'field': 'fee_provided_remaining', 'op': '>=', 'value': 0},
],
'status': 'open',
'show_expired': False,
}, abort_on_error=True)['result']
pair_data = {}
asset_info = {}
def get_price(base_quantity_normalized, quote_quantity_normalized):
return float(D(quote_quantity_normalized / base_quantity_normalized ))
#COMPOSE order depth, lowest ask, and highest bid column data
for o in open_orders:
(base_asset, quote_asset) = util.assets_to_asset_pair(o['give_asset'], o['get_asset'])
pair = '%s/%s' % (base_asset, quote_asset)
base_asset_info = asset_info.get(base_asset, config.mongo_db.tracked_assets.find_one({ 'asset': base_asset }))
if base_asset not in asset_info: asset_info[base_asset] = base_asset_info
quote_asset_info = asset_info.get(quote_asset, config.mongo_db.tracked_assets.find_one({ 'asset': quote_asset }))
if quote_asset not in asset_info: asset_info[quote_asset] = quote_asset_info
pair_data.setdefault(pair, {'open_orders_count': 0, 'lowest_ask': None, 'highest_bid': None,
'completed_trades_count': 0, 'vol_base': 0, 'vol_quote': 0})
#^ highest ask = open order selling base, highest bid = open order buying base
#^ we also initialize completed_trades_count, vol_base, vol_quote because every pair inited here may
# not have cooresponding data out of the trades_data_by_pair aggregation below
pair_data[pair]['open_orders_count'] += 1
base_quantity_normalized = blockchain.normalize_quantity(o['give_quantity'] if base_asset == o['give_asset'] else o['get_quantity'], base_asset_info['divisible'])
quote_quantity_normalized = blockchain.normalize_quantity(o['give_quantity'] if quote_asset == o['give_asset'] else o['get_quantity'], quote_asset_info['divisible'])
order_price = get_price(base_quantity_normalized, quote_quantity_normalized)
if base_asset == o['give_asset']: #selling base
if pair_data[pair]['lowest_ask'] is None or order_price < pair_data[pair]['lowest_ask']:
pair_data[pair]['lowest_ask'] = order_price
elif base_asset == o['get_asset']: #buying base
if pair_data[pair]['highest_bid'] is None or order_price > pair_data[pair]['highest_bid']:
pair_data[pair]['highest_bid'] = order_price
#COMPOSE volume data (in XCP and BTC), and % change data
#loop through all trade volume over the past 24h, and match that to the open orders
trades_data_by_pair = config.mongo_db.trades.aggregate([
{"$match": {
"block_time": {"$gte": start_dt, "$lte": end_dt } }
},
{"$project": {
"base_asset": 1,
"quote_asset": 1,
"base_quantity_normalized": 1, #to derive base volume
"quote_quantity_normalized": 1 #to derive quote volume
}},
{"$group": {
"_id": {"base_asset": "$base_asset", "quote_asset": "$quote_asset"},
"vol_base": {"$sum": "$base_quantity_normalized"},
"vol_quote": {"$sum": "$quote_quantity_normalized"},
"count": {"$sum": 1},
}}
])
for e in trades_data_by_pair:
pair = '%s/%s' % (e['_id']['base_asset'], e['_id']['quote_asset'])
pair_data.setdefault(pair, {'open_orders_count': 0, 'lowest_ask': None, 'highest_bid': None})
#^ initialize an empty pair in the event there are no open orders for that pair, but there ARE completed trades for it
pair_data[pair]['completed_trades_count'] = e['count']
pair_data[pair]['vol_base'] = e['vol_base']
pair_data[pair]['vol_quote'] = e['vol_quote']
#compose price data, relative to BTC and XCP
mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives()
for pair, e in pair_data.iteritems():
base_asset, quote_asset = pair.split('/')
_24h_vol_in_btc = None
_24h_vol_in_xcp = None
#derive asset price data, expressed in BTC and XCP, for the given volumes
if base_asset == config.XCP:
_24h_vol_in_xcp = e['vol_base']
_24h_vol_in_btc = blockchain.round_out(e['vol_base'] * xcp_btc_price) if xcp_btc_price else 0
elif base_asset == config.BTC:
_24h_vol_in_xcp = blockchain.round_out(e['vol_base'] * btc_xcp_price) if btc_xcp_price else 0
_24h_vol_in_btc = e['vol_base']
else: #base is not XCP or BTC
price_summary_in_xcp, price_summary_in_btc, price_in_xcp, price_in_btc, aggregated_price_in_xcp, aggregated_price_in_btc = \
get_xcp_btc_price_info(base_asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price, with_last_trades=0, start_dt=start_dt, end_dt=end_dt)
if price_in_xcp:
_24h_vol_in_xcp = blockchain.round_out(e['vol_base'] * price_in_xcp)
if price_in_btc:
_24h_vol_in_btc = blockchain.round_out(e['vol_base'] * price_in_btc)
if _24h_vol_in_xcp is None or _24h_vol_in_btc is None:
#the base asset didn't have price data against BTC or XCP, or both...try against the quote asset instead
price_summary_in_xcp, price_summary_in_btc, price_in_xcp, price_in_btc, aggregated_price_in_xcp, aggregated_price_in_btc = \
get_xcp_btc_price_info(quote_asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price, with_last_trades=0, start_dt=start_dt, end_dt=end_dt)
if _24h_vol_in_xcp is None and price_in_xcp:
_24h_vol_in_xcp = blockchain.round_out(e['vol_quote'] * price_in_xcp)
if _24h_vol_in_btc is None and price_in_btc:
_24h_vol_in_btc = blockchain.round_out(e['vol_quote'] * price_in_btc)
pair_data[pair]['24h_vol_in_{}'.format(config.XCP.lower())] = _24h_vol_in_xcp #might still be None
pair_data[pair]['24h_vol_in_{}'.format(config.BTC.lower())] = _24h_vol_in_btc #might still be None
#get % change stats -- start by getting the first trade directly before the 24h period starts
prev_trade = config.mongo_db.trades.find({
"base_asset": base_asset,
"quote_asset": quote_asset,
"block_time": {'$lt': start_dt}}).sort('block_time', pymongo.DESCENDING).limit(1)
latest_trade = config.mongo_db.trades.find({
"base_asset": base_asset,
"quote_asset": quote_asset}).sort('block_time', pymongo.DESCENDING).limit(1)
if not prev_trade.count(): #no previous trade before this 24hr period
pair_data[pair]['24h_pct_change'] = None
else:
prev_trade = prev_trade[0]
latest_trade = latest_trade[0]
prev_trade_price = get_price(prev_trade['base_quantity_normalized'], prev_trade['quote_quantity_normalized'])
latest_trade_price = get_price(latest_trade['base_quantity_normalized'], latest_trade['quote_quantity_normalized'])
pair_data[pair]['24h_pct_change'] = ((latest_trade_price - prev_trade_price) / prev_trade_price) * 100
pair_data[pair]['last_updated'] = end_dt
#print "PRODUCED", pair, pair_data[pair]
config.mongo_db.asset_pair_market_info.update( {'base_asset': base_asset, 'quote_asset': quote_asset}, {"$set": pair_data[pair]}, upsert=True)
#remove any old pairs that were not just updated
config.mongo_db.asset_pair_market_info.remove({'last_updated': {'$lt': end_dt}})
logger.info("Recomposed 24h trade statistics for %i asset pairs: %s" % (len(pair_data), ', '.join(pair_data.keys())))
def parse_issuance(msg, msg_data):
if msg['category'] != 'issuances': return
if msg_data['status'] != 'valid': return
cur_block_index = config.state['cur_block']['block_index']
cur_block = config.state['cur_block']
def modify_extended_asset_info(asset, description):
"""adds an asset to asset_extended_info collection if the description is a valid json link. or, if the link
is not a valid json link, will remove the asset entry from the table if it exists"""
if util.is_valid_url(description, suffix='.json', allow_no_protocol=True):
config.mongo_db.asset_extended_info.update({'asset': asset},
{'$set': {
'info_url': description,
'info_status': 'needfetch',
'fetch_info_retry': 0, # retry ASSET_MAX_RETRY times to fetch info from info_url
'info_data': {},
'errors': []
}}, upsert=True)
#^ valid info_status settings: needfetch, valid, invalid, error
#additional fields will be added later in events, once the asset info is pulled
else:
config.mongo_db.asset_extended_info.remove({ 'asset': asset })
#remove any saved asset image data
imagePath = os.path.join(config.data_dir, config.SUBDIR_ASSET_IMAGES, asset + '.png')
if os.path.exists(imagePath):
os.remove(imagePath)
tracked_asset = config.mongo_db.tracked_assets.find_one(
{'asset': msg_data['asset']}, {'_id': 0, '_history': 0})
#^ pulls the tracked asset without the _id and history fields. This may be None
if msg_data['locked']: #lock asset
assert tracked_asset is not None
config.mongo_db.tracked_assets.update(
{'asset': msg_data['asset']},
{"$set": {
'_at_block': cur_block_index,
'_at_block_time': cur_block['block_time_obj'],
'_change_type': 'locked',
'locked': True,
},
"$push": {'_history': tracked_asset } }, upsert=False)
logger.info("Locking asset %s" % (msg_data['asset'],))
elif msg_data['transfer']: #transfer asset
assert tracked_asset is not None
config.mongo_db.tracked_assets.update(
{'asset': msg_data['asset']},
{"$set": {
'_at_block': cur_block_index,
'_at_block_time': cur_block['block_time_obj'],
'_change_type': 'transferred',
'owner': msg_data['issuer'],
},
"$push": {'_history': tracked_asset } }, upsert=False)
logger.info("Transferring asset %s to address %s" % (msg_data['asset'], msg_data['issuer']))
elif msg_data['quantity'] == 0 and tracked_asset is not None: #change description
config.mongo_db.tracked_assets.update(
{'asset': msg_data['asset']},
{"$set": {
'_at_block': cur_block_index,
'_at_block_time': cur_block['block_time_obj'],
'_change_type': 'changed_description',
'description': msg_data['description'],
},
"$push": {'_history': tracked_asset } }, upsert=False)
modify_extended_asset_info(msg_data['asset'], msg_data['description'])
logger.info("Changing description for asset %s to '%s'" % (msg_data['asset'], msg_data['description']))
else: #issue new asset or issue addition qty of an asset
if not tracked_asset: #new issuance
tracked_asset = {
'_change_type': 'created',
'_at_block': cur_block_index, #the block ID this asset is current for
'_at_block_time': cur_block['block_time_obj'],
#^ NOTE: (if there are multiple asset tracked changes updates in a single block for the same
# asset, the last one with _at_block == that block id in the history array is the
# final version for that asset at that block
'asset': msg_data['asset'],
'owner': msg_data['issuer'],
'description': msg_data['description'],
'divisible': msg_data['divisible'],
'locked': False,
'total_issued': int(msg_data['quantity']),
'total_issued_normalized': blockchain.normalize_quantity(msg_data['quantity'], msg_data['divisible']),
'_history': [] #to allow for block rollbacks
}
config.mongo_db.tracked_assets.insert(tracked_asset)
logger.info("Tracking new asset: %s" % msg_data['asset'])
modify_extended_asset_info(msg_data['asset'], msg_data['description'])
else: #issuing additional of existing asset
assert tracked_asset is not None
config.mongo_db.tracked_assets.update(
{'asset': msg_data['asset']},
{"$set": {
'_at_block': cur_block_index,
'_at_block_time': cur_block['block_time_obj'],
'_change_type': 'issued_more',
},
"$inc": {
'total_issued': msg_data['quantity'],
'total_issued_normalized': blockchain.normalize_quantity(msg_data['quantity'], msg_data['divisible'])
},
"$push": {'_history': tracked_asset} }, upsert=False)
logger.info("Adding additional %s quantity for asset %s" % (
blockchain.normalize_quantity(msg_data['quantity'], msg_data['divisible']), msg_data['asset']))
return True
