def compile_asset_market_info():
"""Run through all assets and compose and store market ranking information."""
mongo_db = config.mongo_db
if not config.CAUGHT_UP:
logging.warn("Not updating asset market info as CAUGHT_UP is false.")
return False
#grab the last block # we processed assets data off of
last_block_assets_compiled = mongo_db.app_config.find_one()['last_block_assets_compiled']
last_block_time_assets_compiled = util.get_block_time(last_block_assets_compiled)
#logging.debug("Comping info for assets traded since block %i" % last_block_assets_compiled)
current_block_index = config.CURRENT_BLOCK_INDEX #store now as it may change as we are compiling asset data :)
current_block_time = util.get_block_time(current_block_index)
if current_block_index == last_block_assets_compiled:
#all caught up -- call again in 10 minutes
return True
mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives()
all_traded_assets = list(set(list([config.BTC, config.XCP]) + list(mongo_db.trades.find({}, {'quote_asset': 1, '_id': 0}).distinct('quote_asset'))))
#######################
#get a list of all assets with a trade within the last 24h (not necessarily just against XCP and BTC)
# ^ this is important because compiled market info has a 24h vol parameter that designates total volume for the asset across ALL pairings
start_dt_1d = datetime.datetime.utcnow() - datetime.timedelta(days=1)
assets = list(set(
list(mongo_db.trades.find({'block_time': {'$gte': start_dt_1d}}).distinct('quote_asset'))
+ list(mongo_db.trades.find({'block_time': {'$gte': start_dt_1d}}).distinct('base_asset'))
))
for asset in assets:
market_info_24h = compile_24h_market_info(asset)
mongo_db.asset_market_info.update({'asset': asset}, {"$set": market_info_24h})
#for all others (i.e. no trade in the last 24 hours), zero out the 24h trade data
non_traded_assets = list(set(all_traded_assets) - set(assets))
mongo_db.asset_market_info.update( {'asset': {'$in': non_traded_assets}}, {"$set": {
'24h_summary': {'vol': 0, 'count': 0},
'24h_ohlc_in_{}'.format(config.XCP.lower()): {},
'24h_ohlc_in_{}'.format(config.BTC.lower()): {},
'24h_vol_price_change_in_{}'.format(config.XCP.lower()): None,
'24h_vol_price_change_in_{}'.format(config.BTC.lower()): None,
}}, multi=True)
logging.info("Block: %s -- Calculated 24h stats for: %s" % (current_block_index, ', '.join(assets)))
#######################
#get a list of all assets with a trade within the last 7d up against XCP and BTC
start_dt_7d = datetime.datetime.utcnow() - datetime.timedelta(days=7)
assets = list(set(
list(mongo_db.trades.find({'block_time': {'$gte': start_dt_7d}, 'base_asset': {'$in': [config.XCP, config.BTC]}}).distinct('quote_asset'))
+ list(mongo_db.trades.find({'block_time': {'$gte': start_dt_7d}}).distinct('base_asset'))
))
for asset in assets:
market_info_7d = compile_7d_market_info(asset)
mongo_db.asset_market_info.update({'asset': asset}, {"$set": market_info_7d})
non_traded_assets = list(set(all_traded_assets) - set(assets))
mongo_db.asset_market_info.update( {'asset': {'$in': non_traded_assets}}, {"$set": {
'7d_history_in_{}'.format(config.XCP.lower()): [],
'7d_history_in_{}'.format(config.BTC.lower()): [],
}}, multi=True)
logging.info("Block: %s -- Calculated 7d stats for: %s" % (current_block_index, ', '.join(assets)))
#######################
#update summary market data for assets traded since last_block_assets_compiled
#get assets that were traded since the last check with either BTC or XCP, and update their market summary data
assets = list(set(
list(mongo_db.trades.find({'block_index': {'$gt': last_block_assets_compiled}, 'base_asset': {'$in': [config.XCP, config.BTC]}}).distinct('quote_asset'))
+ list(mongo_db.trades.find({'block_index': {'$gt': last_block_assets_compiled}}).distinct('base_asset'))
))
#update our storage of the latest market info in mongo
for asset in assets:
logging.info("Block: %s -- Updating asset market info for %s ..." % (current_block_index, asset))
summary_info = compile_summary_market_info(asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price)
mongo_db.asset_market_info.update( {'asset': asset}, {"$set": summary_info}, upsert=True)
#######################
#next, compile market cap historicals (and get the market price data that we can use to update assets with new trades)
#NOTE: this algoritm still needs to be fleshed out some...I'm not convinced it's laid out/optimized like it should be
#start by getting all trades from when we last compiled this data
trades = mongo_db.trades.find({'block_index': {'$gt': last_block_assets_compiled}}).sort('block_index', pymongo.ASCENDING)
trades_by_block = [] #tracks assets compiled per block, as we only want to analyze any given asset once per block
trades_by_block_mapping = {}
#organize trades by block
for t in trades:
if t['block_index'] in trades_by_block_mapping:
assert trades_by_block_mapping[t['block_index']]['block_index'] == t['block_index']
assert trades_by_block_mapping[t['block_index']]['block_time'] == t['block_time']
trades_by_block_mapping[t['block_index']]['trades'].append(t)
else:
e = {'block_index': t['block_index'], 'block_time': t['block_time'], 'trades': [t,]}
trades_by_block.append(e)
trades_by_block_mapping[t['block_index']] = e
for t_block in trades_by_block:
#reverse the tradelist per block, and ensure that we only process an asset that hasn't already been processed for this block
# (as there could be multiple trades in a single block for any specific asset). we reverse the list because
# we'd rather process a later trade for a given asset, as the market price for that will take into account
# the earlier trades on that same block for that asset, and we don't want/need multiple cap points per block
assets_in_block = {}
mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives(end_dt=t_block['block_time'])
for t in reversed(t_block['trades']):
assets = []
if t['base_asset'] not in assets_in_block:
assets.append(t['base_asset'])
assets_in_block[t['base_asset']] = True
if t['quote_asset'] not in assets_in_block:
assets.append(t['quote_asset'])
assets_in_block[t['quote_asset']] = True
if not len(assets): continue
for asset in assets:
#recalculate the market cap for the asset this trade is for
asset_info = get_asset_info(asset, at_dt=t['block_time'])
(price_summary_in_xcp, price_summary_in_btc, price_in_xcp, price_in_btc, aggregated_price_in_xcp, aggregated_price_in_btc
) = get_xcp_btc_price_info(asset, mps_xcp_btc, xcp_btc_price, btc_xcp_price, with_last_trades=0, end_dt=t['block_time'])
market_cap_in_xcp, market_cap_in_btc = calc_market_cap(asset_info, price_in_xcp, price_in_btc)
#^ this will get price data from the block time of this trade back the standard number of days and trades
# to determine our standard market price, relative (anchored) to the time of this trade
for market_cap_as in (config.XCP, config.BTC):
market_cap = market_cap_in_xcp if market_cap_as == config.XCP else market_cap_in_btc
#if there is a previously stored market cap for this asset, add a new history point only if the two caps differ
prev_market_cap_history = mongo_db.asset_marketcap_history.find({'market_cap_as': market_cap_as, 'asset': asset,
'block_index': {'$lt': t['block_index']}}).sort('block_index', pymongo.DESCENDING).limit(1)
prev_market_cap_history = list(prev_market_cap_history)[0] if prev_market_cap_history.count() == 1 else None
if market_cap and (not prev_market_cap_history or prev_market_cap_history['market_cap'] != market_cap):
mongo_db.asset_marketcap_history.insert({
'block_index': t['block_index'],
'block_time': t['block_time'],
'asset': asset,
'market_cap': market_cap,
'market_cap_as': market_cap_as,
})
logging.info("Block %i -- Calculated market cap history point for %s as %s (mID: %s)" % (t['block_index'], asset, market_cap_as, t['message_index']))
mongo_db.app_config.update({}, {'$set': {'last_block_assets_compiled': current_block_index}})
return True
def compile_asset_market_info():
"""Run through all assets and compose and store market ranking information."""
mongo_db = config.mongo_db
if not config.CAUGHT_UP:
logging.warn("Not updating asset market info as CAUGHT_UP is false.")
return False
#grab the last block # we processed assets data off of
last_block_assets_compiled = mongo_db.app_config.find_one()['last_block_assets_compiled']
last_block_time_assets_compiled = util.get_block_time(last_block_assets_compiled)
#logging.debug("Comping info for assets traded since block %i" % last_block_assets_compiled)
current_block_index = config.CURRENT_BLOCK_INDEX #store now as it may change as we are compiling asset data :)
current_block_time = util.get_block_time(current_block_index)
if current_block_index == last_block_assets_compiled:
#all caught up -- call again in 10 minutes
return True
mps_xlt_ltc, xlt_ltc_price, ltc_xlt_price = get_price_primatives()
all_traded_assets = list(set(list([config.LTC, config.XLT]) + list(mongo_db.trades.find({}, {'quote_asset': 1, '_id': 0}).distinct('quote_asset'))))
#######################
#get a list of all assets with a trade within the last 24h (not necessarily just against XLT and LTC)
# ^ this is important because compiled market info has a 24h vol parameter that designates total volume for the asset across ALL pairings
start_dt_1d = datetime.datetime.utcnow() - datetime.timedelta(days=1)
assets = list(set(
list(mongo_db.trades.find({'block_time': {'$gte': start_dt_1d}}).distinct('quote_asset'))
+ list(mongo_db.trades.find({'block_time': {'$gte': start_dt_1d}}).distinct('base_asset'))
))
for asset in assets:
market_info_24h = compile_24h_market_info(asset)
mongo_db.asset_market_info.update({'asset': asset}, {"$set": market_info_24h})
#for all others (i.e. no trade in the last 24 hours), zero out the 24h trade data
non_traded_assets = list(set(all_traded_assets) - set(assets))
mongo_db.asset_market_info.update( {'asset': {'$in': non_traded_assets}}, {"$set": {
'24h_summary': {'vol': 0, 'count': 0},
'24h_ohlc_in_{}'.format(config.XLT.lower()): {},
'24h_ohlc_in_{}'.format(config.LTC.lower()): {},
'24h_vol_price_change_in_{}'.format(config.XLT.lower()): None,
'24h_vol_price_change_in_{}'.format(config.LTC.lower()): None,
}}, multi=True)
logging.info("Block: %s -- Calculated 24h stats for: %s" % (current_block_index, ', '.join(assets)))
#######################
#get a list of all assets with a trade within the last 7d up against XLT and LTC
start_dt_7d = datetime.datetime.utcnow() - datetime.timedelta(days=7)
assets = list(set(
list(mongo_db.trades.find({'block_time': {'$gte': start_dt_7d}, 'base_asset': {'$in': [config.XLT, config.LTC]}}).distinct('quote_asset'))
+ list(mongo_db.trades.find({'block_time': {'$gte': start_dt_7d}}).distinct('base_asset'))
))
for asset in assets:
market_info_7d = compile_7d_market_info(asset)
mongo_db.asset_market_info.update({'asset': asset}, {"$set": market_info_7d})
non_traded_assets = list(set(all_traded_assets) - set(assets))
mongo_db.asset_market_info.update( {'asset': {'$in': non_traded_assets}}, {"$set": {
'7d_history_in_{}'.format(config.XLT.lower()): [],
'7d_history_in_{}'.format(config.LTC.lower()): [],
}}, multi=True)
logging.info("Block: %s -- Calculated 7d stats for: %s" % (current_block_index, ', '.join(assets)))
#######################
#update summary market data for assets traded since last_block_assets_compiled
#get assets that were traded since the last check with either LTC or XLT, and update their market summary data
assets = list(set(
list(mongo_db.trades.find({'block_index': {'$gt': last_block_assets_compiled}, 'base_asset': {'$in': [config.XLT, config.LTC]}}).distinct('quote_asset'))
+ list(mongo_db.trades.find({'block_index': {'$gt': last_block_assets_compiled}}).distinct('base_asset'))
))
#update our storage of the latest market info in mongo
for asset in assets:
logging.info("Block: %s -- Updating asset market info for %s ..." % (current_block_index, asset))
summary_info = compile_summary_market_info(asset, mps_xlt_ltc, xlt_ltc_price, ltc_xlt_price)
mongo_db.asset_market_info.update( {'asset': asset}, {"$set": summary_info}, upsert=True)
#######################
#next, compile market cap historicals (and get the market price data that we can use to update assets with new trades)
#NOTE: this algoritm still needs to be fleshed out some...I'm not convinced it's laid out/optimized like it should be
#start by getting all trades from when we last compiled this data
trades = mongo_db.trades.find({'block_index': {'$gt': last_block_assets_compiled}}).sort('block_index', pymongo.ASCENDING)
trades_by_block = [] #tracks assets compiled per block, as we only want to analyze any given asset once per block
trades_by_block_mapping = {}
#organize trades by block
for t in trades:
if t['block_index'] in trades_by_block_mapping:
assert trades_by_block_mapping[t['block_index']]['block_index'] == t['block_index']
assert trades_by_block_mapping[t['block_index']]['block_time'] == t['block_time']
trades_by_block_mapping[t['block_index']]['trades'].append(t)
else:
e = {'block_index': t['block_index'], 'block_time': t['block_time'], 'trades': [t,]}
trades_by_block.append(e)
trades_by_block_mapping[t['block_index']] = e
for t_block in trades_by_block:
#reverse the tradelist per block, and ensure that we only process an asset that hasn't already been processed for this block
# (as there could be multiple trades in a single block for any specific asset). we reverse the list because
# we'd rather process a later trade for a given asset, as the market price for that will take into account
# the earlier trades on that same block for that asset, and we don't want/need multiple cap points per block
assets_in_block = {}
mps_xlt_ltc, xlt_ltc_price, ltc_xlt_price = get_price_primatives(end_dt=t_block['block_time'])
for t in reversed(t_block['trades']):
assets = []
if t['base_asset'] not in assets_in_block:
assets.append(t['base_asset'])
assets_in_block[t['base_asset']] = True
if t['quote_asset'] not in assets_in_block:
assets.append(t['quote_asset'])
assets_in_block[t['quote_asset']] = True
if not len(assets): continue
for asset in assets:
#recalculate the market cap for the asset this trade is for
asset_info = get_asset_info(asset, at_dt=t['block_time'])
(price_summary_in_xlt, price_summary_in_ltc, price_in_xlt, price_in_ltc, aggregated_price_in_xlt, aggregated_price_in_ltc
) = get_xlt_ltc_price_info(asset, mps_xlt_ltc, xlt_ltc_price, ltc_xlt_price, with_last_trades=0, end_dt=t['block_time'])
market_cap_in_xlt, market_cap_in_ltc = calc_market_cap(asset_info, price_in_xlt, price_in_ltc)
#^ this will get price data from the block time of this trade back the standard number of days and trades
# to determine our standard market price, relative (anchored) to the time of this trade
for market_cap_as in (config.XLT, config.LTC):
market_cap = market_cap_in_xlt if market_cap_as == config.XLT else market_cap_in_ltc
#if there is a previously stored market cap for this asset, add a new history point only if the two caps differ
prev_market_cap_history = mongo_db.asset_marketcap_history.find({'market_cap_as': market_cap_as, 'asset': asset,
'block_index': {'$lt': t['block_index']}}).sort('block_index', pymongo.DESCENDING).limit(1)
prev_market_cap_history = list(prev_market_cap_history)[0] if prev_market_cap_history.count() == 1 else None
if market_cap and (not prev_market_cap_history or prev_market_cap_history['market_cap'] != market_cap):
mongo_db.asset_marketcap_history.insert({
'block_index': t['block_index'],
'block_time': t['block_time'],
'asset': asset,
'market_cap': market_cap,
'market_cap_as': market_cap_as,
})
logging.info("Block %i -- Calculated market cap history point for %s as %s (mID: %s)" % (t['block_index'], asset, market_cap_as, t['message_index']))
mongo_db.app_config.update({}, {'$set': {'last_block_assets_compiled': current_block_index}})
return True
def compile_asset_market_info():
"""Run through all assets and compose and store market ranking information."""
mongo_db = config.mongo_db
if not config.CAUGHT_UP:
logging.warn("Not updating asset market info as CAUGHT_UP is false.")
return False
#grab the last block # we processed assets data off of
last_block_assets_compiled = mongo_db.app_config.find_one(
)['last_block_assets_compiled']
last_block_time_assets_compiled = util.get_block_time(
last_block_assets_compiled)
current_block_index = config.CURRENT_BLOCK_INDEX #store now as it may change as we are compiling asset data :)
current_block_time = util.get_block_time(current_block_index)
if current_block_index == last_block_assets_compiled:
#all caught up -- call again in 10 minutes
return True
mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives()
all_traded_assets = list(
set(
list([config.BTC, config.XCP]) + list(
mongo_db.trades.find({}, {
'quote_asset': 1,
'_id': 0
}).distinct('quote_asset'))))
start_dt_1d = datetime.datetime.utcnow() - datetime.timedelta(days=1)
assets = list(
set(
list(
mongo_db.trades.find({
'block_time': {
'$gte': start_dt_1d
}
}).distinct('quote_asset')) + list(
mongo_db.trades.find({
'block_time': {
'$gte': start_dt_1d
}
}).distinct('base_asset'))))
for asset in assets:
market_info_24h = compile_24h_market_info(asset)
mongo_db.asset_market_info.update({'asset': asset},
{"$set": market_info_24h})
#for all others (i.e. no trade in the last 24 hours), zero out the 24h trade data
non_traded_assets = list(set(all_traded_assets) - set(assets))
mongo_db.asset_market_info.update({'asset': {
'$in': non_traded_assets
}}, {
"$set": {
'24h_summary': {
'vol': 0,
'count': 0
},
'24h_ohlc_in_{}'.format(config.XCP.lower()): {},
'24h_ohlc_in_{}'.format(config.BTC.lower()): {},
'24h_vol_price_change_in_{}'.format(config.XCP.lower()): None,
'24h_vol_price_change_in_{}'.format(config.BTC.lower()): None,
}
},
multi=True)
logging.info("Block: %s -- Calculated 24h stats for: %s" %
(current_block_index, ', '.join(assets)))
start_dt_7d = datetime.datetime.utcnow() - datetime.timedelta(days=7)
assets = list(
set(
list(
mongo_db.trades.find({
'block_time': {
'$gte': start_dt_7d
},
'base_asset': {
'$in': [config.XCP, config.BTC]
}
}).distinct('quote_asset')) + list(
mongo_db.trades.find({
'block_time': {
'$gte': start_dt_7d
}
}).distinct('base_asset'))))
for asset in assets:
market_info_7d = compile_7d_market_info(asset)
mongo_db.asset_market_info.update({'asset': asset},
{"$set": market_info_7d})
non_traded_assets = list(set(all_traded_assets) - set(assets))
mongo_db.asset_market_info.update({'asset': {
'$in': non_traded_assets
}}, {
"$set": {
'7d_history_in_{}'.format(config.XCP.lower()): [],
'7d_history_in_{}'.format(config.BTC.lower()): [],
}
},
multi=True)
logging.info("Block: %s -- Calculated 7d stats for: %s" %
(current_block_index, ', '.join(assets)))
assets = list(
set(
list(
mongo_db.trades.find({
'block_index': {
'$gt': last_block_assets_compiled
},
'base_asset': {
'$in': [config.XCP, config.BTC]
}
}).distinct('quote_asset')) + list(
mongo_db.trades.find({
'block_index': {
'$gt': last_block_assets_compiled
}
}).distinct('base_asset'))))
for asset in assets:
logging.info("Block: %s -- Updating asset market info for %s ..." %
(current_block_index, asset))
summary_info = compile_summary_market_info(asset, mps_xcp_btc,
xcp_btc_price,
btc_xcp_price)
mongo_db.asset_market_info.update({'asset': asset},
{"$set": summary_info},
upsert=True)
trades = mongo_db.trades.find({
'block_index': {
'$gt': last_block_assets_compiled
}
}).sort('block_index', pymongo.ASCENDING)
trades_by_block = [
] #tracks assets compiled per block, as we only want to analyze any given asset once per block
trades_by_block_mapping = {}
#organize trades by block
for t in trades:
if t['block_index'] in trades_by_block_mapping:
assert trades_by_block_mapping[
t['block_index']]['block_index'] == t['block_index']
assert trades_by_block_mapping[
t['block_index']]['block_time'] == t['block_time']
trades_by_block_mapping[t['block_index']]['trades'].append(t)
else:
e = {
'block_index': t['block_index'],
'block_time': t['block_time'],
'trades': [
t,
]
}
trades_by_block.append(e)
trades_by_block_mapping[t['block_index']] = e
for t_block in trades_by_block:
assets_in_block = {}
mps_xcp_btc, xcp_btc_price, btc_xcp_price = get_price_primatives(
end_dt=t_block['block_time'])
for t in reversed(t_block['trades']):
assets = []
if t['base_asset'] not in assets_in_block:
assets.append(t['base_asset'])
assets_in_block[t['base_asset']] = True
if t['quote_asset'] not in assets_in_block:
assets.append(t['quote_asset'])
assets_in_block[t['quote_asset']] = True
if not len(assets): continue
for asset in assets:
#recalculate the market cap for the asset this trade is for
asset_info = get_asset_info(asset, at_dt=t['block_time'])
if not asset_info: continue
(price_summary_in_xcp, price_summary_in_btc, price_in_xcp,
price_in_btc, aggregated_price_in_xcp,
aggregated_price_in_btc) = get_xcp_btc_price_info(
asset,
mps_xcp_btc,
xcp_btc_price,
btc_xcp_price,
with_last_trades=0,
end_dt=t['block_time'])
market_cap_in_xcp, market_cap_in_btc = calc_market_cap(
asset_info, price_in_xcp, price_in_btc)
for market_cap_as in (config.XCP, config.BTC):
market_cap = market_cap_in_xcp if market_cap_as == config.XCP else market_cap_in_btc
#if there is a previously stored market cap for this asset, add a new history point only if the two caps differ
prev_market_cap_history = mongo_db.asset_marketcap_history.find(
{
'market_cap_as': market_cap_as,
'asset': asset,
'block_index': {
'$lt': t['block_index']
}
}).sort('block_index', pymongo.DESCENDING).limit(1)
prev_market_cap_history = list(
prev_market_cap_history
)[0] if prev_market_cap_history.count() == 1 else None
if market_cap and (not prev_market_cap_history
or prev_market_cap_history['market_cap']
!= market_cap):
mongo_db.asset_marketcap_history.insert({
'block_index':
t['block_index'],
'block_time':
t['block_time'],
'asset':
asset,
'market_cap':
market_cap,
'market_cap_as':
market_cap_as,
})
logging.info(
"Block %i -- Calculated market cap history point for %s as %s (mID: %s)"
% (t['block_index'], asset, market_cap_as,
t['message_index']))
mongo_db.app_config.update(
{}, {'$set': {
'last_block_assets_compiled': current_block_index
}})
return True