def test_010_getNotExists(self):
cache = Cache()
with self.assertRaises(Exception) as context:
cache.get("Bank1", "key1")
self.assertTrue("Key key1 doesn't exist" in str(context.exception))
def test_070_timeout(self):
cache = Cache()
cache.put("Bank1", CacheItem("key1", "val1", 2))
time.sleep(1)
cache.touch("Bank1", "key1")
time.sleep(1)
self.assertIsNotNone(cache.get("Bank1", "key1"))
time.sleep(1)
with self.assertRaises(Exception) as context:
cache.get("Bank1", "key1")
self.assertTrue("Key key1 doesn't exist" in str(context.exception))
def test_cache_delete_least_popular_elements(self):
cache = Cache(3, 1)
cache.put('key1', 'value1')
cache.put('key2', 'value2')
cache.put('key3', 'value3')
hits_plan = {'key1': 2, 'key2': 6, 'key3': 3}
for key, value in hits_plan.items():
for _ in range(value):
cache.get(key)
cache.put('key4', 'value4')
self.assertFalse(cache.is_key('key1'))
self.assertTrue(cache.is_key('key4'))
def test_080_incr(self):
cache = Cache()
cache.put("BankNum", CacheItem("key1", 1))
self.assertEqual(cache.get("BankNum", "key1").value, 1)
self.assertEqual(cache.incr("BankNum", "key1", 1).value, 2)
self.assertEqual(cache.incr("BankNum", "key1", -3).value, -1)
cache.put("BankNum", CacheItem("key2", "1"))
self.assertEqual(cache.incr("BankNum", "key2", 1).value, 2)
cache.put("BankNum", CacheItem("key3", "1a"))
with self.assertRaises(Exception) as context:
cache.incr("BankNum", "key3", 1)
cache.put("BankNum", CacheItem("key4", 1.3))
self.assertEqual(cache.get("BankNum", "key4").value, 1.3)
self.assertEqual(cache.incr("BankNum", "key4", 1.2).value, 2.5)
def test_cache_store_elements_hits(self):
cache = Cache(5, 1)
cache.put('key1', 'value1')
cache.put('key2', 'value2')
cache.put('key3', 'value3')
cache.put('key4', 'value4')
cache.put('key5', 'value5')
hits_plan = {'key1': 5, 'key2': 4, 'key3': 3, 'key4': 2, 'key5': 1}
for key, value in hits_plan.items():
for _ in range(value):
cache.get(key)
self.assertEqual(5, cache.hits[3])
self.assertEqual(4, cache.hits[4])
self.assertEqual(3, cache.hits[0])
self.assertEqual(2, cache.hits[1])
self.assertEqual(1, cache.hits[2])
class Manager(object):
def __init__(self, basedir=None):
self.cache = Cache(basedir)
self.auto_tagger = AutoTagger(self.get('auto-tags'))
def __backup_thread(self):
while True:
self.cache.dump()
time.sleep(self.get('settings')['backup']['interval'])
def __sync_thread(self):
while True:
try:
resources_tags = self.auto_tagger.process(
Crawler(
self.get('black-list'),
self.get('white-list'),
self.get('crawled-resources'),
).crawl())
SyncAgent(
self.get('settings')['server'],
self.get('settings')['user-token'],
self.get('settings')['device-token'],
).sync(resources_tags)
except Exception as new_exception:
print('[ERROR]: When trying to sync: {0}'.format(
new_exception.message))
else:
self.get('crawled-resources').update(
set(resource for resource, _ in resources_tags))
time.sleep(self.get('settings')['sync']['interval'])
def add_to_black_list(self, directory):
return Crawler.add_to_dirlist(self.get('black-list'), directory)
def add_to_white_list(self, directory):
return Crawler.add_to_dirlist(self.get('white-list'), directory)
def get(self, key):
return self.cache.get(key)
def start_backup_daemon(self):
DaemonThread(target=self.__backup_thread).start()
def start_sync_daemon(self):
DaemonThread(target=self.__sync_thread).start()
class Manager(object):
def __init__(self, basedir=None):
self.cache = Cache(basedir)
self.auto_tagger = AutoTagger(self.get('auto-tags'))
def __backup_thread(self):
while True:
self.cache.dump()
time.sleep(self.get('settings')['backup']['interval'])
def __sync_thread(self):
while True:
try:
resources_tags = self.auto_tagger.process(
Crawler(
self.get('black-list'),
self.get('white-list'),
self.get('crawled-resources'),
).crawl()
)
SyncAgent(
self.get('settings')['server'],
self.get('settings')['user-token'],
self.get('settings')['device-token'],
).sync(resources_tags)
except Exception as new_exception:
print('[ERROR]: When trying to sync: {0}'.format(new_exception.message))
else:
self.get('crawled-resources').update(set(resource for resource, _ in resources_tags))
time.sleep(self.get('settings')['sync']['interval'])
def add_to_black_list(self, directory):
return Crawler.add_to_dirlist(self.get('black-list'), directory)
def add_to_white_list(self, directory):
return Crawler.add_to_dirlist(self.get('white-list'), directory)
def get(self, key):
return self.cache.get(key)
def start_backup_daemon(self):
DaemonThread(target=self.__backup_thread).start()
def start_sync_daemon(self):
DaemonThread(target=self.__sync_thread).start()
class CacheTest(unittest.TestCase):
#===============Initial==========================
def setUp(self):
self.c = Cache(5, 1)
keys = list(map(lambda x: chr(x), range(97, 102)))
values = list(range(2, 7))
for i in range(5):
self.c.put(keys[i], values[i])
def tearDown(self):
del self.c
#==============Testing===================
def test_request_increment(self):
id = self.c.find("a", self.c.keys)
before = self.c.hits[id]
self.c.get("a")
after = self.c.hits[id]
self.assertTrue(after == before + 1)
def test_put(self):
x = Cache(5, 1)
x.put("key", "value")
self.assertTrue(x.get("key") == "value")
def test_remove(self):
self.c.get("a")
self.c.get("b")
self.c.get("c")
self.c.get("e")
self.c.remove()
self.assertEqual(self.c.get("d"), None)
def test_put_in_full(self):
self.c.get("a")
self.c.get("b")
self.c.get("c")
self.c.get("e")
self.assertTrue(self.c.get("key") == None)
self.c.put("key", "value")
self.assertFalse(self.c.get("key") == None)
self.assertTrue(self.c.get("d") == None)
def test_put(self):
x = Cache(5, 1)
x.put("key", "value")
self.assertTrue(x.get("key") == "value")
from Cache import Cache
import sys
parameters = sys.argv[1].split(':')
if((parameters[0] == '') and (parameters[1] == '') and (parameters[0] == '')):
cache = Cache()
else:
cache = Cache(nsets=int(parameters[0]),bsize=int(parameters[1]),assoc=int(parameters[2]))
file = open(sys.argv[2], 'r')
in_address = file.readlines()
i = 0
for address in in_address:
cache.get(address)
i += 1
if i % (len(in_address)/4) == 0:
print('Processing...')
cache.statistic()
file.close()
class DetailProcessor(object):
coinGecko = CoinGeckoAPI()
def __init__(self):
self.isServiceAvailable = True
signal.signal(signal.SIGINT, self.exit_gracefully)
signal.signal(signal.SIGTERM, self.exit_gracefully)
self.logging = error_reporting.Client()
self.cache = Cache(ttl=60)
context = zmq.Context.instance()
self.socket = context.socket(zmq.ROUTER)
self.socket.bind("tcp://*:6900")
print("[Startup]: Detail Server is online")
def exit_gracefully(self):
print("[Startup]: Detail Server is exiting")
self.socket.close()
self.isServiceAvailable = False
def run(self):
while self.isServiceAvailable:
try:
response = None, None
origin, delimeter, clientId, service, request = self.socket.recv_multipart(
)
request = pickle.loads(zlib.decompress(request))
if request.timestamp + 30 < time.time(): continue
if service == b"detail":
response = self.request_detail(request)
except (KeyboardInterrupt, SystemExit):
return
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]:
self.logging.report_exception()
finally:
try:
self.socket.send_multipart([
origin, delimeter,
zlib.compress(pickle.dumps(response, -1))
])
except:
pass
def request_detail(self, request):
payload, tradeMessage, updatedTradeMessage = None, None, None
for platform in request.platforms:
request.set_current(platform=platform)
hashCode = hash(request.requests[platform])
fromCache = False
if request.can_cache() and self.cache.has(hashCode):
payload, updatedQuoteMessage = self.cache.get(hashCode), None
fromCache = True
elif platform == "CoinGecko":
payload, updatedQuoteMessage = self.request_coingecko_details(
request)
elif platform == "IEXC":
payload, updatedQuoteMessage = self.request_iexc_details(
request)
if payload is not None:
if request.can_cache() and not fromCache:
self.cache.set(hashCode, payload)
if request.authorId != 401328409499664394 and request.requests[
platform].ticker.base is not None:
database.document("dataserver/statistics/{}/{}".format(
platform, str(uuid.uuid4()))).set({
"timestamp":
time.time(),
"authorId":
str(request.authorId),
"ticker": {
"base": request.requests[platform].ticker.base,
"quote":
request.requests[platform].ticker.quote,
"id": request.requests[platform].ticker.id,
"bias": request.parserBias
},
"exchange":
None if request.requests[platform].exchange is None
else request.requests[platform].exchange.id
})
return payload, updatedTradeMessage
elif updatedTradeMessage is not None:
tradeMessage = updatedTradeMessage
return None, tradeMessage
def request_coingecko_details(self, request):
ticker = request.get_ticker()
try:
try:
companyData = self.coinGecko.get_coin_by_id(
id=ticker.symbol,
localization="false",
tickers=False,
market_data=True,
community_data=True,
developer_data=True)
except:
return None, None
description = md(companyData["description"].get(
"en", "No description"))
descriptionParagraphs = description.split("\r\n\r\n")
textLength = [len(descriptionParagraphs[0])]
for i in range(1, len(descriptionParagraphs)):
nextLength = textLength[-1] + len(descriptionParagraphs[i])
if nextLength > 1000: break
textLength.append(nextLength)
description = "\n".join(
descriptionParagraphs[:len(textLength)]
) + "\n[Read more on CoinGecko](https://www.coingecko.com/coins/{})".format(
ticker.symbol)
payload = {
"name":
"{} ({})".format(companyData["name"], ticker.base),
"description":
description,
"url":
None if companyData["links"]["homepage"][0] == "" else
companyData["links"]["homepage"][0],
"rank":
companyData["market_data"]["market_cap_rank"],
"image":
companyData["image"]["large"],
"marketcap":
None if companyData["market_data"]["market_cap"] is None else
companyData["market_data"]["market_cap"].get("usd"),
"volume":
None if companyData["market_data"]["total_volume"] is None else
companyData["market_data"]["total_volume"].get("usd"),
"industry":
None,
"info":
None,
"supply": {
"total":
None if companyData["market_data"]["total_supply"] is None
else companyData["market_data"]["total_supply"],
"circulating":
None
if companyData["market_data"]["circulating_supply"] is None
else companyData["market_data"]["circulating_supply"]
},
"score": {
"developer": companyData["developer_score"],
"community": companyData["community_score"],
"liquidity": companyData["liquidity_score"],
"public interest": companyData["public_interest_score"]
},
"price": {
"current":
companyData["market_data"]["current_price"].get("usd"),
"ath":
companyData["market_data"]["ath"].get("usd"),
"atl":
companyData["market_data"]["atl"].get("usd"),
"per":
None
},
"change": {
"past day":
companyData["market_data"]
["price_change_percentage_24h_in_currency"].get("usd"),
"past month":
companyData["market_data"]
["price_change_percentage_30d_in_currency"].get("usd"),
"past year":
companyData["market_data"]
["price_change_percentage_1y_in_currency"].get("usd")
},
"sourceText":
"from CoinGecko",
"platform":
"CoinGecko",
}
return payload, None
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]: self.logging.report_exception()
return None, None
def request_iexc_details(self, request):
ticker = request.get_ticker()
try:
try:
stock = Stock(ticker.id, token=os.environ["IEXC_KEY"])
companyData = stock.get_company().loc[ticker.id]
rawData = stock.get_quote().loc[ticker.id]
except:
return None, None
try:
coinThumbnail = stock.get_logo().loc[ticker.id]["url"]
except:
coinThumbnail = None
payload = {
"name":
"{} ({})".format(companyData["companyName"],
companyData["symbol"]),
"description":
companyData["description"],
"url":
companyData["website"],
"rank":
None,
"image":
coinThumbnail,
"marketcap":
rawData["marketCap"],
"volume":
None,
"industry":
companyData["industry"],
"info": {
"location":
"{}{}, {}, {}, {}".format(
companyData["address"],
"" if companyData["address2"] is None else ", " +
companyData["address2"], companyData["city"],
companyData["state"], companyData["country"]),
"employees":
companyData["employees"]
},
"supply":
None,
"score":
None,
"price": {
"current":
rawData["delayedPrice"] if rawData["latestPrice"] is None
else rawData["latestPrice"],
"ath":
None,
"atl":
None,
"per":
rawData["peRatio"]
},
"change": {
"past day": rawData["changePercent"],
"past month": None,
"past year": None
},
"sourceText":
"provided by IEX Cloud",
"platform":
"IEX Cloud",
}
return payload, None
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]: self.logging.report_exception()
return None, None
class ChainDb(object):
def __init__(self, settings, datadir, log, mempool, netmagic,
readonly=False, fast_dbm=False):
self.settings = settings
self.log = log
self.mempool = mempool
self.readonly = readonly
self.netmagic = netmagic
self.fast_dbm = fast_dbm
self.blk_cache = Cache(750)
self.orphans = {}
self.orphan_deps = {}
if readonly:
mode_str = 'r'
else:
mode_str = 'c'
if fast_dbm:
self.log.write("Opening database in fast mode")
mode_str += 'f'
self.misc = gdbm.open(datadir + '/misc.dat', mode_str)
self.blocks = gdbm.open(datadir + '/blocks.dat', mode_str)
self.height = gdbm.open(datadir + '/height.dat', mode_str)
self.blkmeta = gdbm.open(datadir + '/blkmeta.dat', mode_str)
self.tx = gdbm.open(datadir + '/tx.dat', mode_str)
if 'height' not in self.misc:
self.log.write("INITIALIZING EMPTY BLOCKCHAIN DATABASE")
self.misc['height'] = str(-1)
self.misc['msg_start'] = self.netmagic.msg_start
self.misc['tophash'] = ser_uint256(0L)
self.misc['total_work'] = hex(0L)
if 'msg_start' not in self.misc or (self.misc['msg_start'] != self.netmagic.msg_start):
self.log.write("Database magic number mismatch. Data corruption or incorrect network?")
raise RuntimeError
def dbsync(self):
self.misc.sync()
self.blocks.sync()
self.height.sync()
self.blkmeta.sync()
self.tx.sync()
def puttxidx(self, txhash, txidx):
ser_txhash = ser_uint256(txhash)
if ser_txhash in self.tx:
old_txidx = self.gettxidx(txhash)
self.log.write("WARNING: overwriting duplicate TX %064x, height %d, oldblk %064x, oldspent %x, newblk %064x" % (txhash, self.getheight(), old_txidx.blkhash, old_txidx.spentmask, txidx.blkhash))
self.tx[ser_txhash] = (hex(txidx.blkhash) + ' ' +
hex(txidx.spentmask))
return True
def gettxidx(self, txhash):
ser_txhash = ser_uint256(txhash)
if ser_txhash not in self.tx:
return None
ser_value = self.tx[ser_txhash]
pos = string.find(ser_value, ' ')
txidx = TxIdx()
txidx.blkhash = long(ser_value[:pos], 16)
txidx.spentmask = long(ser_value[pos+1:], 16)
return txidx
def gettx(self, txhash):
txidx = self.gettxidx(txhash)
if txidx is None:
return None
block = self.getblock(txidx.blkhash)
for tx in block.vtx:
tx.calc_sha256()
if tx.sha256 == txhash:
return tx
self.log.write("ERROR: Missing TX %064x in block %064x" % (txhash, txidx.blkhash))
return None
def haveblock(self, blkhash, checkorphans):
if self.blk_cache.exists(blkhash):
return True
if checkorphans and blkhash in self.orphans:
return True
ser_hash = ser_uint256(blkhash)
if ser_hash in self.blocks:
return True
return False
def have_prevblock(self, block):
if self.getheight() < 0 and block.sha256 == self.netmagic.block0:
return True
if self.haveblock(block.hashPrevBlock, False):
return True
return False
def getblock(self, blkhash):
block = self.blk_cache.get(blkhash)
if block is not None:
return block
ser_hash = ser_uint256(blkhash)
if ser_hash not in self.blocks:
return None
f = cStringIO.StringIO(self.blocks[ser_hash])
block = CBlock()
block.deserialize(f)
self.blk_cache.put(blkhash, block)
return block
def spend_txout(self, txhash, n_idx):
txidx = self.gettxidx(txhash)
if txidx is None:
return False
txidx.spentmask |= (1L << n_idx)
self.puttxidx(txhash, txidx)
return True
def clear_txout(self, txhash, n_idx):
txidx = self.gettxidx(txhash)
if txidx is None:
return False
txidx.spentmask &= ~(1L << n_idx)
self.puttxidx(txhash, txidx)
return True
def unique_outpts(self, block):
outpts = {}
txmap = {}
for tx in block.vtx:
if tx.is_coinbase:
continue
txmap[tx.sha256] = tx
for txin in tx.vin:
v = (txin.prevout.hash, txin.prevout.n)
if v in outs:
return None
outpts[v] = False
return (outpts, txmap)
def spent_outpts(self, block):
# list of outpoints this block wants to spend
l = self.unique_outpts(block)
if l is None:
return None
outpts = l[0]
txmap = l[1]
spendlist = {}
# pass 1: if outpoint in db, make sure it is unspent
for k in outpts.iterkeys():
txidx = self.gettxidx(k[0])
if txidx is None:
continue
if k[1] > 100000: # outpoint index sanity check
return None
if txidx.spentmask & (1L << k[1]):
return None
outpts[k] = True # skip in pass 2
# pass 2: remaining outpoints must exist in this block
for k, v in outpts.iteritems():
if v:
continue
if k[0] not in txmap: # validate txout hash
return None
tx = txmap[k[0]] # validate txout index (n)
if k[1] >= len(tx.vout):
return None
# outpts[k] = True # not strictly necessary
return outpts.keys()
def test_000_basic(self):
cache = Cache()
cache.put("Bank1", CacheItem("key1", "val1"))
cache.put("Bank2", CacheItem("key1", "val2"))
self.assertEqual(cache.get("Bank1", "key1").value, 'val1')
self.assertEqual(cache.get("Bank2", "key1").value, 'val2')
def run(movie_dir, html_output_flag, limit):
"""This is the real entry point for the program"""
#A class to help lookup movie titles
movielookup = MovieLookup()
#Match files in a given directory
matcher = Matcher(Config.movie_match_regex, Config.allowed_file_types)
#Used to find an imdb id from movie filename
id_finder = IdFinder()
#Used for caching movie data
movie_cache = Cache(Config.movie_cache_file)
#First, let's match files which match the regex and have the
#required file extensions in the given directory
matcher.find_in_directory(movie_dir)
movie_matches = matcher.get_matches()
unmatched = matcher.get_ignored()
#normalise the matches (the filenames will be used as movie titles)
normalised_movie_matches = Normaliser\
.normalise_list_and_remove_trailing_number(movie_matches)
#Now we lookup successful matches, first in the cache, then online
movie_data = {} #successful lookup data will go here
failed_lookups = [] #we will do something with failed lookups later...
count = 0 #used to limit the number of lookups we will do
for title in normalised_movie_matches:
count += 1
if count >= limit:#check that we don't go over the arbitrary limit
break
#Check if the movie is in our cache
cached_movie = movie_cache.get(title)
if cached_movie:
movie_data[title] = cached_movie
#Otherwise, lookup using API
else:
#look up each movie in the list
lookup_data = movielookup.lookup_by_title(title)
#check if we found a movie
if MovieDataUtil.is_valid_lookup_result(lookup_data):
movie_data[title] = lookup_data
#great, let's also add it to the cache
movie_cache.add_to_cache(title, lookup_data)
else:
failed_lookups.append(title)
#now we will try to correct the failed lookups
#by using google to find each imdb id
id_lookup_dict = id_finder.find_id_by_title_list(failed_lookups)
#reset the failed lookups
failed_lookups = [] #there should be a lot less now...
title_corrections = 0 #count how many corrections we actually found
#Now lookup using the new ids which we found
for title, found_id in id_lookup_dict.items():
if found_id != None:
#we found an id, now let's look the movie up by its id
lookup_data = movielookup.lookup_by_id(found_id)
#theoretically this should always be true
#unless we got an invalid id somehow...
if MovieDataUtil.is_valid_lookup_result(lookup_data):
movie_data[title] = lookup_data
title_corrections += 1
#great, let's also add it to the cache
movie_cache.add_to_cache(title, lookup_data)
else:
failed_lookups.append(title)
else:
failed_lookups.append(title)
#Save the updated cache
movie_cache.save_cache_to_disk()
#sort the data by imdb id
movie_data = MovieDataUtil.sort_movie_data(movie_data)
#Output the data
if html_output_flag:
logging.debug('Loading template from: %s', Config.template_directory)
template_environment = Environment( \
loader=FileSystemLoader( \
Config.template_directory), trim_blocks=True)
print template_environment.get_template('main.html').render(
movie_lookup_data=movie_data,
failed_lookups=failed_lookups,
unmatched=unmatched,
title_corrections=title_corrections,
datetime=time.strftime("%c"),
version=__version__,
author=__author__,
cache_stats=movie_cache.cache_stats(),
)
else:
simple_output(movie_data, failed_lookups, unmatched)
# and open the template in the editor.
__author__ = "ilausuch"
__date__ = "$13-jun-2017 20:33:29$"
import sys
import os
sys.path.append(os.path.abspath("../Core"))
sys.path.append(os.path.abspath("../Addons"))
from Timer import Timer
from Cache import Cache, CacheItem
if __name__ == "__main__":
cache = Cache()
count = 100000
timer = Timer()
for i in range(0, count):
cache.put("Bank1", CacheItem(i, i))
print("{0} puts in {1} seconds".format(count, timer.end()))
timer = Timer()
for i in range(0, count):
cache.get("Bank1", i)
print("{0} gets in {1} seconds".format(count, timer.end()))
class ChainDb(object):
def __init__(self, settings, datadir, log, mempool, netmagic,
readonly=False, fast_dbm=False):
self.settings = settings
self.log = log
self.mempool = mempool
self.readonly = readonly
self.netmagic = netmagic
self.fast_dbm = fast_dbm
self.blk_cache = Cache(750)
self.orphans = {}
self.orphan_deps = {}
# LevelDB to hold:
# tx:* transaction outputs
# misc:* state
# height:* list of blocks at height h
# blkmeta:* block metadata
# blocks:* block seek point in stream
self.blk_write = io.BufferedWriter(io.FileIO(datadir + '/blocks.dat','ab'))
self.blk_read = io.BufferedReader(io.FileIO(datadir + '/blocks.dat','rb'))
self.db = leveldb.LevelDB(datadir + '/leveldb')
try:
self.db.Get('misc:height')
except KeyError:
self.log.write("INITIALIZING EMPTY BLOCKCHAIN DATABASE")
batch = leveldb.WriteBatch()
batch.Put('misc:height', str(-1))
batch.Put('misc:msg_start', self.netmagic.msg_start)
batch.Put('misc:tophash', ser_uint256(0L))
batch.Put('misc:total_work', hex(0L))
self.db.Write(batch)
try:
start = self.db.Get('misc:msg_start')
if start != self.netmagic.msg_start: raise KeyError
except KeyError:
self.log.write("Database magic number mismatch. Data corruption or incorrect network?")
raise RuntimeError
def puttxidx(self, txhash, txidx, batch=None):
ser_txhash = ser_uint256(txhash)
try:
self.db.Get('tx:'+ser_txhash)
old_txidx = self.gettxidx(txhash)
self.log.write("WARNING: overwriting duplicate TX %064x, height %d, oldblk %064x, oldspent %x, newblk %064x" % (txhash, self.getheight(), old_txidx.blkhash, old_txidx.spentmask, txidx.blkhash))
except KeyError:
pass
batch = self.db if batch is not None else batch
batch.Put('tx:'+ser_txhash, hex(txidx.blkhash) + ' ' +
hex(txidx.spentmask))
return True
def gettxidx(self, txhash):
ser_txhash = ser_uint256(txhash)
try:
ser_value = self.db.Get('tx:'+ser_txhash)
except KeyError:
return None
pos = string.find(ser_value, ' ')
txidx = TxIdx()
txidx.blkhash = long(ser_value[:pos], 16)
txidx.spentmask = long(ser_value[pos+1:], 16)
return txidx
def gettx(self, txhash):
txidx = self.gettxidx(txhash)
if txidx is None:
return None
block = self.getblock(txidx.blkhash)
for tx in block.vtx:
tx.calc_sha256()
if tx.sha256 == txhash:
return tx
self.log.write("ERROR: Missing TX %064x in block %064x" % (txhash, txidx.blkhash))
return None
def haveblock(self, blkhash, checkorphans):
if self.blk_cache.exists(blkhash):
return True
if checkorphans and blkhash in self.orphans:
return True
ser_hash = ser_uint256(blkhash)
try:
self.db.Get('blocks:'+ser_hash)
return True
except KeyError:
return False
def have_prevblock(self, block):
if self.getheight() < 0 and block.sha256 == self.netmagic.block0:
return True
if self.haveblock(block.hashPrevBlock, False):
return True
return False
def getblock(self, blkhash):
block = self.blk_cache.get(blkhash)
if block is not None:
return block
ser_hash = ser_uint256(blkhash)
try:
# Lookup the block index, seek in the file
fpos = long(self.db.Get('blocks:'+ser_hash))
self.blk_read.seek(fpos)
# read and decode "block" msg
msg = message_read(self.netmagic, self.blk_read)
if msg is None:
return None
block = msg.block
except KeyError:
return None
self.blk_cache.put(blkhash, block)
return block
def spend_txout(self, txhash, n_idx, batch=None):
txidx = self.gettxidx(txhash)
if txidx is None:
return False
txidx.spentmask |= (1L << n_idx)
self.puttxidx(txhash, txidx, batch)
return True
def clear_txout(self, txhash, n_idx, batch=None):
txidx = self.gettxidx(txhash)
if txidx is None:
return False
txidx.spentmask &= ~(1L << n_idx)
self.puttxidx(txhash, txidx, batch)
return True
def unique_outpts(self, block):
outpts = {}
txmap = {}
for tx in block.vtx:
if tx.is_coinbase:
continue
txmap[tx.sha256] = tx
for txin in tx.vin:
v = (txin.prevout.hash, txin.prevout.n)
if v in outs:
return None
outpts[v] = False
return (outpts, txmap)
def spent_outpts(self, block):
# list of outpoints this block wants to spend
l = self.unique_outpts(block)
if l is None:
return None
outpts = l[0]
txmap = l[1]
spendlist = {}
# pass 1: if outpoint in db, make sure it is unspent
for k in outpts.iterkeys():
txidx = self.gettxidx(k[0])
if txidx is None:
continue
if k[1] > 100000: # outpoint index sanity check
return None
if txidx.spentmask & (1L << k[1]):
return None
outpts[k] = True # skip in pass 2
# pass 2: remaining outpoints must exist in this block
for k, v in outpts.iteritems():
if v:
continue
if k[0] not in txmap: # validate txout hash
return None
tx = txmap[k[0]] # validate txout index (n)
if k[1] >= len(tx.vout):
return None
# outpts[k] = True # not strictly necessary
return outpts.keys()
class Core(object):
""" Starts the :class:`Cache`
Arguments
-----------
db : :data:`bfly.Butterfly._db_type`
A fully-loaded database
Attributes
------------
_db: :data:`bfly.Butterfly._db_type`
Taken from first argument ``db``
_cache: :class:`Cache`
Able to store images and metadata using \
:class:`UtilityLayer.RUNTIME` instance \
from ``db`` argument
"""
def __init__(self, db):
# Get DB Terms
self._db = db
RUNTIME = db.RUNTIME
# Make Cache with keywords
self._cache = Cache(RUNTIME)
#####
# All methods to load data
# 1. get_info answers an InfoQuery i_query.
# 2. get_data answers a DataQuery d_query.
#
# Both get_info or get_data call update_query.
#
# To give answers , update_query uses _cache or:
# 1. make_data_query if only i_query given.
# 2. make_tile_query with new or given d_query.
#####
@staticmethod
def get_groups(i_query):
""" dumps group list for ``i_query`` as a string
Arguments
----------
i_query: :class:`QueryLayer.InfoQuery`
A request for a list of groups
Returns
--------
str
A list of all groups for the ``i_query``
"""
return i_query.dump
def get_edits(self, i_query, msg={}):
""" dumps websocket updates to ``i_query`` as a string
Calls :meth:`update_query` with more information\
from the cache or from the properties of a tile.
Arguments
----------
i_query: :class:`QueryLayer.InfoQuery`
A request for information
Returns
--------
str
Wesocket info for :class:`QueryLayer.InfoQuery`
"""
keywords = self.update_query(i_query)
# Update current query with preloaded terms
i_query.update_source(keywords)
# Execute weboscket command if needed
changed = i_query.websocket_edit(msg)
# Add to cache and query
if len(changed):
keywords.update(changed)
i_query.update_source(keywords)
self._cache.set(i_query.key, keywords)
# Return the i_query info
return i_query.dump
def get_info(self, i_query):
""" dumps answer to ``i_query`` as a string
Calls :meth:`update_query` with more information\
from the cache or from the properties of a tile.
Arguments
----------
i_query: :class:`QueryLayer.InfoQuery`
A request for information
Returns
--------
str
Channel info for :class:`QueryLayer.InfoQuery`
"""
keywords = self.update_query(i_query)
# Update current query with preloaded terms
i_query.update_source(keywords)
# Return the i_query info
return i_query.dump
def get_dataset(self, i_query):
""" dumps dataset from ``i_query`` as a string
Calls :meth:`update_query` with more information\
from the cache or from the properties of a tile.
Arguments
----------
i_query: :class:`QueryLayer.InfoQuery`
A request for information
Returns
--------
str
Dataset info for :class:`QueryLayer.InfoQuery`
"""
all_channels = []
# Update all the channels in the dataset
for channel in i_query.channels:
# Set the i_query to a given channel
i_query.set_channel(channel)
# Update the info for the channel
keywords = self.update_query(i_query)
# Add additionl keywords if needed
channel_key = i_query.OUTPUT.INFO.CHANNEL.NAME
keywords[channel_key] = channel[channel_key]
# Add to list of channels
all_channels.append(keywords)
# Clear the channel data
i_query.set_channel({})
# Update current query with preloaded terms
return i_query.dump_dataset(all_channels)
def get_data(self, d_query):
""" dumps answer to ``d_query`` as a string
Calls :meth:`update_query` with more information\
from the cache or from the properties of a tile. \
Also calls :meth:`find_tiles` to get the complete\
image needed to answer the ``d_query``.
Arguments
----------
i_query: :class:`QueryLayer.InfoQuery`
A request for information
Returns
--------
str
Answer for the :class:`QueryLayer.InfoQuery`
"""
keywords = self.update_query(d_query)
# Update current query with preloaded terms
d_query.update_source(keywords)
# Get the image for the d_query
image = self.find_tiles(d_query)
return self.write_image(d_query, image)
@staticmethod
def make_data_query(i_query):
""" Make a data query from an info query
Arguments
----------
i_query: :class:`InfoQuery`
only needs ``PATH`` set in :data:`OUTPUT.INFO`
Returns
--------
:class:`DataQuery`
takes only the `PATH` from ``i_query``
"""
# Begin building needed keywords
i_path = i_query.OUTPUT.INFO.PATH
return DataQuery(**{
i_query.INPUT.METHODS.NAME: 'data',
i_path.NAME: i_path.VALUE
})
@staticmethod
def make_tile_query(d_query, t_index=np.uint32([0, 0, 0])):
""" Make a :class:`TileQuery` from :class:`DataQuery`
Arguments
----------
d_query: :class:`DataQuery`
only needs ``PATH`` set in :data:`OUTPUT.INFO`
t_index: numpy.ndarray
The 3x1 count of tiles form the origin
Returns
--------
:class:`TileQuery`
One tile request in the given data request
"""
tile_crop = d_query.all_in_some(t_index)
return TileQuery(d_query, t_index, tile_crop)
def update_query(self, query):
""" Finds missing query details from cache or tile
Makes ``keywords`` from either the :data:`_cache` or \
from a new :class:`TileQuery` to update the given ``query``
Arguments
----------
query: :class:`Query`
Either an :class:`InfoQuery` or a :class:`DataQuery`
Returns
--------
keywords: dict
Can pass to :meth:`Query.update_source` or combine \
to pass to :meth:`Query.update_dataset`.
"""
keywords = self._cache.get(query.key)
if not len(keywords):
d_query = query
# Create a preparatory data_query
if not isinstance(query, DataQuery):
d_query = self.make_data_query(query)
# Create a preparatory tile_query
t0_query = self.make_tile_query(d_query)
# Update keywords and set the cache
keywords = t0_query.preload_source
self._cache.set(query.key, keywords)
# Return the updated keywords
return keywords
#####
# Image Specific Methods
#####
def find_tiles(self, d_query):
""" Load the requested image for a :class:`DataQuery`
Arguments
----------
d_query: :class:`DataQuery`
Request for a scaled subvolume of a source image
Returns
numpy.ndarray
The full image data for the requested region
"""
first_tile_index = d_query.tile_bounds[0]
all_tiles = np.argwhere(np.ones(d_query.tile_shape))
cutout = np.zeros(d_query.target_shape, d_query.dtype)
tiles_needed = first_tile_index + all_tiles
for t_index in tiles_needed:
# Make a query for the given tile
t_query = self.make_tile_query(d_query, t_index)
tile = self.load_tile(t_query)
if not len(tile):
continue
# Fill the tile into the full cutout
to_cut = [t_query.target_origin, tile.shape]
[Z0, Y0, X0], [Z1, Y1, X1] = d_query.some_in_all(*to_cut)
cutout[Z0:Z1, Y0:Y1, X0:X1] = tile
return cutout
def find_unique(self, d_query):
""" Get unique values for a :class:`DataQuery`
Arguments
----------
d_query: :class:`DataQuery`
Request for a scaled subvolume of a source image
Returns
set
The set of unique values for the request
"""
first_tile_index = d_query.tile_bounds[0]
all_tiles = np.argwhere(np.ones(d_query.tile_shape))
tiles_needed = first_tile_index + all_tiles
# Set of all unique values
unique = set()
for t_index in tiles_needed:
# Make a query for the given tile
t_query = self.make_tile_query(d_query, t_index)
tile = self.load_tile(t_query)
# Union of unique values with the full set
tile_bins = np.bincount(tile.flatten()) > 0
unique = unique | set(np.where(tile_bins)[0])
return unique
def load_tile(self, t_query):
""" Load a single tile from the cache or from disk
Arguments
----------
t_query: :class:`TileQuery`
With tile coordinates and volume within the tile
Returns
--------
numpy.ndarray
The subregion image data for the requested tile
"""
# grab request size for query
t_bounds = t_query.target_bounds
t_origin = t_query.target_tile_bounds[0]
(K0, J0, I0), (K1, J1, I1) = t_bounds - t_origin
# Load from cache or from disk if needed
cache_tile = self._cache.get(t_query.key)
if len(cache_tile):
return cache_tile[K0:K1, J0:J1, I0:I1]
# Load from disk
tile = t_query.tile
if not len(tile):
return []
self._cache.set(t_query.key, tile)
return tile[K0:K1, J0:J1, I0:I1]
@staticmethod
def view_volume(view, vol):
""" Display a volume in color or grayscale
Arguments
----------
view: str
The requested color or gray view of the data
vol: str
Raw volume from :class:`Cache` / :class:`Datasource`
Returns
--------
numpy.ndarray
Colorized or original raw volume
"""
# Set up a colormap
def id_to_color(vol):
colors = np.zeros((3, ) + vol.shape).astype(np.uint8)
colors[0] = np.mod(107 * vol, 700).astype(np.uint8)
colors[1] = np.mod(509 * vol, 900).astype(np.uint8)
colors[2] = np.mod(200 * vol, 777).astype(np.uint8)
return np.moveaxis(colors, 0, -1)
# Colormap if a colormap view
if view.VALUE == view.COLOR.NAME:
return id_to_color(vol)
return vol
def write_image(self, d_query, volume):
""" Format a volume for a given :class:`DataQuery`
Arguments
----------
d_query: :class:`DataQuery`
With the format and view for the requested volume
volume: numpy.ndarray
Raw volume from :class:`Cache` / :class:`Datasource`
Returns
--------
str:
The image response as a formatted bytestring
"""
img_format = d_query.INPUT.IMAGE.FORMAT
img_view = d_query.INPUT.IMAGE.VIEW
img_type = d_query.OUTPUT.INFO.TYPE
# Only if grayscale view is set
if img_view.VALUE == img_view.GRAY.NAME:
# set the view based on the format
is_big_int = img_type.VALUE in img_type.ID_LIST
no_big_int_gray = img_format.VALUE in img_format.COLOR_LIST
# If big integers must not be grayscale, try colormap
if is_big_int and no_big_int_gray:
img_view.VALUE = img_view.COLOR.NAME
# If Multiple slices cannot be formatted
if img_format.VALUE not in img_format.VOL_LIST:
shape = volume.shape
if shape[0] > 1:
# Flatten the volume to image
volume = volume.reshape(1, -1, shape[-1])
# Use colormap / RGB style encoding of ID data
vol = self.view_volume(img_view, volume)
if img_format.VALUE in ['raw']:
output = StringIO.StringIO()
output.write(vol.tobytes())
vol_string = output.getvalue()
return vol_string
if img_format.VALUE in ['npz']:
output = StringIO.StringIO()
np.save(output, vol[np.newaxis])
vol_string = output.getvalue()
return zlib.compress(vol_string)
if img_format.VALUE in img_format.ZIP_LIST:
output = StringIO.StringIO()
volstring = vol[0].T.tostring('F')
output.write(zlib.compress(volstring))
return output.getvalue()
if img_format.VALUE in img_format.TIF_LIST:
output = StringIO.StringIO()
tiffvol = vol[0]
tifffile.imsave(output, tiffvol)
return output.getvalue()
filetype = "." + img_format.VALUE
image = cv2.imencode(filetype, vol[0])
return image[1].tostring()
def run(movie_dir, html_output_flag, limit):
"""This is the real entry point for the program"""
#A class to help lookup movie titles
movielookup = MovieLookup()
#Match files in a given directory
matcher = Matcher(Config.movie_match_regex, Config.allowed_file_types)
#Used to find an imdb id from movie filename
id_finder = IdFinder()
#Used for caching movie data
movie_cache = Cache(Config.movie_cache_file)
#First, let's match files which match the regex and have the
#required file extensions in the given directory
matcher.find_in_directory(movie_dir)
movie_matches = matcher.get_matches()
unmatched = matcher.get_ignored()
#normalise the matches (the filenames will be used as movie titles)
normalised_movie_matches = Normaliser\
.normalise_list_and_remove_trailing_number(movie_matches)
#Now we lookup successful matches, first in the cache, then online
movie_data = {} #successful lookup data will go here
failed_lookups = [] #we will do something with failed lookups later...
count = 0 #used to limit the number of lookups we will do
for title in normalised_movie_matches:
count += 1
if count >= limit: #check that we don't go over the arbitrary limit
break
#Check if the movie is in our cache
cached_movie = movie_cache.get(title)
if cached_movie:
movie_data[title] = cached_movie
#Otherwise, lookup using API
else:
#look up each movie in the list
lookup_data = movielookup.lookup_by_title(title)
#check if we found a movie
if MovieDataUtil.is_valid_lookup_result(lookup_data):
movie_data[title] = lookup_data
#great, let's also add it to the cache
movie_cache.add_to_cache(title, lookup_data)
else:
failed_lookups.append(title)
#now we will try to correct the failed lookups
#by using google to find each imdb id
id_lookup_dict = id_finder.find_id_by_title_list(failed_lookups)
#reset the failed lookups
failed_lookups = [] #there should be a lot less now...
title_corrections = 0 #count how many corrections we actually found
#Now lookup using the new ids which we found
for title, found_id in id_lookup_dict.items():
if found_id != None:
#we found an id, now let's look the movie up by its id
lookup_data = movielookup.lookup_by_id(found_id)
#theoretically this should always be true
#unless we got an invalid id somehow...
if MovieDataUtil.is_valid_lookup_result(lookup_data):
movie_data[title] = lookup_data
title_corrections += 1
#great, let's also add it to the cache
movie_cache.add_to_cache(title, lookup_data)
else:
failed_lookups.append(title)
else:
failed_lookups.append(title)
#Save the updated cache
movie_cache.save_cache_to_disk()
#sort the data by imdb id
movie_data = MovieDataUtil.sort_movie_data(movie_data)
#Output the data
if html_output_flag:
logging.debug('Loading template from: %s', Config.template_directory)
template_environment = Environment( \
loader=FileSystemLoader( \
Config.template_directory), trim_blocks=True)
print template_environment.get_template('main.html').render(
movie_lookup_data=movie_data,
failed_lookups=failed_lookups,
unmatched=unmatched,
title_corrections=title_corrections,
datetime=time.strftime("%c"),
version=__version__,
author=__author__,
cache_stats=movie_cache.cache_stats(),
)
else:
simple_output(movie_data, failed_lookups, unmatched)
class CandleProcessor(object):
stableCoinTickers = [
"USD", "USDT", "USDC", "DAI", "HUSD", "TUSD", "PAX", "USDK", "USDN",
"BUSD", "GUSD", "USDS"
]
def __init__(self):
self.isServiceAvailable = True
signal.signal(signal.SIGINT, self.exit_gracefully)
signal.signal(signal.SIGTERM, self.exit_gracefully)
self.logging = error_reporting.Client()
self.cache = Cache(ttl=30)
context = zmq.Context.instance()
self.socket = context.socket(zmq.ROUTER)
self.socket.bind("tcp://*:6900")
print("[Startup]: Candle Server is online")
def exit_gracefully(self):
print("[Startup]: Candle Server is exiting")
self.socket.close()
self.isServiceAvailable = False
def run(self):
while self.isServiceAvailable:
try:
response = None, None
origin, delimeter, clientId, service, request = self.socket.recv_multipart(
)
request = pickle.loads(zlib.decompress(request))
if request.timestamp + 30 < time.time(): continue
if service == b"candle":
response = self.request_candle(request)
except (KeyboardInterrupt, SystemExit):
return
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]:
self.logging.report_exception()
finally:
try:
self.socket.send_multipart([
origin, delimeter,
zlib.compress(pickle.dumps(response, -1))
])
except:
pass
def request_candle(self, request):
payload, candleMessage, updatedCandleMessage = None, None, None
for platform in request.platforms:
request.set_current(platform=platform)
hashCode = hash(request.requests[platform])
fromCache = False
if request.can_cache() and self.cache.has(hashCode):
payload, updatedCandleMessage = self.cache.get(hashCode), None
fromCache = True
elif platform == "CCXT":
payload, updatedCandleMessage = self.request_ccxt_candles(
request)
elif platform == "IEXC":
payload, updatedCandleMessage = self.request_iexc_candles(
request)
elif platform == "Quandl":
pass
if payload is not None:
if request.can_cache() and not fromCache:
self.cache.set(hashCode, payload)
return payload, updatedCandleMessage
elif updatedCandleMessage is not None:
candleMessage = updatedCandleMessage
return None, candleMessage
def request_ccxt_candles(self, request):
ticker = request.get_ticker()
exchange = request.get_exchange()
try:
if exchange is None: return None, None
exchange = Exchange(exchange.id)
try:
rawData = exchange.properties.fetch_ohlcv(ticker.symbol,
timeframe="1m",
limit=60)
if len(rawData) == 0 or rawData[-1][4] is None or rawData[0][
1] is None:
return None, None
except:
return None, None
payload = {
"candles": [],
"title":
ticker.name,
"baseTicker":
"USD" if ticker.base in CandleProcessor.stableCoinTickers else
ticker.base,
"quoteTicker":
"USD" if ticker.quote in CandleProcessor.stableCoinTickers else
ticker.quote,
"sourceText":
"on {}".format(exchange.name),
"platform":
"CCXT"
}
for e in rawData:
timestamp = e[0] / 1000
if ticker.isReversed:
payload["candles"].append(
[timestamp, 1 / e[1], 1 / e[2], 1 / e[3], 1 / e[4]])
else:
payload["candles"].append(
[timestamp, e[1], e[2], e[3], e[4]])
return payload, None
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]: self.logging.report_exception()
return None, None
def request_iexc_candles(self, request):
ticker = request.get_ticker()
exchange = request.get_exchange()
try:
try:
stock = Stock(ticker.id, token=os.environ["IEXC_KEY"])
rawData = stock.get_quote().loc[ticker.id]
if rawData is None: return None, None
except:
return None, None
latestPrice = rawData["delayedPrice"] if rawData[
"latestPrice"] is None else rawData["latestPrice"]
lastPrice = float(
latestPrice if "isUSMarketOpen" not in rawData or
rawData["isUSMarketOpen"] or "extendedPrice" not in rawData or
rawData["extendedPrice"] is None else rawData["extendedPrice"])
openPrice, highPrice, lowPrice = None, None, None
if "open" in rawData: openPrice = rawData["open"]
if "high" in rawData: highPrice = rawData["high"]
if "low" in rawData: lowPrice = rawData["low"]
if openPrice is None: openPrice = lastPrice
if highPrice is None: highPrice = lastPrice
if lowPrice is None: lowPrice = lastPrice
payload = {
"candles": [[
float(rawData["latestUpdate"]) / 1000,
float(openPrice),
float(highPrice),
float(lowPrice), lastPrice
]],
"title":
ticker.name,
"baseTicker":
"shares",
"quoteTicker":
ticker.quote,
"sourceText":
"provided by IEX Cloud ● {} on {}".format(
rawData["latestSource"], rawData["primaryExchange"]),
"platform":
"IEXC",
}
return payload, None
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]: self.logging.report_exception()
return None, None
class ChainDb(object):
def __init__(self, settings, datadir, log, mempool, netmagic,
readonly=False, fast_dbm=False,compression=False):
self.settings = settings
self.log = log
self.mempool = mempool
self.readonly = readonly
self.netmagic = netmagic
self.fast_dbm = fast_dbm
self.blk_cache = Cache(1000)
self.orphans = {}
self.orphan_deps = {}
self.compress_on_write = compression
# LevelDB to hold:
# tx:* transaction outputs
# misc:* state
# height:* list of blocks at height h
# blkmeta:* block metadata
# blocks:* block seek point in stream
self.blk_write = io.BufferedWriter(io.FileIO(datadir + '/blocks.dat','ab'))
self.blk_read = io.BufferedReader(io.FileIO(datadir + '/blocks.dat','rb'))
self.db = leveldb.LevelDB(datadir + '/leveldb')
try:
self.db.Get('misc:height')
except KeyError:
self.log.write("INITIALIZING EMPTY BLOCKCHAIN DATABASE")
batch = leveldb.WriteBatch()
batch.Put('misc:height', str(-1))
batch.Put('misc:msg_start', self.netmagic.msg_start)
batch.Put('misc:tophash', ser_uint256(0L))
batch.Put('misc:total_work', hex(0L))
self.db.Write(batch)
try:
start = self.db.Get('misc:msg_start')
if start != self.netmagic.msg_start: raise KeyError
except KeyError:
self.log.write("Database magic number mismatch. Data corruption or incorrect network?")
raise RuntimeError
def puttxidx(self, txhash, txidx, batch=None):
ser_txhash = ser_uint256(txhash)
try:
self.db.Get('tx:'+ser_txhash)
old_txidx = self.gettxidx(txhash)
self.log.write("WARNING: overwriting duplicate TX %064x, height %d, oldblk %064x, oldspent %x, newblk %064x" % (txhash, self.getheight(), old_txidx.blkhash, old_txidx.spentmask, txidx.blkhash))
except KeyError:
pass
batch = self.db if batch is not None else batch
batch.Put('tx:'+ser_txhash, hex(txidx.blkhash) + ' ' +
hex(txidx.spentmask))
return True
def gettxidx(self, txhash):
ser_txhash = ser_uint256(txhash)
try:
ser_value = self.db.Get('tx:'+ser_txhash)
except KeyError:
return None
pos = string.find(ser_value, ' ')
txidx = TxIdx()
txidx.blkhash = long(ser_value[:pos], 16)
txidx.spentmask = long(ser_value[pos+1:], 16)
return txidx
def gettx(self, txhash):
txidx = self.gettxidx(txhash)
if txidx is None:
return None
block = self.getblock(txidx.blkhash)
if block:
for tx in block.vtx:
tx.calc_sha256()
if tx.sha256 == txhash:
return tx
self.log.write("ERROR: Missing TX %064x in block %064x" % (txhash, txidx.blkhash))
return None
def haveblock(self, blkhash, checkorphans):
if self.blk_cache.exists(blkhash):
return True
if checkorphans and blkhash in self.orphans:
return True
ser_hash = ser_uint256(blkhash)
try:
self.db.Get('blocks:'+ser_hash)
return True
except KeyError:
return False
def have_prevblock(self, block):
if self.getheight() < 0 and block.sha256 == self.netmagic.block0:
return True
if self.haveblock(block.hashPrevBlock, False):
return True
return False
def getblock(self, blkhash):
block = self.blk_cache.get(blkhash)
if block is not None:
return block
ser_hash = ser_uint256(blkhash)
try:
# Lookup the block index, seek in the file
fpos = long(self.db.Get('blocks:'+ser_hash))
self.blk_read.seek(fpos)
# read and decode "block" msg
recvbuf = self.blk_read.read(4+4)
if recvbuf[:4] == 'ZLIB':
msg_len = int(recvbuf[4:8].encode('hex'),16)
recvbuf = self.blk_read.read(msg_len)
f = cStringIO.StringIO(zlib.decompress(recvbuf))
msg = message_read(self.netmagic, f)
else:
self.blk_read.seek(fpos)
msg = message_read(self.netmagic, self.blk_read)
if msg is None:
return None
block = msg.block
except KeyError:
return None
self.blk_cache.put(blkhash, block)
return block
def spend_txout(self, txhash, n_idx, batch=None):
txidx = self.gettxidx(txhash)
if txidx is None:
return False
txidx.spentmask |= (1L << n_idx)
self.puttxidx(txhash, txidx, batch)
return True
def clear_txout(self, txhash, n_idx, batch=None):
txidx = self.gettxidx(txhash)
if txidx is None:
return False
txidx.spentmask &= ~(1L << n_idx)
self.puttxidx(txhash, txidx, batch)
return True
def unique_outpts(self, block):
outpts = {}
txmap = {}
for tx in block.vtx:
if tx.is_coinbase:
continue
txmap[tx.sha256] = tx
for txin in tx.vin:
v = (txin.prevout.hash, txin.prevout.n)
if v in outs:
return None
outpts[v] = False
return (outpts, txmap)
def txout_spent(self, txout):
txidx = self.gettxidx(txout.hash)
if txidx is None:
return None
if txout.n > 100000: # outpoint index sanity check
return None
if txidx.spentmask & (1L << txout.n):
return True
return False
class ChainDb(object):
def __init__(self,
settings,
datadir,
log,
mempool,
netmagic,
readonly=False,
fast_dbm=False):
self.settings = settings
self.log = log
self.mempool = mempool
self.readonly = readonly
self.netmagic = netmagic
self.fast_dbm = fast_dbm
self.blk_cache = Cache(750)
self.orphans = {}
self.orphan_deps = {}
# LevelDB to hold:
# tx:* transaction outputs
# misc:* state
# height:* list of blocks at height h
# blkmeta:* block metadata
# blocks:* block seek point in stream
self.blk_write = io.BufferedWriter(
io.FileIO(datadir + '/blocks.dat', 'ab'))
self.blk_read = io.BufferedReader(
io.FileIO(datadir + '/blocks.dat', 'rb'))
self.db = leveldb.LevelDB(datadir + '/leveldb')
try:
self.db.Get('misc:height')
except KeyError:
self.log.write("INITIALIZING EMPTY BLOCKCHAIN DATABASE")
batch = leveldb.WriteBatch()
batch.Put('misc:height', str(-1))
batch.Put('misc:msg_start', self.netmagic.msg_start)
batch.Put('misc:tophash', ser_uint256(0L))
batch.Put('misc:total_work', hex(0L))
self.db.Write(batch)
try:
start = self.db.Get('misc:msg_start')
if start != self.netmagic.msg_start: raise KeyError
except KeyError:
self.log.write(
"Database magic number mismatch. Data corruption or incorrect network?"
)
raise RuntimeError
def puttxidx(self, txhash, txidx, batch=None):
ser_txhash = ser_uint256(txhash)
try:
self.db.Get('tx:' + ser_txhash)
old_txidx = self.gettxidx(txhash)
self.log.write(
"WARNING: overwriting duplicate TX %064x, height %d, oldblk %064x, oldspent %x, newblk %064x"
% (txhash, self.getheight(), old_txidx.blkhash,
old_txidx.spentmask, txidx.blkhash))
except KeyError:
pass
batch = self.db if batch is not None else batch
batch.Put('tx:' + ser_txhash,
hex(txidx.blkhash) + ' ' + hex(txidx.spentmask))
return True
def gettxidx(self, txhash):
ser_txhash = ser_uint256(txhash)
try:
ser_value = self.db.Get('tx:' + ser_txhash)
except KeyError:
return None
pos = string.find(ser_value, ' ')
txidx = TxIdx()
txidx.blkhash = long(ser_value[:pos], 16)
txidx.spentmask = long(ser_value[pos + 1:], 16)
return txidx
def gettx(self, txhash):
txidx = self.gettxidx(txhash)
if txidx is None:
return None
block = self.getblock(txidx.blkhash)
for tx in block.vtx:
tx.calc_sha256()
if tx.sha256 == txhash:
return tx
self.log.write("ERROR: Missing TX %064x in block %064x" %
(txhash, txidx.blkhash))
return None
def haveblock(self, blkhash, checkorphans):
if self.blk_cache.exists(blkhash):
return True
if checkorphans and blkhash in self.orphans:
return True
ser_hash = ser_uint256(blkhash)
try:
self.db.Get('blocks:' + ser_hash)
return True
except KeyError:
return False
def have_prevblock(self, block):
if self.getheight() < 0 and block.sha256 == self.netmagic.block0:
return True
if self.haveblock(block.hashPrevBlock, False):
return True
return False
def getblock(self, blkhash):
block = self.blk_cache.get(blkhash)
if block is not None:
return block
ser_hash = ser_uint256(blkhash)
try:
# Lookup the block index, seek in the file
fpos = long(self.db.Get('blocks:' + ser_hash))
self.blk_read.seek(fpos)
# read and decode "block" msg
msg = message_read(self.netmagic, self.blk_read)
if msg is None:
return None
block = msg.block
except KeyError:
return None
self.blk_cache.put(blkhash, block)
return block
def spend_txout(self, txhash, n_idx, batch=None):
txidx = self.gettxidx(txhash)
if txidx is None:
return False
txidx.spentmask |= (1L << n_idx)
self.puttxidx(txhash, txidx, batch)
return True
def clear_txout(self, txhash, n_idx, batch=None):
txidx = self.gettxidx(txhash)
if txidx is None:
return False
txidx.spentmask &= ~(1L << n_idx)
self.puttxidx(txhash, txidx, batch)
return True
def unique_outpts(self, block):
outpts = {}
txmap = {}
for tx in block.vtx:
if tx.is_coinbase:
continue
txmap[tx.sha256] = tx
for txin in tx.vin:
v = (txin.prevout.hash, txin.prevout.n)
if v in outs:
return None
outpts[v] = False
return (outpts, txmap)
def spent_outpts(self, block):
# list of outpoints this block wants to spend
l = self.unique_outpts(block)
if l is None:
return None
outpts = l[0]
txmap = l[1]
spendlist = {}
# pass 1: if outpoint in db, make sure it is unspent
for k in outpts.iterkeys():
txidx = self.gettxidx(k[0])
if txidx is None:
continue
if k[1] > 100000: # outpoint index sanity check
return None
if txidx.spentmask & (1L << k[1]):
return None
outpts[k] = True # skip in pass 2
# pass 2: remaining outpoints must exist in this block
for k, v in outpts.iteritems():
if v:
continue
if k[0] not in txmap: # validate txout hash
return None
tx = txmap[k[0]] # validate txout index (n)
if k[1] >= len(tx.vout):
return None
# outpts[k] = True # not strictly necessary
return outpts.keys()
try:
message = udp_connection_client.blocked_recv() # Get message from the client
except ConnectionResetError:
continue
dns_message.parse_dns(message) # Parse the message received
search_key = (dns_message.QueriesList[0].QNAME, dns_message.QueriesList[0].QTYPE) # the search key is for query the cache: qname and qtype
transaction_id = dns_message.Header.TransactionID # The transaction ID of the query from client
if Config.VERBOSE:
print("Transaction ID:{}. Search Key: {}".format(transaction_id, search_key))
"""
Find Answer in cache
"""
if Config.VERBOSE:
print("Cache: {}".format(cache.itemdict.keys()))
cache_msg = cache.get(search_key) # Find the answer in the cache
if cache_msg != Cache.ANSWER_EXPIRE and cache_msg != Cache.ANSWER_NOT_FOUND: # The answer is in the cache and it is not out of date
if Config.CACHE_VERBOSE:
print("Cache hit.")
udp_connection_client.sendto(dns_message.changeTransactionID(cache_msg, transaction_id)) # Send the answer to the client directly
continue
else: # The answer is not found or out of date
if Config.CACHE_VERBOSE:
print("Cache miss :", cache_msg)
"""
Forward query to upper level DNS sever
"""
udp_connection_DNS = UDPConnection() # Connect to the upper level server
udp_connection_DNS.sendto_server(message, Config.DNS_server_IPaddr, Config.DNS_server_Port) # Forward the query to the server
try:
message = udp_connection_DNS.blocked_recv() # Receive the response from the server
class QuoteProcessor(object):
imageOverlays = {
"Alpha depth":
Image.open("app/assets/overlays/quotes/depth.png").convert("RGBA")
}
stableCoinTickers = [
"USD", "USDT", "USDC", "DAI", "HUSD", "TUSD", "PAX", "USDK", "USDN",
"BUSD", "GUSD", "USDS"
]
lastBitcoinQuote = {}
def __init__(self):
self.isServiceAvailable = True
signal.signal(signal.SIGINT, self.exit_gracefully)
signal.signal(signal.SIGTERM, self.exit_gracefully)
self.logging = error_reporting.Client()
self.cache1 = Cache(ttl=5)
self.cache2 = Cache(ttl=5)
self.coinGecko = CoinGeckoAPI()
self.lastBitcoinQuote = {
"quotePrice": [0],
"quoteVolume":
None,
"ticker":
Ticker("BTCUSD", "BTCUSD", "BTC", "USD", "BTC/USD",
hasParts=False),
"exchange":
None,
"timestamp":
time.time()
}
try:
rawData = self.coinGecko.get_coin_by_id(id="bitcoin",
localization="false",
tickers=False,
market_data=True,
community_data=False,
developer_data=False)
self.lastBitcoinQuote["quotePrice"] = [
rawData["market_data"]["current_price"]["usd"]
]
self.lastBitcoinQuote["quoteVolume"] = rawData["market_data"][
"total_volume"]["usd"]
except:
pass
context = zmq.Context.instance()
self.socket = context.socket(zmq.ROUTER)
self.socket.bind("tcp://*:6900")
print("[Startup]: Quote Server is online")
def exit_gracefully(self):
print("[Startup]: Quote Server is exiting")
self.socket.close()
self.isServiceAvailable = False
def run(self):
while self.isServiceAvailable:
try:
response = None, None
origin, delimeter, clientId, service, request = self.socket.recv_multipart(
)
request = pickle.loads(zlib.decompress(request))
if request.timestamp + 30 < time.time(): continue
if service == b"quote":
response = self.request_quote(request)
elif service == b"depth":
response = self.request_depth(request)
except (KeyboardInterrupt, SystemExit):
return
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]:
self.logging.report_exception()
finally:
try:
self.socket.send_multipart([
origin, delimeter,
zlib.compress(pickle.dumps(response, -1))
])
except:
pass
def request_depth(self, request):
payload, quoteMessage, updatedQuoteMessage = None, None, None
for platform in request.platforms:
request.set_current(platform=platform)
hashCode = hash(request.requests[platform])
fromCache = False
if request.can_cache() and self.cache1.has(hashCode):
payload, updatedQuoteMessage = self.cache1.get(hashCode), None
fromCache = True
elif platform == "CCXT":
payload, updatedQuoteMessage = self.request_ccxt_depth(request)
elif platform == "IEXC":
payload, updatedQuoteMessage = self.request_iexc_depth(request)
if payload is not None:
if request.can_cache() and not fromCache:
self.cache1.set(hashCode, payload)
if request.authorId != 401328409499664394 and request.requests[
platform].ticker.base is not None and request.authorId not in constants.satellites:
database.document("dataserver/statistics/{}/{}".format(
platform, str(uuid.uuid4()))).set({
"timestamp":
time.time(),
"authorId":
str(request.authorId),
"ticker": {
"base": request.requests[platform].ticker.base,
"quote":
request.requests[platform].ticker.quote,
"id": request.requests[platform].ticker.id,
"bias": request.parserBias
},
"exchange":
None if request.requests[platform].exchange is None
else request.requests[platform].exchange.id
})
return payload, updatedQuoteMessage
elif updatedQuoteMessage is not None:
quoteMessage = updatedQuoteMessage
return None, quoteMessage
def request_quote(self, request):
payload, quoteMessage, updatedQuoteMessage = None, None, None
for platform in request.platforms:
request.set_current(platform=platform)
hashCode = hash(request.requests[platform])
fromCache = False
if request.can_cache() and self.cache2.has(hashCode):
payload, updatedQuoteMessage = self.cache2.get(hashCode), None
fromCache = True
elif platform == "Alternative.me":
payload, updatedQuoteMessage = self.request_fear_greed_index(
request)
elif platform == "LLD":
payload, updatedQuoteMessage = self.request_lld_quote(request)
elif platform == "CoinGecko":
payload, updatedQuoteMessage = self.request_coingecko_quote(
request)
elif platform == "CCXT":
payload, updatedQuoteMessage = self.request_ccxt_quote(request)
elif platform == "IEXC":
payload, updatedQuoteMessage = self.request_iexc_quote(request)
elif platform == "Quandl":
pass
if payload is not None:
if request.can_cache() and not fromCache:
self.cache2.set(hashCode, payload)
if request.authorId != 401328409499664394 and request.requests[
platform].ticker.base is not None and request.authorId not in constants.satellites:
database.document("dataserver/statistics/{}/{}".format(
platform, str(uuid.uuid4()))).set({
"timestamp":
time.time(),
"authorId":
str(request.authorId),
"ticker": {
"base": request.requests[platform].ticker.base,
"quote":
request.requests[platform].ticker.quote,
"id": request.requests[platform].ticker.id,
"bias": request.parserBias
},
"exchange":
None if request.requests[platform].exchange is None
else request.requests[platform].exchange.id
})
return payload, updatedQuoteMessage
elif updatedQuoteMessage is not None:
quoteMessage = updatedQuoteMessage
return None, quoteMessage
def request_coingecko_quote(self, request):
ticker = request.get_ticker()
exchange = request.get_exchange()
try:
try:
rawData = self.coinGecko.get_coin_by_id(id=ticker.symbol,
localization="false",
tickers=False,
market_data=True,
community_data=False,
developer_data=False)
except:
return None, None
if ticker.quote.lower() not in rawData["market_data"][
"current_price"] or ticker.quote.lower(
) not in rawData["market_data"]["total_volume"]:
return None, "Requested price for `{}` is not available.".format(
ticker.name)
price = rawData["market_data"]["current_price"][
ticker.quote.lower()]
if ticker.isReversed: price = 1 / price
volume = rawData["market_data"]["total_volume"][
ticker.quote.lower()]
priceChange = rawData["market_data"][
"price_change_percentage_24h_in_currency"][ticker.quote.lower(
)] if ticker.quote.lower() in rawData["market_data"][
"price_change_percentage_24h_in_currency"] else 0
if ticker.isReversed:
priceChange = (1 / (priceChange / 100 + 1) - 1) * 100
payload = {
"quotePrice":
("{:,.%df}" %
(4 if TickerParser.check_if_fiat(ticker.quote)[0]
and not ticker.isReversed else 8)).format(price),
"quoteVolume":
volume,
"quoteConvertedPrice":
None if ticker.quote == "USD" else "≈ ${:,.6f}".format(
rawData["market_data"]["current_price"]["usd"]),
"quoteConvertedVolume":
None if ticker.quote == "USD" else "≈ ${:,.4f}".format(
rawData["market_data"]["total_volume"]["usd"]),
"title":
ticker.name,
"baseTicker":
ticker.base if ticker.base.lower()
in rawData["market_data"]["current_price"] else "BTC",
"quoteTicker":
ticker.quote if ticker.quote.lower()
in rawData["market_data"]["current_price"] else "BTC",
"change":
priceChange,
"thumbnailUrl":
TickerParser.get_coingecko_image(ticker.base),
"messageColor":
"amber" if priceChange == 0 else
("green" if priceChange > 0 else "red"),
"sourceText":
"from CoinGecko",
"platform":
"CoinGecko",
"raw": {
"quotePrice": [price],
"quoteVolume": volume,
"ticker": ticker,
"exchange": exchange,
"timestamp": time.time()
}
}
if ticker == Ticker("BTCUSD",
"BTCUSD",
"BTC",
"USD",
"BTC/USD",
hasParts=False):
self.lastBitcoinQuote = payload["raw"]
return payload, None
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]: self.logging.report_exception()
return None, None
def request_ccxt_quote(self, request):
ticker = request.get_ticker()
exchange = request.get_exchange()
try:
if exchange is None: return None, None
exchange = Exchange(exchange.id)
tf, limitTimestamp, candleOffset = Utils.get_highest_supported_timeframe(
exchange.properties,
datetime.datetime.now().astimezone(pytz.utc))
try:
rawData = exchange.properties.fetch_ohlcv(ticker.symbol,
timeframe=tf.lower(),
since=limitTimestamp,
limit=300)
if len(rawData) == 0 or rawData[-1][4] is None or rawData[0][
1] is None:
return None, None
except:
return None, None
price = [rawData[-1][4], rawData[0][1]
] if len(rawData) < candleOffset else [
rawData[-1][4], rawData[-candleOffset][1]
]
if ticker.isReversed: price = [1 / price[0], 1 / price[1]]
volume = None if price[0] is None else sum([
candle[5] for candle in rawData if int(candle[0] / 1000) >=
int(exchange.properties.milliseconds() / 1000) - 86400
]) / (
price[0] if exchange.id in ["bitmex", "binancefutures"] else 1)
priceChange = 0 if tf == "1m" or price[1] == 0 else (
price[0] / price[1]) * 100 - 100
payload = {
"quotePrice":
"{:,.8f}".format(price[0])
if ticker.isReversed else Utils.format_price(
exchange.properties, ticker.symbol, price[0]),
"quoteVolume":
volume,
"quoteConvertedPrice":
"≈ ${:,.6f}".format(price[0] *
self.lastBitcoinQuote["quotePrice"][0])
if ticker.quote == "BTC" else None,
"quoteConvertedVolume":
"≈ ${:,.4f}".format(volume *
self.lastBitcoinQuote["quotePrice"][0])
if ticker.quote == "BTC" else None,
"title":
ticker.name,
"baseTicker":
"USD" if ticker.base in QuoteProcessor.stableCoinTickers else
ticker.base,
"quoteTicker":
"USD" if ticker.quote in QuoteProcessor.stableCoinTickers else
ticker.quote,
"change":
priceChange,
"thumbnailUrl":
TickerParser.get_coingecko_image(ticker.base),
"messageColor":
"amber" if priceChange == 0 else
("green" if priceChange > 0 else "red"),
"sourceText":
"on {}".format(exchange.name),
"platform":
"CCXT",
"raw": {
"quotePrice": [price[0]] if tf == "1m" else price[:1],
"quoteVolume": volume,
"ticker": ticker,
"exchange": exchange,
"timestamp": time.time()
}
}
return payload, None
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]: self.logging.report_exception()
return None, None
def request_lld_quote(self, request):
ticker = request.get_ticker()
exchange = request.get_exchange()
filters = request.get_filters()
action = request.find_parameter_in_list("lld", filters)
try:
if exchange is not None: exchange = Exchange(exchange.id)
if action == "funding":
if exchange.id in ["bitmex"]:
try:
rawData = exchange.properties.public_get_instrument(
{"symbol": ticker.id})[0]
except:
return None, "Requested funding data for `{}` is not available.".format(
ticker.name)
if rawData["fundingTimestamp"] is None:
fundingDate = datetime.datetime.strptime(
rawData["fundingTimestamp"],
"%Y-%m-%dT%H:%M:00.000Z").replace(tzinfo=pytz.utc)
else:
fundingDate = datetime.datetime.now().replace(
tzinfo=pytz.utc)
indicativeFundingTimestamp = datetime.datetime.timestamp(
fundingDate) + 28800
indicativeFundingDate = datetime.datetime.utcfromtimestamp(
indicativeFundingTimestamp).replace(tzinfo=pytz.utc)
deltaFunding = fundingDate - datetime.datetime.now(
).astimezone(pytz.utc)
deltaIndicative = indicativeFundingDate - datetime.datetime.now(
).astimezone(pytz.utc)
hours1, seconds1 = divmod(
deltaFunding.days * 86400 + deltaFunding.seconds, 3600)
minutes1 = int(seconds1 / 60)
hoursFunding = "{:d} {} ".format(
hours1, "hours"
if hours1 > 1 else "hour") if hours1 > 0 else ""
minutesFunding = "{:d} {}".format(
minutes1 if hours1 > 0 or minutes1 > 0 else seconds1,
"{}".format("minute" if minutes1 == 1 else "minutes")
if hours1 > 0 or minutes1 > 0 else
("second" if seconds1 == 1 else "seconds"))
deltaFundingText = "{}{}".format(hoursFunding,
minutesFunding)
hours2, seconds2 = divmod(
deltaIndicative.days * 86400 + deltaIndicative.seconds,
3600)
minutes2 = int(seconds2 / 60)
hoursIndicative = "{:d} {} ".format(
hours2, "hours"
if hours2 > 1 else "hour") if hours2 > 0 else ""
minutesIndicative = "{:d} {}".format(
minutes2 if hours2 > 0 or minutes2 > 0 else seconds2,
"{}".format("minute" if minutes2 == 1 else "minutes")
if hours2 > 0 or minutes2 > 0 else
("second" if seconds2 == 1 else "seconds"))
deltaIndicativeText = "{}{}".format(
hoursIndicative, minutesIndicative)
fundingRate = rawData["fundingRate"] * 100
predictedFundingRate = rawData[
"indicativeFundingRate"] * 100
averageFundingRate = (fundingRate +
predictedFundingRate) / 2
payload = {
"quotePrice":
"Funding Rate: {:+.4f} % *(in {})*\nPredicted Rate: {:+.4f} % *(in {})*"
.format(fundingRate, deltaFundingText,
predictedFundingRate, deltaIndicativeText),
"quoteVolume":
None,
"quoteConvertedPrice":
None,
"quoteConvertedVolume":
None,
"title":
ticker.name,
"baseTicker":
ticker.base,
"quoteTicker":
ticker.quote,
"change":
None,
"thumbnailUrl":
TickerParser.get_coingecko_image(ticker.base),
"messageColor":
"yellow" if averageFundingRate == 0.01 else
("light green"
if averageFundingRate < 0.01 else "deep orange"),
"sourceText":
"Contract details on {}".format(exchange.name),
"platform":
"LLD",
"raw": {
"quotePrice": [fundingRate, predictedFundingRate],
"quoteVolume": None,
"ticker": ticker,
"exchange": exchange,
"timestamp": time.time()
}
}
return payload, None
return None, "Funding data is only available on BitMEX."
elif action == "oi":
if exchange.id in ["bitmex"]:
try:
rawData = exchange.properties.public_get_instrument(
{"symbol": ticker.id})[0]
except:
return None, "Requested open interest data for `{}` is not available.".format(
ticker.name)
payload = {
"quotePrice":
"Open interest: {:,.0f} {}\nOpen value: {:,.4f} XBT".
format(rawData["openInterest"],
"USD" if ticker.id == "XBTUSD" else "contracts",
rawData["openValue"] / 100000000),
"quoteVolume":
None,
"quoteConvertedPrice":
None,
"quoteConvertedVolume":
None,
"title":
ticker.name,
"baseTicker":
ticker.base,
"quoteTicker":
ticker.quote,
"change":
None,
"thumbnailUrl":
TickerParser.get_coingecko_image(ticker.base),
"messageColor":
"deep purple",
"sourceText":
"Contract details on {}".format(exchange.name),
"platform":
"LLD",
"raw": {
"quotePrice": [
rawData["openInterest"],
rawData["openValue"] / 100000000
],
"quoteVolume":
None,
"ticker":
ticker,
"exchange":
exchange,
"timestamp":
time.time()
}
}
return payload, None
return None, "Open interest and open value data is only available on BitMEX."
elif action == "ls":
if exchange.id in ["bitfinex2"]:
try:
longs = exchange.properties.publicGetStats1KeySizeSymbolLongLast(
{
"key": "pos.size",
"size": "1m",
"symbol": "t{}".format(ticker.id),
"side": "long",
"section": "last"
})
shorts = exchange.properties.publicGetStats1KeySizeSymbolShortLast(
{
"key": "pos.size",
"size": "1m",
"symbol": "t{}".format(ticker.id),
"side": "long",
"section": "last"
})
ratio = longs[1] / (longs[1] + shorts[1]) * 100
except:
return None, None
payload = {
"quotePrice":
"{:.1f} % longs / {:.1f} % shorts".format(
ratio, 100 - ratio),
"quoteVolume":
None,
"quoteConvertedPrice":
None,
"quoteConvertedVolume":
None,
"title":
"{} longs/shorts ratio".format(ticker.name),
"baseTicker":
ticker.base,
"quoteTicker":
ticker.quote,
"change":
None,
"thumbnailUrl":
TickerParser.get_coingecko_image(ticker.base),
"messageColor":
"deep purple",
"sourceText":
"Data on {}".format(exchange.name),
"platform":
"LLD",
"raw": {
"quotePrice": [longs[1], shorts[1]],
"quoteVolume": None,
"ticker": ticker,
"exchange": exchange,
"timestamp": time.time()
}
}
return payload, None
return None, "Longs and shorts data is only available on Bitfinex."
elif action == "sl":
if exchange.id in ["bitfinex2"]:
try:
longs = exchange.properties.publicGetStats1KeySizeSymbolLongLast(
{
"key": "pos.size",
"size": "1m",
"symbol": "t{}".format(ticker.id),
"side": "short",
"section": "last"
})
shorts = exchange.properties.publicGetStats1KeySizeSymbolShortLast(
{
"key": "pos.size",
"size": "1m",
"symbol": "t{}".format(ticker.id),
"side": "short",
"section": "last"
})
ratio = shorts[1] / (longs[1] + shorts[1]) * 100
except:
return None, None
payload = {
"quotePrice":
"{:.1f} % shorts / {:.1f} % longs".format(
ratio, 100 - ratio),
"quoteVolume":
None,
"quoteConvertedPrice":
None,
"quoteConvertedVolume":
None,
"title":
"{} shorts/longs ratio".format(ticker.name),
"baseTicker":
ticker.base,
"quoteTicker":
ticker.quote,
"change":
None,
"thumbnailUrl":
TickerParser.get_coingecko_image(ticker.base),
"messageColor":
"deep purple",
"sourceText":
"Data on {}".format(exchange.name),
"platform":
"LLD",
"raw": {
"quotePrice": [longs[1], shorts[1]],
"quoteVolume": None,
"ticker": ticker,
"exchange": exchange,
"timestamp": time.time()
}
}
return payload, None
return None, "Longs and shorts data is only available on Bitfinex."
elif action == "dom":
try:
rawData = self.coinGecko.get_global()
except:
return None, "Requested dominance data for `{}` is not available.".format(
ticker.name)
if ticker.base.lower() not in rawData["market_cap_percentage"]:
return None, "Dominance for {} does not exist.".format(
ticker.base)
coinDominance = rawData["market_cap_percentage"][
ticker.base.lower()]
payload = {
"quotePrice":
"{} dominance: {:,.2f} %".format(ticker.base,
coinDominance),
"quoteVolume":
None,
"quoteConvertedPrice":
None,
"quoteConvertedVolume":
None,
"title":
"Market Dominance",
"baseTicker":
ticker.base,
"quoteTicker":
ticker.quote,
"change":
None,
"thumbnailUrl":
TickerParser.get_coingecko_image(ticker.base),
"messageColor":
"deep purple",
"sourceText":
"Market information from CoinGecko",
"platform":
"LLD",
"raw": {
"quotePrice": coinDominance,
"quoteVolume": None,
"ticker": ticker,
"exchange": None,
"timestamp": time.time()
}
}
return payload, None
else:
return None, None
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]: self.logging.report_exception()
return None, None
def request_iexc_quote(self, request):
payload, quoteMessage, updatedQuoteMessage = None, None, None
for platform in ["Stocks", "Forex"]:
if platform == "Stocks":
payload, updatedQuoteMessage = self.request_iexc_stocks(
request)
elif platform == "Forex":
payload, updatedQuoteMessage = self.request_iexc_forex(request)
if payload is not None:
return payload, updatedQuoteMessage
elif updatedQuoteMessage is not None:
quoteMessage = updatedQuoteMessage
return None, quoteMessage
def request_iexc_stocks(self, request):
ticker = request.get_ticker()
exchange = request.get_exchange()
try:
try:
stock = Stock(ticker.id, token=os.environ["IEXC_KEY"])
rawData = stock.get_quote().loc[ticker.id]
if ticker.quote is None and exchange is not None:
return None, "Price for `{}` is only available on `{}`.".format(
ticker.id, rawData["primaryExchange"])
if rawData is None or (rawData["latestPrice"] is None
and rawData["delayedPrice"] is None):
return None, None
except:
return None, None
try:
coinThumbnail = stock.get_logo().loc[ticker.id]["url"]
except:
coinThumbnail = static_storage.icon
latestPrice = rawData["delayedPrice"] if rawData[
"latestPrice"] is None else rawData["latestPrice"]
price = float(latestPrice if "isUSMarketOpen" not in rawData or
rawData["isUSMarketOpen"] or "extendedPrice" not in
rawData or rawData["extendedPrice"] is None else
rawData["extendedPrice"])
if ticker.isReversed: price = 1 / price
volume = float(rawData["latestVolume"])
priceChange = (1 / rawData["change"]
if ticker.isReversed else rawData["change"]
) / price * 100 if "change" in rawData and rawData[
"change"] is not None else 0
payload = {
"quotePrice":
"{:,.5f}".format(price)
if ticker.isReversed else "{}".format(price),
"quoteVolume":
volume,
"quoteConvertedPrice":
None,
"quoteConvertedVolume":
None,
"title":
ticker.name,
"baseTicker":
"contracts",
"quoteTicker":
ticker.quote,
"change":
priceChange,
"thumbnailUrl":
coinThumbnail,
"messageColor":
"amber" if priceChange == 0 else
("green" if priceChange > 0 else "red"),
"sourceText":
"provided by IEX Cloud",
"platform":
"IEXC",
"raw": {
"quotePrice": [price],
"quoteVolume": volume,
"ticker": ticker,
"exchange": exchange,
"timestamp": time.time()
}
}
return payload, None
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]: self.logging.report_exception()
return None, None
def request_iexc_forex(self, request):
ticker = request.get_ticker()
exchange = request.get_exchange()
try:
try:
if exchange is not None: return None, None
rawData = requests.get(
"https://cloud.iexapis.com/stable/fx/latest?symbols={}&token={}"
.format(ticker.id, os.environ["IEXC_KEY"])).json()
if rawData is None or type(rawData) is not list or len(
rawData) == 0:
return None, None
except:
return None, None
price = rawData[0]["rate"]
if ticker.isReversed: price = 1 / price
payload = {
"quotePrice": "{:,.5f}".format(price),
"quoteVolume": None,
"quoteConvertedPrice": None,
"quoteConvertedVolume": None,
"title": ticker.name,
"baseTicker": ticker.base,
"quoteTicker": ticker.quote,
"change": None,
"thumbnailUrl": static_storage.icon,
"messageColor": "deep purple",
"sourceText": "provided by IEX Cloud",
"platform": "IEXC",
"raw": {
"quotePrice": [price],
"quoteVolume": None,
"ticker": ticker,
"exchange": exchange,
"timestamp": time.time()
}
}
return payload, None
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]: self.logging.report_exception()
return None, None
def request_fear_greed_index(self, request):
try:
requestUrl, _ = request.build_url()
r = requests.get(requestUrl).json()
greedIndex = int(r["data"][0]["value"])
payload = {
"quotePrice":
greedIndex,
"quoteVolume":
None,
"quoteConvertedPrice":
"≈ {}".format(r["data"][0]["value_classification"].lower()),
"quoteConvertedVolume":
None,
"title":
"Fear & Greed Index",
"baseTicker":
None,
"quoteTicker":
None,
"change":
greedIndex - int(r["data"][1]["value"]),
"thumbnailUrl":
static_storage.icon,
"messageColor":
"deep purple",
"sourceText":
"Data provided by Alternative.me",
"platform":
"Alternative.me",
"raw": {
"quotePrice": greedIndex,
"quoteVolume": None,
"ticker": request.get_ticker(),
"exchange": None,
"timestamp": time.time()
}
}
return payload, None
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]: self.logging.report_exception()
return None, None
def request_ccxt_depth(self, request):
ticker = request.get_ticker()
exchange = request.get_exchange()
try:
if exchange is None: return None, None
exchange = Exchange(exchange.id)
try:
depthData = exchange.properties.fetch_order_book(ticker.symbol)
bestBid = depthData["bids"][0]
bestAsk = depthData["asks"][0]
lastPrice = (bestBid[0] + bestAsk[0]) / 2
except:
return None, None
imageData = self.generate_depth_image(depthData, bestBid, bestAsk,
lastPrice)
return imageData, None
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]: self.logging.report_exception()
return None, None
def request_iexc_depth(self, request):
ticker = request.get_ticker()
exchange = request.get_exchange()
try:
try:
stock = Stock(ticker.id, token=os.environ["IEXC_KEY"])
depthData = stock.get_book()[ticker.id]
rawData = stock.get_quote().loc[ticker.id]
if ticker.quote is None and exchange is not None:
return None, "Orderbook visualization for `{}` is only available on `{}`.".format(
ticker.id, rawData["primaryExchange"])
bestBid = depthData["bids"][0]
bestAsk = depthData["asks"][0]
lastPrice = (bestBid[0] + bestAsk[0]) / 2
except:
return None, None
imageData = self.generate_depth_image(depthData, bestBid, bestAsk,
lastPrice)
return imageData, None
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]: self.logging.report_exception()
return None, None
def generate_depth_image(self, depthData, bestBid, bestAsk, lastPrice):
bidTotal = 0
xBids = [bestBid[0]]
yBids = [0]
for bid in depthData['bids']:
if len(xBids) < 10 or bid[0] > lastPrice * 0.9:
bidTotal += bid[1]
xBids.append(bid[0])
yBids.append(bidTotal)
askTotal = 0
xAsks = [bestAsk[0]]
yAsks = [0]
for ask in depthData['asks']:
if len(xAsks) < 10 or ask[0] < lastPrice * 1.1:
askTotal += ask[1]
xAsks.append(ask[0])
yAsks.append(askTotal)
fig = plt.figure(facecolor="#131722")
ax = fig.add_subplot(1, 1, 1)
ax.tick_params(color="#787878", labelcolor="#D9D9D9")
ax.step(xBids, yBids, where="post", color="#27A59A")
ax.step(xAsks, yAsks, where="post", color="#EF534F")
ax.fill_between(xBids,
yBids,
0,
facecolor="#27A59A",
interpolate=True,
step="post",
alpha=0.33,
zorder=2)
ax.fill_between(xAsks,
yAsks,
0,
facecolor="#EF534F",
interpolate=True,
step="post",
alpha=0.33,
zorder=2)
plt.axvline(x=lastPrice, color="#758696", linestyle="--")
ax.set_facecolor("#131722")
for spine in ax.spines.values():
spine.set_edgecolor("#787878")
ax.autoscale(enable=True, axis="both", tight=True)
def on_draw(event):
bboxes = []
for label in ax.get_yticklabels():
bbox = label.get_window_extent()
bboxi = bbox.transformed(fig.transFigure.inverted())
bboxes.append(bboxi)
bbox = mtransforms.Bbox.union(bboxes)
if fig.subplotpars.left < bbox.width:
fig.subplots_adjust(left=1.1 * bbox.width)
fig.canvas.draw()
return False
ax.yaxis.set_major_formatter(
tkr.FuncFormatter(lambda x, p: format(int(x), ',')))
plt.setp(ax.get_xticklabels(),
rotation=45,
horizontalalignment='right')
lastPriceLabel = bestAsk[0] if bestAsk[1] >= bestBid[1] else bestBid[0]
xLabels = list(plt.xticks()[0][1:])
yLabels = list(plt.yticks()[0][1:])
for label in xLabels:
plt.axvline(x=label, color="#363C4F", linewidth=1, zorder=1)
for label in yLabels:
plt.axhline(y=label, color="#363C4F", linewidth=1, zorder=1)
diffLabels = 1 - xLabels[0] / xLabels[1]
bottomBound, topBound = lastPriceLabel * (
1 - diffLabels * (1 / 4)), lastPriceLabel * (1 + diffLabels *
(1 / 4))
xLabels = [l for l in xLabels if not (bottomBound <= l <= topBound)]
plt.xticks(xLabels + [lastPriceLabel])
plt.yticks(yLabels)
ax.set_xlim([xBids[-1], xAsks[-1]])
ax.set_ylim([0, max(bidTotal, askTotal)])
fig.canvas.mpl_connect("draw_event", on_draw)
plt.tight_layout()
rawImageData = BytesIO()
plt.savefig(rawImageData, format="png", edgecolor="none")
rawImageData.seek(0)
imageBuffer = BytesIO()
chartImage = Image.new("RGBA", (1600, 1200))
chartImage.paste(Image.open(rawImageData))
chartImage = Image.alpha_composite(chartImage,
self.imageOverlays["Alpha depth"])
chartImage.save(imageBuffer, format="png")
imageData = base64.b64encode(imageBuffer.getvalue())
imageBuffer.close()
return imageData
class DetailProcessor(object):
coinGecko = CoinGeckoAPI()
def __init__(self):
self.isServiceAvailable = True
signal.signal(signal.SIGINT, self.exit_gracefully)
signal.signal(signal.SIGTERM, self.exit_gracefully)
self.logging = error_reporting.Client()
self.cache = Cache()
context = zmq.Context.instance()
self.socket = context.socket(zmq.ROUTER)
self.socket.bind("tcp://*:6900")
print("[Startup]: Detail Server is online")
def exit_gracefully(self):
print("[Startup]: Detail Server is exiting")
self.socket.close()
self.isServiceAvailable = False
def run(self):
while self.isServiceAvailable:
try:
response = None, None
origin, delimeter, clientId, service, request = self.socket.recv_multipart(
)
request = pickle.loads(zlib.decompress(request))
if request.timestamp + 30 < time.time(): continue
if service == b"detail":
response = self.request_detail(request)
except (KeyboardInterrupt, SystemExit):
return
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]:
self.logging.report_exception()
finally:
try:
self.socket.send_multipart([
origin, delimeter,
zlib.compress(pickle.dumps(response, -1))
])
except:
pass
def request_detail(self, request):
payload, tradeMessage, updatedTradeMessage = None, None, None
for platform in request.platforms:
request.set_current(platform=platform)
hashCode = hash(request.requests[platform])
fromCache = False
if request.can_cache() and self.cache.has(hashCode):
payload, updatedQuoteMessage = self.cache.get(hashCode), None
fromCache = True
elif platform == "CoinGecko":
payload, updatedQuoteMessage = self.request_coingecko_details(
request)
if payload is not None:
if request.can_cache() and not fromCache:
self.cache.set(hashCode, payload)
if request.authorId != 401328409499664394 and request.requests[
platform].ticker.base is not None:
database.document("dataserver/statistics/{}/{}".format(
platform, str(uuid.uuid4()))).set({
"timestamp":
time.time(),
"authorId":
str(request.authorId),
"ticker": {
"base": request.requests[platform].ticker.base,
"quote":
request.requests[platform].ticker.quote,
"id": request.requests[platform].ticker.id,
"bias": request.parserBias
},
"exchange":
None if request.requests[platform].exchange is None
else request.requests[platform].exchange.id
})
return payload, updatedTradeMessage
elif updatedTradeMessage is not None:
tradeMessage = updatedTradeMessage
return None, tradeMessage
def request_coingecko_details(self, request):
ticker = request.get_ticker()
try:
try:
rawData = self.coinGecko.get_coin_by_id(id=ticker.symbol,
localization="false",
tickers=False,
market_data=True,
community_data=True,
developer_data=True)
except:
return None, None
payload = {
"name":
"{} ({})".format(rawData["name"], ticker.base),
"rank":
rawData["market_data"]["market_cap_rank"],
"image":
rawData["image"]["large"],
"marketcap":
"" if rawData["market_data"]["market_cap"] is None else
rawData["market_data"]["market_cap"].get("usd", ""),
"volume":
None if rawData["market_data"]["total_volume"] is None else
rawData["market_data"]["total_volume"].get("usd"),
"supply": {
"total":
None if rawData["market_data"]["total_supply"] is None else
rawData["market_data"]["total_supply"],
"circulating":
None
if rawData["market_data"]["circulating_supply"] is None
else rawData["market_data"]["circulating_supply"]
},
"score": {
"developer": rawData["developer_score"],
"community": rawData["community_score"],
"liquidity": rawData["liquidity_score"],
"public interest": rawData["public_interest_score"]
},
"price": {
"current":
rawData["market_data"]["current_price"].get("usd"),
"ath": rawData["market_data"]["ath"].get("usd"),
"atl": rawData["market_data"]["atl"].get("usd")
},
"change": {
"past day":
rawData["market_data"]
["price_change_percentage_24h_in_currency"].get("usd"),
"past month":
rawData["market_data"]
["price_change_percentage_30d_in_currency"].get("usd"),
"past year":
rawData["market_data"]
["price_change_percentage_1y_in_currency"].get("usd")
},
"sourceText":
"from CoinGecko",
"platform":
"CoinGecko",
}
return payload, None
except Exception:
print(traceback.format_exc())
if os.environ["PRODUCTION_MODE"]: self.logging.report_exception()
return None, None
print("ERR ERR wrong number of arguments for 'set' command")
continue
key = command[0]
attributes = {}
for i in range(0, len(command), 2):
attributes[command[i]] = command[i + 1]
# print(attributes)
print(cache.set(key, attributes[key], attributes))
elif option == "GET":
if len(command) != 1:
print("ERR ERR wrong number of arguments for 'get' command")
continue
key = command[0]
print(cache.get(key))
elif option == "EXPIRE":
if len(command) != 2:
print("ERR ERR wrong number of arguments for 'expire' command")
continue
key = command[0]
time = int(command[1])
print(cache.set_expire(key, time))
elif option == "TTL":
if len(command) != 1:
print("ERR ERR wrong number of arguments for 'ttl' command")
continue
key = command[0]
print(cache.get_TTL(key))
class ChainDb(object):
def __init__(self,
settings,
datadir,
log,
mempool,
netmagic,
readonly=False,
fast_dbm=False,
compression=False):
self.settings = settings
self.log = log
self.mempool = mempool
self.readonly = readonly
self.netmagic = netmagic
self.fast_dbm = fast_dbm
self.blk_cache = Cache(1000)
self.orphans = {}
self.orphan_deps = {}
self.compress_on_write = compression
# LevelDB to hold:
# tx:* transaction outputs
# misc:* state
# height:* list of blocks at height h
# blkmeta:* block metadata
# blocks:* block seek point in stream
self.blk_write = io.BufferedWriter(
io.FileIO(datadir + '/blocks.dat', 'ab'))
self.blk_read = io.BufferedReader(
io.FileIO(datadir + '/blocks.dat', 'rb'))
self.db = leveldb.LevelDB(datadir + '/leveldb')
try:
self.db.Get('misc:height')
except KeyError:
self.log.write("INITIALIZING EMPTY BLOCKCHAIN DATABASE")
batch = leveldb.WriteBatch()
batch.Put('misc:height', str(-1))
batch.Put('misc:msg_start', self.netmagic.msg_start)
batch.Put('misc:tophash', ser_uint256(0L))
batch.Put('misc:total_work', hex(0L))
self.db.Write(batch)
try:
start = self.db.Get('misc:msg_start')
if start != self.netmagic.msg_start: raise KeyError
except KeyError:
self.log.write(
"Database magic number mismatch. Data corruption or incorrect network?"
)
raise RuntimeError
def puttxidx(self, txhash, txidx, batch=None):
ser_txhash = ser_uint256(txhash)
try:
self.db.Get('tx:' + ser_txhash)
old_txidx = self.gettxidx(txhash)
self.log.write(
"WARNING: overwriting duplicate TX %064x, height %d, oldblk %064x, oldspent %x, newblk %064x"
% (txhash, self.getheight(), old_txidx.blkhash,
old_txidx.spentmask, txidx.blkhash))
except KeyError:
pass
batch = self.db if batch is not None else batch
batch.Put('tx:' + ser_txhash,
hex(txidx.blkhash) + ' ' + hex(txidx.spentmask))
return True
def gettxidx(self, txhash):
ser_txhash = ser_uint256(txhash)
try:
ser_value = self.db.Get('tx:' + ser_txhash)
except KeyError:
return None
pos = string.find(ser_value, ' ')
txidx = TxIdx()
txidx.blkhash = long(ser_value[:pos], 16)
txidx.spentmask = long(ser_value[pos + 1:], 16)
return txidx
def gettx(self, txhash):
txidx = self.gettxidx(txhash)
if txidx is None:
return None
block = self.getblock(txidx.blkhash)
if block:
for tx in block.vtx:
tx.calc_sha256()
if tx.sha256 == txhash:
return tx
self.log.write("ERROR: Missing TX %064x in block %064x" %
(txhash, txidx.blkhash))
return None
def haveblock(self, blkhash, checkorphans):
if self.blk_cache.exists(blkhash):
return True
if checkorphans and blkhash in self.orphans:
return True
ser_hash = ser_uint256(blkhash)
try:
self.db.Get('blocks:' + ser_hash)
return True
except KeyError:
return False
def have_prevblock(self, block):
if self.getheight() < 0 and block.sha256 == self.netmagic.block0:
return True
if self.haveblock(block.hashPrevBlock, False):
return True
return False
def getblock(self, blkhash):
block = self.blk_cache.get(blkhash)
if block is not None:
return block
ser_hash = ser_uint256(blkhash)
try:
# Lookup the block index, seek in the file
fpos = long(self.db.Get('blocks:' + ser_hash))
self.blk_read.seek(fpos)
# read and decode "block" msg
recvbuf = self.blk_read.read(4 + 4)
if recvbuf[:4] == 'ZLIB':
msg_len = int(recvbuf[4:8].encode('hex'), 16)
recvbuf = self.blk_read.read(msg_len)
f = cStringIO.StringIO(zlib.decompress(recvbuf))
msg = message_read(self.netmagic, f)
else:
self.blk_read.seek(fpos)
msg = message_read(self.netmagic, self.blk_read)
if msg is None:
return None
block = msg.block
except KeyError:
return None
self.blk_cache.put(blkhash, block)
return block
def spend_txout(self, txhash, n_idx, batch=None):
txidx = self.gettxidx(txhash)
if txidx is None:
return False
txidx.spentmask |= (1L << n_idx)
self.puttxidx(txhash, txidx, batch)
return True
def clear_txout(self, txhash, n_idx, batch=None):
txidx = self.gettxidx(txhash)
if txidx is None:
return False
txidx.spentmask &= ~(1L << n_idx)
self.puttxidx(txhash, txidx, batch)
return True
def unique_outpts(self, block):
outpts = {}
txmap = {}
for tx in block.vtx:
if tx.is_coinbase:
continue
txmap[tx.sha256] = tx
for txin in tx.vin:
v = (txin.prevout.hash, txin.prevout.n)
if v in outs:
return None
outpts[v] = False
return (outpts, txmap)
def txout_spent(self, txout):
txidx = self.gettxidx(txout.hash)
if txidx is None:
return None
if txout.n > 100000: # outpoint index sanity check
return None
if txidx.spentmask & (1L << txout.n):
return True
return False
class Core(object):
""" Starts the :class:`Cache`
Arguments
-----------
db : :data:`bfly.Butterfly._db_type`
A fully-loaded database
Attributes
------------
_db: :data:`bfly.Butterfly._db_type`
Taken from first argument ``db``
_cache: :class:`Cache`
Able to store images and metadata using \
:class:`UtilityLayer.RUNTIME` instance \
from ``db`` argument
"""
def __init__(self, db):
# Get DB Terms
self._db = db
RUNTIME = db.RUNTIME
# Make Cache with keywords
self._cache = Cache(RUNTIME)
#####
# All methods to load data
# 1. get_info answers an InfoQuery i_query.
# 2. get_data answers a DataQuery d_query.
#
# Both get_info or get_data call update_query.
#
# To give answers , update_query uses _cache or:
# 1. make_data_query if only i_query given.
# 2. make_tile_query with new or given d_query.
#####
@staticmethod
def get_groups(i_query):
""" dumps group list for ``i_query`` as a string
Arguments
----------
i_query: :class:`QueryLayer.InfoQuery`
A request for a list of groups
Returns
--------
str
A list of all groups for the ``i_query``
"""
return i_query.dump
def get_edits(self, i_query, msg={}):
""" dumps websocket updates to ``i_query`` as a string
Calls :meth:`update_query` with more information\
from the cache or from the properties of a tile.
Arguments
----------
i_query: :class:`QueryLayer.InfoQuery`
A request for information
Returns
--------
str
Wesocket info for :class:`QueryLayer.InfoQuery`
"""
keywords = self.update_query(i_query)
# Update current query with preloaded terms
i_query.update_source(keywords)
# Execute weboscket command if needed
changed = i_query.websocket_edit(msg)
# Add to cache and query
if len(changed):
keywords.update(changed)
i_query.update_source(keywords)
self._cache.set(i_query.key, keywords)
# Return the i_query info
return i_query.dump
def get_info(self, i_query):
""" dumps answer to ``i_query`` as a string
Calls :meth:`update_query` with more information\
from the cache or from the properties of a tile.
Arguments
----------
i_query: :class:`QueryLayer.InfoQuery`
A request for information
Returns
--------
str
Channel info for :class:`QueryLayer.InfoQuery`
"""
keywords = self.update_query(i_query)
# Update current query with preloaded terms
i_query.update_source(keywords)
# Return the i_query info
return i_query.dump
def get_dataset(self, i_query):
""" dumps dataset from ``i_query`` as a string
Calls :meth:`update_query` with more information\
from the cache or from the properties of a tile.
Arguments
----------
i_query: :class:`QueryLayer.InfoQuery`
A request for information
Returns
--------
str
Dataset info for :class:`QueryLayer.InfoQuery`
"""
all_channels = []
# Update all the channels in the dataset
for channel in i_query.channels:
# Set the i_query to a given channel
i_query.set_channel(channel)
# Update the info for the channel
keywords = self.update_query(i_query)
# Add additionl keywords if needed
channel_key = i_query.OUTPUT.INFO.CHANNEL.NAME
keywords[channel_key] = channel[channel_key]
# Add to list of channels
all_channels.append(keywords)
# Clear the channel data
i_query.set_channel({})
# Update current query with preloaded terms
return i_query.dump_dataset(all_channels)
def get_data(self, d_query):
""" dumps answer to ``d_query`` as a string
Calls :meth:`update_query` with more information\
from the cache or from the properties of a tile. \
Also calls :meth:`find_tiles` to get the complete\
image needed to answer the ``d_query``.
Arguments
----------
i_query: :class:`QueryLayer.InfoQuery`
A request for information
Returns
--------
str
Answer for the :class:`QueryLayer.InfoQuery`
"""
keywords = self.update_query(d_query)
# Update current query with preloaded terms
d_query.update_source(keywords)
# Get the image for the d_query
image = self.find_tiles(d_query)
return self.write_image(d_query, image)
@staticmethod
def make_data_query(i_query):
""" Make a data query from an info query
Arguments
----------
i_query: :class:`InfoQuery`
only needs ``PATH`` set in :data:`OUTPUT.INFO`
Returns
--------
:class:`DataQuery`
takes only the `PATH` from ``i_query``
"""
# Begin building needed keywords
i_path = i_query.OUTPUT.INFO.PATH
return DataQuery(**{
i_query.INPUT.METHODS.NAME: 'data',
i_path.NAME: i_path.VALUE
})
@staticmethod
def make_tile_query(d_query, t_index=np.uint32([0,0,0])):
""" Make a :class:`TileQuery` from :class:`DataQuery`
Arguments
----------
d_query: :class:`DataQuery`
only needs ``PATH`` set in :data:`OUTPUT.INFO`
t_index: numpy.ndarray
The 3x1 count of tiles form the origin
Returns
--------
:class:`TileQuery`
One tile request in the given data request
"""
tile_crop = d_query.all_in_some(t_index)
return TileQuery(d_query, t_index, tile_crop)
def update_query(self, query):
""" Finds missing query details from cache or tile
Makes ``keywords`` from either the :data:`_cache` or \
from a new :class:`TileQuery` to update the given ``query``
Arguments
----------
query: :class:`Query`
Either an :class:`InfoQuery` or a :class:`DataQuery`
Returns
--------
keywords: dict
Can pass to :meth:`Query.update_source` or combine \
to pass to :meth:`Query.update_dataset`.
"""
keywords = self._cache.get(query.key)
if not len(keywords):
d_query = query
# Create a preparatory data_query
if not isinstance(query, DataQuery):
d_query = self.make_data_query(query)
# Create a preparatory tile_query
t0_query = self.make_tile_query(d_query)
# Update keywords and set the cache
keywords = t0_query.preload_source
self._cache.set(query.key, keywords)
# Return the updated keywords
return keywords
#####
# Image Specific Methods
#####
def find_tiles(self, d_query):
""" Load the requested image for a :class:`DataQuery`
Arguments
----------
d_query: :class:`DataQuery`
Request for a scaled subvolume of a source image
Returns
numpy.ndarray
The full image data for the requested region
"""
first_tile_index = d_query.tile_bounds[0]
all_tiles = np.argwhere(np.ones(d_query.tile_shape))
cutout = np.zeros(d_query.target_shape, d_query.dtype)
tiles_needed = first_tile_index + all_tiles
for t_index in tiles_needed:
# Make a query for the given tile
t_query = self.make_tile_query(d_query, t_index)
tile = self.load_tile(t_query)
if not len(tile):
continue
# Fill the tile into the full cutout
to_cut = [t_query.target_origin, tile.shape]
[Z0,Y0,X0],[Z1,Y1,X1] = d_query.some_in_all(*to_cut)
cutout[Z0:Z1,Y0:Y1,X0:X1] = tile
return cutout
def find_unique(self, d_query):
""" Get unique values for a :class:`DataQuery`
Arguments
----------
d_query: :class:`DataQuery`
Request for a scaled subvolume of a source image
Returns
set
The set of unique values for the request
"""
first_tile_index = d_query.tile_bounds[0]
all_tiles = np.argwhere(np.ones(d_query.tile_shape))
tiles_needed = first_tile_index + all_tiles
# Set of all unique values
unique = set()
for t_index in tiles_needed:
# Make a query for the given tile
t_query = self.make_tile_query(d_query, t_index)
tile = self.load_tile(t_query)
# Union of unique values with the full set
tile_bins = np.bincount(tile.flatten()) > 0
unique = unique | set(np.where(tile_bins)[0])
return unique
def load_tile(self, t_query):
""" Load a single tile from the cache or from disk
Arguments
----------
t_query: :class:`TileQuery`
With tile coordinates and volume within the tile
Returns
--------
numpy.ndarray
The subregion image data for the requested tile
"""
# grab request size for query
t_bounds = t_query.target_bounds
t_origin = t_query.target_tile_bounds[0]
(K0,J0,I0),(K1,J1,I1) = t_bounds-t_origin
# Load from cache or from disk if needed
cache_tile = self._cache.get(t_query.key)
if len(cache_tile):
return cache_tile[K0:K1,J0:J1,I0:I1]
# Load from disk
tile = t_query.tile
if not len(tile):
return []
self._cache.set(t_query.key, tile)
return tile[K0:K1,J0:J1,I0:I1]
@staticmethod
def view_volume(view, vol):
""" Display a volume in color or grayscale
Arguments
----------
view: str
The requested color or gray view of the data
vol: str
Raw volume from :class:`Cache` / :class:`Datasource`
Returns
--------
numpy.ndarray
Colorized or original raw volume
"""
# Set up a colormap
def id_to_color(vol):
colors = np.zeros((3,)+ vol.shape).astype(np.uint8)
colors[0] = np.mod(107 * vol, 700).astype(np.uint8)
colors[1] = np.mod(509 * vol, 900).astype(np.uint8)
colors[2] = np.mod(200 * vol, 777).astype(np.uint8)
return np.moveaxis(colors,0,-1)
# Colormap if a colormap view
if view.VALUE == view.COLOR.NAME:
return id_to_color(vol)
return vol
def write_image(self, d_query, volume):
""" Format a volume for a given :class:`DataQuery`
Arguments
----------
d_query: :class:`DataQuery`
With the format and view for the requested volume
volume: numpy.ndarray
Raw volume from :class:`Cache` / :class:`Datasource`
Returns
--------
str:
The image response as a formatted bytestring
"""
img_format = d_query.INPUT.IMAGE.FORMAT
img_view = d_query.INPUT.IMAGE.VIEW
img_type = d_query.OUTPUT.INFO.TYPE
# Only if grayscale view is set
if img_view.VALUE == img_view.GRAY.NAME:
# set the view based on the format
is_big_int = img_type.VALUE in img_type.ID_LIST
no_big_int_gray = img_format.VALUE in img_format.COLOR_LIST
# If big integers must not be grayscale, try colormap
if is_big_int and no_big_int_gray:
img_view.VALUE = img_view.COLOR.NAME
# If Multiple slices cannot be formatted
if img_format.VALUE not in img_format.VOL_LIST:
shape = volume.shape
if shape[0] > 1:
# Flatten the volume to image
volume = volume.reshape(1, -1, shape[-1])
# Use colormap / RGB style encoding of ID data
vol = self.view_volume(img_view, volume)
if img_format.VALUE in ['raw']:
output = StringIO.StringIO()
output.write(vol.tobytes())
vol_string = output.getvalue()
return vol_string
if img_format.VALUE in ['npz']:
output = StringIO.StringIO()
np.save(output, vol[np.newaxis])
vol_string = output.getvalue()
return zlib.compress(vol_string)
if img_format.VALUE in img_format.ZIP_LIST:
output = StringIO.StringIO()
volstring = vol[0].T.tostring('F')
output.write(zlib.compress(volstring))
return output.getvalue()
if img_format.VALUE in img_format.TIF_LIST:
output = StringIO.StringIO()
tiffvol = vol[0]
tifffile.imsave(output, tiffvol)
return output.getvalue()
filetype = "." + img_format.VALUE
image = cv2.imencode(filetype, vol[0])
return image[1].tostring()
