def init_headers(self, db_height):
self.chunk_cache = {}
self.headers_filename = os.path.join(self.headers_path, "blockchain_headers")
if os.path.exists(self.headers_filename):
height = os.path.getsize(self.headers_filename) / 80 - 1 # the current height
if height > 0:
prev_hash = self.hash_header(self.read_header(height))
else:
prev_hash = None
else:
open(self.headers_filename, "wb").close()
prev_hash = None
height = -1
if height < db_height:
print_log("catching up missing headers:", height, db_height)
try:
while height < db_height:
height = height + 1
header = self.get_header(height)
if height > 1:
assert prev_hash == header.get("prev_block_hash")
self.write_header(header, sync=False)
prev_hash = self.hash_header(header)
if (height % 1000) == 0:
print_log("headers file:", height)
except KeyboardInterrupt:
self.flush_headers()
sys.exit()
self.flush_headers()
def get_path(self, target):
word = target
key = ''
path = [ '' ]
i = self.db_utxo.iterator(start='')
while key != target:
i.seek(key + word[0])
try:
new_key, _ = i.next()
is_child = new_key.startswith(key + word[0])
except StopIteration:
is_child = False
if is_child:
if target.startswith(new_key):
# add value to the child node
key = new_key
word = target[len(key):]
if key == target:
break
else:
assert key not in path
path.append(key)
else:
print_log('not in tree', self.db_utxo.get(key+word[0]), new_key.encode('hex'))
return False
else:
assert key in path
break
return path
def getfullblock(self, block_hash):
block = self.bitcoind("getblock", [block_hash])
rawtxreq = []
i = 0
for txid in block["tx"]:
rawtxreq.append({"method": "getrawtransaction", "params": [txid], "id": i})
i += 1
postdata = dumps(rawtxreq)
try:
respdata = urllib.urlopen(self.bitcoind_url, postdata).read()
except:
print_log("darkcoind error (getfullblock)")
traceback.print_exc(file=sys.stdout)
self.shared.stop()
r = loads(respdata)
rawtxdata = []
for ir in r:
if ir["error"] is not None:
self.shared.stop()
print_log("Error: make sure you run darkcoind with txindex=1; use -reindex if needed.")
raise BaseException(ir["error"])
rawtxdata.append(ir["result"])
block["tx"] = rawtxdata
return block
def get_history(self, addr, cache_only=False):
with self.cache_lock:
hist = self.history_cache.get(addr)
if hist is not None:
return hist
if cache_only:
return -1
with self.dblock:
try:
hist = self.storage.get_history(addr)
is_known = True
except:
print_log("error get_history")
traceback.print_exc(file=sys.stdout)
raise
if hist:
is_known = True
else:
hist = []
is_known = False
# add memory pool
with self.mempool_lock:
for txid, delta in self.mempool_hist.get(addr, []):
hist.append({"tx_hash": txid, "height": 0})
# add something to distinguish between unused and empty addresses
if hist == [] and is_known:
hist = ["*"]
with self.cache_lock:
self.history_cache[addr] = hist
return hist
def get_path(self, target, new=False):
x = self.db_utxo.get(target)
if not new and x is None:
raise BaseException('key not in tree', target.encode('hex'))
if new and x is not None:
# raise BaseException('key already in tree', target.encode('hex'))
# occurs at block 91880 (duplicate txid)
print_log('key already in tree', target.encode('hex'))
return True
remaining = target
key = ''
path = []
while key != target:
node = self.get_node(key)
if node is None:
break
#raise # should never happen
path.append(key)
c = remaining[0]
if not node.has(c):
break
skip = self.get_skip(key + c)
key = key + c + skip
if not target.startswith(key):
break
remaining = target[len(key):]
return path
def update_tx_cache(self, txid):
inrows = self.get_tx_inputs(txid, False)
for row in inrows:
_hash = self.binout(row[6])
if not _hash:
#print_log("WARNING: missing tx_in for tx", txid)
continue
address = hash_to_address(chr(self.addrtype), _hash)
with self.cache_lock:
if address in self.tx_cache:
print_log("cache: invalidating", address)
self.tx_cache.pop(address)
self.address_queue.put(address)
outrows = self.get_tx_outputs(txid, False)
for row in outrows:
_hash = self.binout(row[6])
if not _hash:
#print_log("WARNING: missing tx_out for tx", txid)
continue
address = hash_to_address(chr(self.addrtype), _hash)
with self.cache_lock:
if address in self.tx_cache:
print_log("cache: invalidating", address)
self.tx_cache.pop(address)
self.address_queue.put(address)
def getfullblock(self, block_hash):
block = self.bitcoind('getblock', [block_hash])
rawtxreq = []
i = 0
for txid in block['tx']:
rawtxreq.append({
"method": "getrawtransaction",
"params": [txid],
"id": i,
})
i += 1
postdata = dumps(rawtxreq)
try:
respdata = urllib.urlopen(self.bitcoind_url, postdata).read()
except:
logger.error("groestlcoind error (getfullblock)",exc_info=True)
self.shared.stop()
r = loads(respdata)
rawtxdata = []
for ir in r:
if ir['error'] is not None:
self.shared.stop()
print_log("Error: make sure you run groestlcoind with txindex=1; use -reindex if needed.")
raise BaseException(ir['error'])
rawtxdata.append(ir['result'])
block['tx'] = rawtxdata
return block
def process(self, session, request, cache_only=False):
message_id = request['id']
method = request['method']
params = request.get('params', [])
result = None
error = None
if method == 'blockchain.numblocks.subscribe':
with self.watch_lock:
if session not in self.watch_blocks:
self.watch_blocks.append(session)
result = self.height
elif method == 'blockchain.headers.subscribe':
with self.watch_lock:
if session not in self.watch_headers:
self.watch_headers.append(session)
result = self.header
elif method == 'blockchain.address.subscribe':
try:
address = params[0]
result = self.get_status(address, cache_only)
with self.watch_lock:
l = self.watched_addresses.get(address)
if l is None:
self.watched_addresses[address] = [session]
elif session not in l:
l.append(session)
except BaseException, e:
error = str(e) + ': ' + address
print_log("error:", error)
def print_time(self, num_tx):
delta = time.time() - self.time_ref
# leaky averages
seconds_per_block, tx_per_second, n = self.avg_time
alpha = (1. + 0.01 * n)/(n+1)
seconds_per_block = (1-alpha) * seconds_per_block + alpha * delta
alpha2 = alpha * delta / seconds_per_block
tx_per_second = (1-alpha2) * tx_per_second + alpha2 * num_tx / delta
self.avg_time = seconds_per_block, tx_per_second, n+1
remaining_blocks = self.bitcoind_height - self.storage.height
modulo = remaining_blocks / 1000
if ( modulo == 0 ):
modulo = 1
if (self.storage.height%modulo == 0) or (remaining_blocks < 1000):
msg = "block %d (%d %.2fs) %s" %(self.storage.height, num_tx, delta, self.storage.get_root_hash().encode('hex'))
msg += " (%.2ftx/s, %.2fs/block)" % (tx_per_second, seconds_per_block)
run_blocks = self.storage.height - self.start_catchup_height
remaining_blocks = self.bitcoind_height - self.storage.height
if run_blocks>0 and remaining_blocks>0:
remaining_minutes = remaining_blocks * seconds_per_block / 60
new_blocks = int(remaining_minutes / 10) # number of new blocks expected during catchup
blocks_to_process = remaining_blocks + new_blocks
minutes = blocks_to_process * seconds_per_block / 60
rt = "%.0fmin"%minutes if minutes < 300 else "%.1f hours"%(minutes/60)
msg += " (eta %s, %d blocks)" % (rt, remaining_blocks)
print_log(msg)
def bitcoind(self, method, params=[]):
postdata = dumps({"method": method, 'params': params, 'id': 'jsonrpc'})
while True:
try:
if self.bitcoind_rpc_ssl:
parsedurl = urlparse.urlparse(self.bitcoind_url)
conn = httplib.HTTPSConnection(parsedurl.hostname, parsedurl.port, None, None, False)
conn.request('POST', parsedurl.path, postdata, {
'Host': parsedurl.hostname, 'User-Agent': 'electrum-server',
'Authorization': 'Basic {}'.format( b64encode('{}:{}'.format(parsedurl.username, parsedurl.password)) ),
'Content-type': 'application/json'
})
resp = conn.getresponse()
respdata = resp.read().decode('utf8')
else:
respdata = urllib.urlopen(self.bitcoind_url, postdata).read()
except:
print_log("cannot reach bitcoind...")
self.wait_on_bitcoind()
else:
r = loads(respdata)
if r['error'] is not None:
if r['error'].get('code') == -28:
print_log("bitcoind still warming up...")
self.wait_on_bitcoind()
continue
raise BaseException(r['error'])
break
return r.get('result')
def init_headers(self, db_height):
self.chunk_cache = {}
self.headers_filename = os.path.join( self.dbpath, 'blockchain_headers')
height = 0
if os.path.exists(self.headers_filename):
height = os.path.getsize(self.headers_filename)/80
if height:
prev_header = self.read_header(height -1)
prev_hash = self.hash_header(prev_header)
else:
open(self.headers_filename,'wb').close()
prev_hash = None
if height != db_height:
print_log( "catching up missing headers:", height, db_height)
s = ''
try:
for i in range(height, db_height):
header = self.get_header(i)
assert prev_hash == header.get('prev_block_hash')
self.write_header(header, sync=False)
prev_hash = self.hash_header(header)
if i%1000==0: print_log("headers file:",i)
except KeyboardInterrupt:
self.flush_headers()
sys.exit()
self.flush_headers()
def catch_up(self, sync=True):
self.start_catchup_height = self.storage.height
prev_root_hash = None
n = 0
while not self.shared.stopped():
# are we done yet?
info = self.bitcoind('getinfo')
self.relayfee = info.get('relayfee')
self.bitcoind_height = info.get('blocks')
bitcoind_block_hash = self.bitcoind('getblockhash', (self.bitcoind_height,))
if self.storage.last_hash == bitcoind_block_hash:
self.up_to_date = True
break
self.set_time()
revert = (random.randint(1, 100) == 1) if self.test_reorgs and self.storage.height>100 else False
# not done..
self.up_to_date = False
try:
next_block_hash = self.bitcoind('getblockhash', (self.storage.height + 1,))
except BaseException, e:
revert = True
next_block = self.get_block(next_block_hash if not revert else self.storage.last_hash)
if (next_block.get('previousblockhash') == self.storage.last_hash) and not revert:
prev_root_hash = self.storage.get_root_hash()
n = self.import_block(next_block, next_block_hash, self.storage.height+1)
self.storage.height = self.storage.height + 1
self.write_header(self.block2header(next_block), sync)
self.storage.last_hash = next_block_hash
else:
# revert current block
block = self.get_block(self.storage.last_hash)
print_log("blockchain reorg", self.storage.height, block.get('previousblockhash'), self.storage.last_hash)
n = self.import_block(block, self.storage.last_hash, self.storage.height, revert=True)
self.pop_header()
self.flush_headers()
self.storage.height -= 1
# read previous header from disk
self.header = self.read_header(self.storage.height)
self.storage.last_hash = self.hash_header(self.header)
if prev_root_hash:
assert prev_root_hash == self.storage.get_root_hash()
prev_root_hash = None
# print time
self.print_time(n)
def __init__(self, config, shared):
Processor.__init__(self)
# monitoring
self.avg_time = 0,0,0
self.time_ref = time.time()
self.shared = shared
self.config = config
self.up_to_date = False
self.watch_lock = threading.Lock()
self.watch_blocks = []
self.watch_headers = []
self.watched_addresses = {}
self.history_cache = {}
self.merkle_cache = {}
self.max_cache_size = 100000
self.chunk_cache = {}
self.cache_lock = threading.Lock()
self.headers_data = ''
self.headers_path = config.get('leveldb', 'path')
self.mempool_fees = {}
self.mempool_values = {}
self.mempool_addresses = {}
self.mempool_hist = {} # addr -> (txid, delta)
self.mempool_unconfirmed = {} # txid -> set of unconfirmed inputs
self.mempool_hashes = set()
self.mempool_lock = threading.Lock()
self.address_queue = Queue()
try:
self.test_reorgs = config.getboolean('leveldb', 'test_reorgs') # simulate random blockchain reorgs
except:
self.test_reorgs = False
self.storage = Storage(config, shared, self.test_reorgs)
self.bitcoind_url = 'http://%s:%s@%s:%s/' % (
config.get('bitcoind', 'bitcoind_user'),
config.get('bitcoind', 'bitcoind_password'),
config.get('bitcoind', 'bitcoind_host'),
config.get('bitcoind', 'bitcoind_port'))
self.sent_height = 0
self.sent_header = None
# catch_up headers
self.init_headers(self.storage.height)
# start catch_up thread
if config.getboolean('leveldb', 'profiler'):
filename = os.path.join(config.get('leveldb', 'path'), 'profile')
print_log('profiled thread', filename)
self.blockchain_thread = ProfiledThread(filename, target = self.do_catch_up)
else:
self.blockchain_thread = threading.Thread(target = self.do_catch_up)
self.blockchain_thread.start()
def invalidate_cache(self, address):
with self.cache_lock:
if address in self.history_cache:
print_log("cache: invalidating", address)
self.history_cache.pop(address)
if address in self.watched_addresses:
self.address_queue.put(address)
def invalidate_cache(self, address):
with self.cache_lock:
if address in self.history_cache:
print_log("cache: invalidating", address)
self.history_cache.pop(address)
if address in self.watched_addresses:
# TODO: update cache here. if new value equals cached value, do not send notification
self.address_queue.put(address)
def get_mempool_transaction(self, txid):
try:
raw_tx = self.bitcoind('getrawtransaction', (txid, 0))
except:
return None
vds = deserialize.BCDataStream()
vds.write(raw_tx.decode('hex'))
try:
return deserialize.parse_Transaction(vds, is_coinbase=False)
except:
print_log("ERROR: cannot parse", txid)
return None
def __init__(self, config, shared):
Processor.__init__(self)
self.store = AbeStore(config)
self.watched_addresses = []
self.shared = shared
# catch_up first
self.block_header, time_catch_up, time_mempool, n = self.store.main_iteration()
self.block_number = self.block_header.get('block_height')
print_log("blockchain: %d blocks" % self.block_number)
threading.Timer(10, self.run_store_iteration).start()
def bitcoind(self, method, params=[]):
postdata = dumps({"method": method, 'params': params, 'id': 'jsonrpc'})
try:
respdata = urllib.urlopen(self.bitcoind_url, postdata).read()
except:
print_log("error calling vertcoind")
traceback.print_exc(file=sys.stdout)
self.shared.stop()
r = loads(respdata)
if r['error'] is not None:
raise BaseException(r['error'])
return r.get('result')
def bitcoind(self, method, params=[]):
postdata = dumps({"method": method, "params": params, "id": "jsonrpc"})
try:
respdata = urllib.urlopen(self.bitcoind_url, postdata).read()
except:
print_log("error calling darkcoind")
traceback.print_exc(file=sys.stdout)
self.shared.stop()
r = loads(respdata)
if r["error"] is not None:
raise BaseException(r["error"])
return r.get("result")
def import_block(self, b, chain_ids=frozenset()):
#print_log("import block")
block_id = super(AbeStore, self).import_block(b, chain_ids)
for pos in xrange(len(b['transactions'])):
tx = b['transactions'][pos]
if 'hash' not in tx:
tx['hash'] = Hash(tx['tx'])
tx_id = self.tx_find_id_and_value(tx)
if tx_id:
self.update_tx_cache(tx_id)
else:
print_log("error: import_block: no tx_id")
return block_id
def catch_up(self, sync = True):
t1 = time.time()
while not self.shared.stopped():
# are we done yet?
info = self.bitcoind('getinfo')
self.bitcoind_height = info.get('blocks')
bitcoind_block_hash = self.bitcoind('getblockhash', [self.bitcoind_height])
if self.last_hash == bitcoind_block_hash:
self.up_to_date = True
break
# not done..
self.up_to_date = False
next_block_hash = self.bitcoind('getblockhash', [self.height+1])
next_block = self.bitcoind('getblock', [next_block_hash, 1])
# fixme: this is unsafe, if we revert when the undo info is not yet written
revert = (random.randint(1, 100)==1) if self.is_test else False
if (next_block.get('previousblockhash') == self.last_hash) and not revert:
self.import_block(next_block, next_block_hash, self.height+1, sync)
self.height = self.height + 1
self.write_header(self.block2header(next_block), sync)
self.last_hash = next_block_hash
if (self.height)%100 == 0 and not sync:
t2 = time.time()
print_log( "catch_up: block %d (%.3fs)"%( self.height, t2 - t1 ) )
t1 = t2
else:
# revert current block
block = self.bitcoind('getblock', [self.last_hash, 1])
print_log( "blockchain reorg", self.height, block.get('previousblockhash'), self.last_hash )
self.import_block(block, self.last_hash, self.height, sync, revert=True)
self.pop_header()
self.flush_headers()
self.height = self.height -1
# read previous header from disk
self.header = self.read_header(self.height)
self.last_hash = self.hash_header(self.header)
self.header = self.block2header(self.bitcoind('getblock', [self.last_hash]))
def run(self):
if self.use_ssl:
print_log( "TCP/SSL server started.")
else:
print_log( "TCP server started.")
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
sock.bind((self.host, self.port))
sock.listen(1)
while not self.shared.stopped():
session = TcpSession(*sock.accept(), use_ssl=self.use_ssl, ssl_certfile=self.ssl_certfile, ssl_keyfile=self.ssl_keyfile)
self.dispatcher.add_session(session)
self.dispatcher.collect_garbage()
client_req = TcpClientRequestor(self.dispatcher, session)
client_req.start()
def main_iteration(self):
if self.shared.stopped():
print_log("Stopping timer")
return
with self.dblock:
t1 = time.time()
self.catch_up()
t2 = time.time()
self.memorypool_update()
if self.sent_height != self.storage.height:
self.sent_height = self.storage.height
for session in self.watch_blocks:
self.push_response(session, {
'id': None,
'method': 'blockchain.numblocks.subscribe',
'params': [self.storage.height],
})
if self.sent_header != self.header:
print_log("blockchain: %d (%.3fs)" % (self.storage.height, t2 - t1))
self.sent_header = self.header
for session in self.watch_headers:
self.push_response(session, {
'id': None,
'method': 'blockchain.headers.subscribe',
'params': [self.header],
})
while True:
try:
addr, sessions = self.address_queue.get(False)
except:
break
status = self.get_status(addr)
for session in sessions:
self.push_response(session, {
'id': None,
'method': 'blockchain.address.subscribe',
'params': [addr, status],
})
# next iteration
self.timer = threading.Timer(10, self.main_iteration)
self.timer.start()
def catch_up(self, sync = True):
t1 = time.time()
while not self.shared.stopped():
# are we done yet?
info = self.bitcoind('getinfo')
bitcoind_height = info.get('blocks')
bitcoind_block_hash = self.bitcoind('getblockhash', [bitcoind_height])
if self.last_hash() == bitcoind_block_hash:
self.up_to_date = True
break
# not done..
self.up_to_date = False
block_hash = self.bitcoind('getblockhash', [self.height+1])
block = self.bitcoind('getblock', [block_hash, 1])
if block.get('previousblockhash') == self.last_hash():
self.import_block(block, block_hash, self.height+1, sync)
self.height = self.height + 1
self.write_header(self.block2header(block), sync)
self.block_hashes.append(block_hash)
self.block_hashes = self.block_hashes[-10:]
if (self.height+1)%100 == 0 and not sync:
t2 = time.time()
print_log( "catch_up: block %d (%.3fs)"%( self.height, t2 - t1 ) )
t1 = t2
else:
# revert current block
print_log( "bc2: reorg", self.height, block.get('previousblockhash'), self.last_hash() )
block_hash = self.last_hash()
block = self.bitcoind('getblock', [block_hash, 1])
self.height = self.height -1
self.pop_header()
self.block_hashes.remove(block_hash)
self.import_block(block, self.last_hash(), self.height, revert=True)
self.header = self.block2header(self.bitcoind('getblock', [self.last_hash()]))
def get_chunk(self, index):
with self.cache_lock:
msg = self.chunk_cache.get(index)
if msg:
return msg
sql = """
SELECT
block_hash,
block_version,
block_hashMerkleRoot,
block_nTime,
block_nBits,
block_nNonce,
block_height,
prev_block_hash,
block_height
FROM chain_summary
WHERE block_height >= %d AND block_height< %d AND in_longest = 1 ORDER BY block_height""" % (index * 2016, (index+1) * 2016)
out = self.safe_sql(sql)
msg = ''
for row in out:
(block_hash, block_version, hashMerkleRoot, nTime, nBits, nNonce, height, prev_block_hash, block_height) \
= (self.hashout_hex(row[0]), int(row[1]), self.hashout_hex(row[2]), int(row[3]), int(row[4]), int(row[5]), int(row[6]), self.hashout_hex(row[7]), int(row[8]))
h = {
"block_height": block_height,
"version": block_version,
"prev_block_hash": prev_block_hash,
"merkle_root": hashMerkleRoot,
"timestamp": nTime,
"bits": nBits,
"nonce": nNonce,
}
if h.get('block_height') == 0:
h['prev_block_hash'] = "0" * 64
msg += header_to_string(h)
#print_log("hash", encode(Hash(msg.decode('hex'))))
#if h.get('block_height')==1:break
with self.cache_lock:
self.chunk_cache[index] = msg
print_log("get_chunk", index, len(msg))
return msg
def run(self):
# see http://code.google.com/p/jsonrpclib/
from SocketServer import ThreadingMixIn
if self.use_ssl:
class StratumThreadedServer(ThreadingMixIn, StratumHTTPSSLServer): pass
self.server = StratumThreadedServer(( self.host, self.port), self.certfile, self.keyfile)
print_log( "HTTPS server started.")
else:
class StratumThreadedServer(ThreadingMixIn, StratumHTTPServer): pass
self.server = StratumThreadedServer(( self.host, self.port))
print_log( "HTTP server started.")
self.server.dispatcher = self.dispatcher
self.server.register_function(None, 'server.stop')
self.server.register_function(None, 'server.info')
self.server.serve_forever()
def delete_address(self, leaf):
path = self.get_path(leaf)
if path is False:
print_log("addr not in tree", leaf.encode('hex'),
self.key_to_address(leaf[0:20]), self.db_utxo.get(leaf))
raise
s = self.db_utxo.get(leaf)
self.db_utxo.delete(leaf)
if leaf in self.hash_list:
self.hash_list.pop(leaf)
parent = path[-1]
letter = leaf[len(parent)]
items = self.get_node(parent)
items.pop(letter)
# remove key if it has a single child
if len(items) == 1:
letter, v = items.items()[0]
self.db_utxo.delete(parent)
if parent in self.hash_list:
self.hash_list.pop(parent)
# we need the exact length for the iteration
i = self.db_utxo.iterator()
i.seek(parent + letter)
k, v = i.next()
# note: k is not necessarily a leaf
if len(k) == KEYLENGTH:
_hash, value = k[20:52], hex_to_int(v[0:8])
else:
_hash, value = None, None
self.update_node_hash(k, path[:-1], _hash, value)
else:
self.put_node(parent, items)
_hash, value = None, None
self.update_node_hash(parent, path[:-1], _hash, value)
return s
def deserialize_block(self, block):
txlist = block.get('tx')
tx_hashes = [] # ordered txids
txdict = {} # deserialized tx
for i, raw_tx in enumerate(txlist):
tx_hash = hash_encode(Hash(raw_tx.decode('hex')))
vds = deserialize.BCDataStream()
vds.write(raw_tx.decode('hex'))
try:
tx = deserialize.parse_Transaction(vds, i == 0) # first transaction is always coinbase
except:
print_log("ERROR: cannot parse", tx_hash)
continue
tx_hashes.append(tx_hash)
txdict[tx_hash] = tx
return tx_hashes, txdict
def bitcoind(self, method, params=[]):
postdata = dumps({"method": method, 'params': params, 'id': 'jsonrpc'})
while True:
try:
respdata = urllib.urlopen(self.bitcoind_url, postdata).read()
except:
print_log("cannot reach groestlcoind...")
self.wait_on_bitcoind()
else:
r = loads(respdata)
if r['error'] is not None:
if r['error'].get('code') == -28:
print_log("groestlcoind still warming up...")
self.wait_on_bitcoind()
continue
raise BaseException(r['error'])
break
return r.get('result')
def deserialize_block(block):
txlist = block.get('tx')
tx_hashes = [] # ordered txids
txdict = {} # deserialized tx
is_coinbase = True
for raw_tx in txlist:
tx_hash = hash_encode(Hash(raw_tx.decode('hex')))
vds = deserialize.BCDataStream()
vds.write(raw_tx.decode('hex'))
try:
tx = deserialize.parse_Transaction(vds, is_coinbase)
except:
print_log("ERROR: cannot parse", tx_hash)
continue
tx_hashes.append(tx_hash)
txdict[tx_hash] = tx
is_coinbase = False
return tx_hashes, txdict
def print_mtime(self):
s = ''
for k, v in self.mtimes.items():
s += k + ':' + "%.2f" % v + ' '
print_log(s)
def close(self):
self.blockchain_thread.join()
print_log("Closing database...")
self.storage.close()
print_log("Database is closed")
def close(self):
self.timer.join()
print_log("Closing database...")
self.storage.close()
print_log("Database is closed")
def __init__(self, config, shared):
Processor.__init__(self)
# monitoring
self.avg_time = 0, 0, 0
self.time_ref = time.time()
self.shared = shared
self.config = config
self.up_to_date = False
self.watch_lock = threading.Lock()
self.watch_blocks = []
self.watch_headers = []
self.watched_addresses = {}
self.history_cache = {}
self.merkle_cache = {}
self.max_cache_size = 100000
self.chunk_cache = {}
self.cache_lock = threading.Lock()
self.headers_data = ''
self.headers_path = config.get('leveldb', 'path')
self.mempool_fees = {}
self.mempool_values = {}
self.mempool_addresses = {}
self.mempool_hist = {} # addr -> (txid, delta)
self.mempool_unconfirmed = {} # txid -> set of unconfirmed inputs
self.mempool_hashes = set()
self.mempool_lock = threading.Lock()
self.address_queue = Queue()
try:
self.test_reorgs = config.getboolean(
'leveldb', 'test_reorgs') # simulate random blockchain reorgs
except:
self.test_reorgs = False
self.storage = Storage(config, shared, self.test_reorgs)
self.bitcoind_url = 'http://%s:%s@%s:%s/' % (
config.get('bitcoind', 'bitcoind_user'),
config.get('bitcoind', 'bitcoind_password'),
config.get('bitcoind', 'bitcoind_host'),
config.get('bitcoind', 'bitcoind_port'))
self.sent_height = 0
self.sent_header = None
# catch_up headers
self.init_headers(self.storage.height)
# start catch_up thread
if config.getboolean('leveldb', 'profiler'):
filename = os.path.join(config.get('leveldb', 'path'), 'profile')
print_log('profiled thread', filename)
self.blockchain_thread = ProfiledThread(filename,
target=self.do_catch_up)
else:
self.blockchain_thread = threading.Thread(target=self.do_catch_up)
self.blockchain_thread.start()
def catch_up(self, sync=True):
self.start_catchup_height = self.storage.height
prev_root_hash = None
n = 0
while not self.shared.stopped():
# are we done yet?
info = self.bitcoind('getinfo')
self.relayfee = info.get('relayfee')
self.bitcoind_height = info.get('blocks')
bitcoind_block_hash = self.bitcoind('getblockhash',
(self.bitcoind_height, ))
if self.storage.last_hash == bitcoind_block_hash:
self.up_to_date = True
break
self.set_time()
revert = (
random.randint(1, 100) == 1
) if self.test_reorgs and self.storage.height > 100 else False
# not done..
self.up_to_date = False
try:
next_block_hash = self.bitcoind('getblockhash',
(self.storage.height + 1, ))
except BaseException, e:
revert = True
next_block = self.get_block(
next_block_hash if not revert else self.storage.last_hash)
if (next_block.get('previousblockhash')
== self.storage.last_hash) and not revert:
prev_root_hash = self.storage.get_root_hash()
n = self.import_block(next_block, next_block_hash,
self.storage.height + 1)
self.storage.height = self.storage.height + 1
self.write_header(self.block2header(next_block), sync)
self.storage.last_hash = next_block_hash
else:
# revert current block
block = self.get_block(self.storage.last_hash)
print_log("blockchain reorg", self.storage.height,
block.get('previousblockhash'),
self.storage.last_hash)
n = self.import_block(block,
self.storage.last_hash,
self.storage.height,
revert=True)
self.pop_header()
self.flush_headers()
self.storage.height -= 1
# read previous header from disk
self.header = self.read_header(self.storage.height)
self.storage.last_hash = self.hash_header(self.header)
if prev_root_hash:
assert prev_root_hash == self.storage.get_root_hash()
prev_root_hash = None
# print time
self.print_time(n)
self.storage.height -= 1
# read previous header from disk
self.header = self.read_header(self.storage.height)
self.storage.last_hash = self.hash_header(self.header)
if prev_root_hash:
assert prev_root_hash == self.storage.get_root_hash()
prev_root_hash = None
self.header = self.block2header(
self.bitcoind('getblock', [self.storage.last_hash]))
self.header['utxo_root'] = self.storage.get_root_hash().encode('hex')
if self.shared.stopped():
print_log("closing database")
self.storage.close()
def memorypool_update(self):
mempool_hashes = set(self.bitcoind('getrawmempool'))
touched_addresses = set([])
# get new transactions
new_tx = {}
for tx_hash in mempool_hashes:
if tx_hash in self.mempool_hashes:
continue
tx = self.get_mempool_transaction(tx_hash)
if not tx:
continue
class BlockchainProcessor(Processor):
def __init__(self, config, shared):
Processor.__init__(self)
self.mtimes = {} # monitoring
self.shared = shared
self.config = config
self.up_to_date = False
self.watch_lock = threading.Lock()
self.watch_blocks = []
self.watch_headers = []
self.watched_addresses = {}
self.history_cache = {}
self.chunk_cache = {}
self.cache_lock = threading.Lock()
self.headers_data = ''
self.headers_path = config.get('leveldb', 'path_fulltree')
self.mempool_values = {}
self.mempool_addresses = {}
self.mempool_hist = {}
self.mempool_hashes = set([])
self.mempool_lock = threading.Lock()
self.address_queue = Queue()
try:
self.test_reorgs = config.getboolean(
'leveldb', 'test_reorgs') # simulate random blockchain reorgs
except:
self.test_reorgs = False
self.storage = Storage(config, shared, self.test_reorgs)
self.dblock = threading.Lock()
self.bitcoind_url = 'http://%s:%s@%s:%s/' % (
config.get('bitcoind', 'user'), config.get('bitcoind', 'password'),
config.get('bitcoind', 'host'), config.get('bitcoind', 'port'))
while True:
try:
self.bitcoind('getinfo')
break
except:
print_log('cannot contact litecoind...')
time.sleep(5)
continue
self.sent_height = 0
self.sent_header = None
# catch_up headers
self.init_headers(self.storage.height)
threading.Timer(0, lambda: self.catch_up(sync=False)).start()
while not shared.stopped() and not self.up_to_date:
try:
time.sleep(1)
except:
print "keyboard interrupt: stopping threads"
shared.stop()
sys.exit(0)
print_log("Blockchain is up to date.")
self.memorypool_update()
print_log("Memory pool initialized.")
self.timer = threading.Timer(10, self.main_iteration)
self.timer.start()
def mtime(self, name):
now = time.time()
if name != '':
delta = now - self.now
t = self.mtimes.get(name, 0)
self.mtimes[name] = t + delta
self.now = now
def print_mtime(self):
s = ''
for k, v in self.mtimes.items():
s += k + ':' + "%.2f" % v + ' '
print_log(s)
def bitcoind(self, method, params=[]):
postdata = dumps({"method": method, 'params': params, 'id': 'jsonrpc'})
try:
respdata = urllib.urlopen(self.bitcoind_url, postdata).read()
except:
print_log("error calling litecoind")
traceback.print_exc(file=sys.stdout)
self.shared.stop()
r = loads(respdata)
if r['error'] is not None:
raise BaseException(r['error'])
return r.get('result')
def block2header(self, b):
return {
"block_height": b.get('height'),
"version": b.get('version'),
"prev_block_hash": b.get('previousblockhash'),
"merkle_root": b.get('merkleroot'),
"timestamp": b.get('time'),
"bits": int(b.get('bits'), 16),
"nonce": b.get('nonce'),
}
def get_header(self, height):
block_hash = self.bitcoind('getblockhash', [height])
b = self.bitcoind('getblock', [block_hash])
return self.block2header(b)
def init_headers(self, db_height):
self.chunk_cache = {}
self.headers_filename = os.path.join(self.headers_path,
'blockchain_headers')
if os.path.exists(self.headers_filename):
height = os.path.getsize(
self.headers_filename) / 80 - 1 # the current height
if height > 0:
prev_hash = self.hash_header(self.read_header(height))
else:
prev_hash = None
else:
open(self.headers_filename, 'wb').close()
prev_hash = None
height = -1
if height < db_height:
print_log("catching up missing headers:", height, db_height)
try:
while height < db_height:
height = height + 1
header = self.get_header(height)
if height > 1:
assert prev_hash == header.get('prev_block_hash')
self.write_header(header, sync=False)
prev_hash = self.hash_header(header)
if (height % 1000) == 0:
print_log("headers file:", height)
except KeyboardInterrupt:
self.flush_headers()
sys.exit()
self.flush_headers()
def hash_header(self, header):
return rev_hex(
Hash(header_to_string(header).decode('hex')).encode('hex'))
def read_header(self, block_height):
if os.path.exists(self.headers_filename):
with open(self.headers_filename, 'rb') as f:
f.seek(block_height * 80)
h = f.read(80)
if len(h) == 80:
h = header_from_string(h)
return h
def read_chunk(self, index):
with open(self.headers_filename, 'rb') as f:
f.seek(index * 2016 * 80)
chunk = f.read(2016 * 80)
return chunk.encode('hex')
def write_header(self, header, sync=True):
if not self.headers_data:
self.headers_offset = header.get('block_height')
self.headers_data += header_to_string(header).decode('hex')
if sync or len(self.headers_data) > 40 * 100:
self.flush_headers()
with self.cache_lock:
chunk_index = header.get('block_height') / 2016
if self.chunk_cache.get(chunk_index):
self.chunk_cache.pop(chunk_index)
def pop_header(self):
# we need to do this only if we have not flushed
if self.headers_data:
self.headers_data = self.headers_data[:-40]
def flush_headers(self):
if not self.headers_data:
return
with open(self.headers_filename, 'rb+') as f:
f.seek(self.headers_offset * 80)
f.write(self.headers_data)
self.headers_data = ''
def get_chunk(self, i):
# store them on disk; store the current chunk in memory
with self.cache_lock:
chunk = self.chunk_cache.get(i)
if not chunk:
chunk = self.read_chunk(i)
self.chunk_cache[i] = chunk
return chunk
def get_mempool_transaction(self, txid):
try:
raw_tx = self.bitcoind('getrawtransaction', [txid, 0])
except:
return None
vds = deserialize.BCDataStream()
vds.write(raw_tx.decode('hex'))
try:
return deserialize.parse_Transaction(vds, is_coinbase=False)
except:
print_log("ERROR: cannot parse", txid)
return None
def get_history(self, addr, cache_only=False):
with self.cache_lock:
hist = self.history_cache.get(addr)
if hist is not None:
return hist
if cache_only:
return -1
with self.dblock:
try:
hist = self.storage.get_history(addr)
is_known = True
except:
print_log("error get_history")
traceback.print_exc(file=sys.stdout)
raise
if hist:
is_known = True
else:
hist = []
is_known = False
# add memory pool
with self.mempool_lock:
for txid, delta in self.mempool_hist.get(addr, []):
hist.append({'tx_hash': txid, 'height': 0})
# add something to distinguish between unused and empty addresses
if hist == [] and is_known:
hist = ['*']
with self.cache_lock:
self.history_cache[addr] = hist
return hist
def get_unconfirmed_value(self, addr):
v = 0
with self.mempool_lock:
for txid, delta in self.mempool_hist.get(addr, []):
v += delta
return v
def get_status(self, addr, cache_only=False):
tx_points = self.get_history(addr, cache_only)
if cache_only and tx_points == -1:
return -1
if not tx_points:
return None
if tx_points == ['*']:
return '*'
status = ''
for tx in tx_points:
status += tx.get('tx_hash') + ':%d:' % tx.get('height')
return hashlib.sha256(status).digest().encode('hex')
def get_merkle(self, tx_hash, height):
block_hash = self.bitcoind('getblockhash', [height])
b = self.bitcoind('getblock', [block_hash])
tx_list = b.get('tx')
tx_pos = tx_list.index(tx_hash)
merkle = map(hash_decode, tx_list)
target_hash = hash_decode(tx_hash)
s = []
while len(merkle) != 1:
if len(merkle) % 2:
merkle.append(merkle[-1])
n = []
while merkle:
new_hash = Hash(merkle[0] + merkle[1])
if merkle[0] == target_hash:
s.append(hash_encode(merkle[1]))
target_hash = new_hash
elif merkle[1] == target_hash:
s.append(hash_encode(merkle[0]))
target_hash = new_hash
n.append(new_hash)
merkle = merkle[2:]
merkle = n
return {"block_height": height, "merkle": s, "pos": tx_pos}
def add_to_history(self, addr, tx_hash, tx_pos, tx_height):
# keep it sorted
s = self.serialize_item(tx_hash, tx_pos, tx_height) + 40 * chr(0)
assert len(s) == 80
serialized_hist = self.batch_list[addr]
l = len(serialized_hist) / 80
for i in range(l - 1, -1, -1):
item = serialized_hist[80 * i:80 * (i + 1)]
item_height = int(rev_hex(item[36:39].encode('hex')), 16)
if item_height <= tx_height:
serialized_hist = serialized_hist[0:80 * (
i + 1)] + s + serialized_hist[80 * (i + 1):]
break
else:
serialized_hist = s + serialized_hist
self.batch_list[addr] = serialized_hist
# backlink
txo = (tx_hash + int_to_hex(tx_pos, 4)).decode('hex')
self.batch_txio[txo] = addr
def deserialize_block(self, block):
txlist = block.get('tx')
tx_hashes = [] # ordered txids
txdict = {} # deserialized tx
is_coinbase = True
for raw_tx in txlist:
tx_hash = hash_encode(Hash(raw_tx.decode('hex')))
vds = deserialize.BCDataStream()
vds.write(raw_tx.decode('hex'))
try:
tx = deserialize.parse_Transaction(vds, is_coinbase)
except:
print_log("ERROR: cannot parse", tx_hash)
continue
tx_hashes.append(tx_hash)
txdict[tx_hash] = tx
is_coinbase = False
return tx_hashes, txdict
def import_block(self,
block,
block_hash,
block_height,
sync,
revert=False):
touched_addr = set([])
# deserialize transactions
tx_hashes, txdict = self.deserialize_block(block)
# undo info
if revert:
undo_info = self.storage.get_undo_info(block_height)
tx_hashes.reverse()
else:
undo_info = {}
for txid in tx_hashes: # must be ordered
tx = txdict[txid]
if not revert:
undo = self.storage.import_transaction(txid, tx, block_height,
touched_addr)
undo_info[txid] = undo
else:
undo = undo_info.pop(txid)
self.storage.revert_transaction(txid, tx, block_height,
touched_addr, undo)
if revert:
assert undo_info == {}
# add undo info
if not revert:
self.storage.write_undo_info(block_height, self.bitcoind_height,
undo_info)
# add the max
self.storage.db_undo.put(
'height', repr(
(block_hash, block_height, self.storage.db_version)))
for addr in touched_addr:
self.invalidate_cache(addr)
self.storage.update_hashes()
def add_request(self, session, request):
# see if we can get if from cache. if not, add to queue
if self.process(session, request, cache_only=True) == -1:
self.queue.put((session, request))
def do_subscribe(self, method, params, session):
with self.watch_lock:
if method == 'blockchain.numblocks.subscribe':
if session not in self.watch_blocks:
self.watch_blocks.append(session)
elif method == 'blockchain.headers.subscribe':
if session not in self.watch_headers:
self.watch_headers.append(session)
elif method == 'blockchain.address.subscribe':
address = params[0]
l = self.watched_addresses.get(address)
if l is None:
self.watched_addresses[address] = [session]
elif session not in l:
l.append(session)
def do_unsubscribe(self, method, params, session):
with self.watch_lock:
if method == 'blockchain.numblocks.subscribe':
if session in self.watch_blocks:
self.watch_blocks.remove(session)
elif method == 'blockchain.headers.subscribe':
if session in self.watch_headers:
self.watch_headers.remove(session)
elif method == "blockchain.address.subscribe":
addr = params[0]
l = self.watched_addresses.get(addr)
if not l:
return
if session in l:
l.remove(session)
if session in l:
print_log("error rc!!")
self.shared.stop()
if l == []:
self.watched_addresses.pop(addr)
def process(self, session, request, cache_only=False):
message_id = request['id']
method = request['method']
params = request.get('params', [])
result = None
error = None
if method == 'blockchain.numblocks.subscribe':
result = self.storage.height
elif method == 'blockchain.headers.subscribe':
result = self.header
elif method == 'blockchain.address.subscribe':
try:
address = str(params[0])
result = self.get_status(address, cache_only)
except BaseException, e:
error = str(e) + ': ' + address
print_log("error:", error)
elif method == 'blockchain.address.get_history':
try:
address = str(params[0])
result = self.get_history(address, cache_only)
except BaseException, e:
error = str(e) + ': ' + address
print_log("error:", error)
def memorypool_update(self):
t0 = time.time()
mempool_hashes = set(self.bitcoind('getrawmempool'))
touched_addresses = set()
# get new transactions
new_tx = {}
for tx_hash in mempool_hashes:
if tx_hash in self.mempool_hashes:
continue
tx = self.get_mempool_transaction(tx_hash)
if not tx:
continue
new_tx[tx_hash] = tx
# remove older entries from mempool_hashes
self.mempool_hashes = mempool_hashes
# check all tx outputs
for tx_hash, tx in new_tx.iteritems():
mpa = self.mempool_addresses.get(tx_hash, {})
out_values = []
out_sum = 0
for x in tx.get('outputs'):
addr = x.get('address', '')
value = x['value']
out_values.append((addr, value))
if not addr:
continue
v = mpa.get(addr, 0)
v += value
mpa[addr] = v
touched_addresses.add(addr)
out_sum += value
self.mempool_fees[tx_hash] = -out_sum
self.mempool_addresses[tx_hash] = mpa
self.mempool_values[tx_hash] = out_values
self.mempool_unconfirmed[tx_hash] = set()
# check all inputs
for tx_hash, tx in new_tx.iteritems():
mpa = self.mempool_addresses.get(tx_hash, {})
# are we spending unconfirmed inputs?
input_sum = 0
for x in tx.get('inputs'):
prev_hash = x.get('prevout_hash')
prev_n = x.get('prevout_n')
mpv = self.mempool_values.get(prev_hash)
if mpv:
addr, value = mpv[prev_n]
self.mempool_unconfirmed[tx_hash].add(prev_hash)
else:
txi = (prev_hash + int_to_hex4(prev_n)).decode('hex')
try:
addr = self.storage.get_address(txi)
value = self.storage.get_utxo_value(addr,txi)
except:
print_log("utxo not in database; postponing mempool update")
return
# we can proceed
input_sum += value
if not addr:
continue
v = mpa.get(addr, 0)
v -= value
mpa[addr] = v
touched_addresses.add(addr)
self.mempool_addresses[tx_hash] = mpa
self.mempool_fees[tx_hash] += input_sum
# remove deprecated entries from mempool_addresses
for tx_hash, addresses in self.mempool_addresses.items():
if tx_hash not in self.mempool_hashes:
del self.mempool_addresses[tx_hash]
del self.mempool_values[tx_hash]
del self.mempool_unconfirmed[tx_hash]
del self.mempool_fees[tx_hash]
touched_addresses.update(addresses)
# remove deprecated entries from mempool_hist
new_mempool_hist = {}
for addr in self.mempool_hist.iterkeys():
h = self.mempool_hist[addr]
hh = []
for tx_hash, delta in h:
if tx_hash in self.mempool_addresses:
hh.append((tx_hash, delta))
if hh:
new_mempool_hist[addr] = hh
# add new transactions to mempool_hist
for tx_hash in new_tx.iterkeys():
addresses = self.mempool_addresses[tx_hash]
for addr, delta in addresses.iteritems():
h = new_mempool_hist.get(addr, [])
if (tx_hash, delta) not in h:
h.append((tx_hash, delta))
new_mempool_hist[addr] = h
with self.mempool_lock:
self.mempool_hist = new_mempool_hist
# invalidate cache for touched addresses
for addr in touched_addresses:
self.invalidate_cache(addr)
t1 = time.time()
if t1-t0>1:
print_log('mempool_update', t1-t0, len(self.mempool_hashes), len(self.mempool_hist))
def memorypool_update(self):
mempool_hashes = set(self.bitcoind('getrawmempool'))
touched_addresses = set([])
# get new transactions
new_tx = {}
for tx_hash in mempool_hashes:
if tx_hash in self.mempool_hashes:
continue
tx = self.get_mempool_transaction(tx_hash)
if not tx:
continue
new_tx[tx_hash] = tx
self.mempool_hashes.add(tx_hash)
# remove older entries from mempool_hashes
self.mempool_hashes = mempool_hashes
# check all tx outputs
for tx_hash, tx in new_tx.items():
mpa = self.mempool_addresses.get(tx_hash, {})
out_values = []
for x in tx.get('outputs'):
out_values.append(x['value'])
addr = x.get('address')
if not addr:
continue
v = mpa.get(addr, 0)
v += x['value']
mpa[addr] = v
touched_addresses.add(addr)
self.mempool_addresses[tx_hash] = mpa
self.mempool_values[tx_hash] = out_values
# check all inputs
for tx_hash, tx in new_tx.items():
mpa = self.mempool_addresses.get(tx_hash, {})
for x in tx.get('inputs'):
# we assume that the input address can be parsed by deserialize(); this is true for Electrum transactions
addr = x.get('address')
if not addr:
continue
v = self.mempool_values.get(x.get('prevout_hash'))
if v:
value = v[x.get('prevout_n')]
else:
txi = (x.get('prevout_hash') +
int_to_hex(x.get('prevout_n'), 4)).decode('hex')
try:
value = self.storage.get_utxo_value(addr, txi)
except:
print_log(
"utxo not in database; postponing mempool update")
return
v = mpa.get(addr, 0)
v -= value
mpa[addr] = v
touched_addresses.add(addr)
self.mempool_addresses[tx_hash] = mpa
# remove deprecated entries from mempool_addresses
for tx_hash, addresses in self.mempool_addresses.items():
if tx_hash not in self.mempool_hashes:
self.mempool_addresses.pop(tx_hash)
self.mempool_values.pop(tx_hash)
for addr in addresses:
touched_addresses.add(addr)
# rebuild mempool histories
new_mempool_hist = {}
for tx_hash, addresses in self.mempool_addresses.items():
for addr, delta in addresses.items():
h = new_mempool_hist.get(addr, [])
if tx_hash not in h:
h.append((tx_hash, delta))
new_mempool_hist[addr] = h
with self.mempool_lock:
self.mempool_hist = new_mempool_hist
# invalidate cache for touched addresses
for addr in touched_addresses:
self.invalidate_cache(addr)
elif method == 'blockchain.address.get_history':
try:
address = str(params[0])
result = self.get_history(address, cache_only)
except BaseException, e:
error = str(e) + ': ' + address
print_log("error:", error)
elif method == 'blockchain.address.get_mempool':
try:
address = str(params[0])
result = self.get_unconfirmed_history(address, cache_only)
except BaseException, e:
error = str(e) + ': ' + address
print_log("error:", error)
elif method == 'blockchain.address.get_balance':
try:
address = str(params[0])
confirmed = self.storage.get_balance(address)
unconfirmed = self.get_unconfirmed_value(address)
result = {'confirmed': confirmed, 'unconfirmed': unconfirmed}
except BaseException, e:
error = str(e) + ': ' + address
print_log("error:", error)
elif method == 'blockchain.address.get_proof':
try:
address = str(params[0])
result = self.storage.get_proof(address)
def get_undo_info(self, height):
s = self.db_undo.get("undo_info_%d" % (height % 100))
if s is None: print_log("no undo info for ", height)
return eval(s)
def __init__(self, config, shared, test_reorgs):
self.dbpath = config.get('leveldb', 'path')
if not os.path.exists(self.dbpath):
os.mkdir(self.dbpath)
self.pruning_limit = config.getint('leveldb', 'pruning_limit')
self.shared = shared
self.hash_list = {}
self.parents = {}
self.test_reorgs = test_reorgs
try:
self.db_utxo = plyvel.DB(os.path.join(self.dbpath, 'utxo'),
create_if_missing=True,
compression=None)
self.db_addr = plyvel.DB(os.path.join(self.dbpath, 'addr'),
create_if_missing=True,
compression=None)
self.db_hist = plyvel.DB(os.path.join(self.dbpath, 'hist'),
create_if_missing=True,
compression=None)
self.db_undo = plyvel.DB(os.path.join(self.dbpath, 'undo'),
create_if_missing=True,
compression=None)
except:
logger.error('db init', exc_info=True)
self.shared.stop()
self.db_version = 3 # increase this when database needs to be updated
try:
self.last_hash, self.height, db_version = ast.literal_eval(
self.db_undo.get('height'))
print_log("Database version", self.db_version)
print_log("Blockchain height", self.height)
except:
print_log('initializing database')
self.height = 0
self.last_hash = GENESIS_HASH
db_version = self.db_version
# write root
self.put_node('', {})
# check version
if self.db_version != db_version:
print_log(
"Your database '%s' is deprecated. Please create a new database"
% self.dbpath)
self.shared.stop()
return
# compute root hash
d = self.get_node('')
self.root_hash, v = self.get_node_hash('', d, None)
print_log("UTXO tree root hash:", self.root_hash.encode('hex'))
print_log("Coins in database:", v)
def handler(signum = None, frame = None):
print_log('Signal handler called with signal', signum)
shared.stop()
def memorypool_update(self):
t0 = time.time()
mempool_hashes = set(self.bitcoind('getrawmempool'))
touched_addresses = set([])
# get new transactions
new_tx = {}
for tx_hash in mempool_hashes:
if tx_hash in self.mempool_hashes:
continue
tx = self.get_mempool_transaction(tx_hash)
if not tx:
continue
new_tx[tx_hash] = tx
self.mempool_hashes.add(tx_hash)
# remove older entries from mempool_hashes
self.mempool_hashes = mempool_hashes
# check all tx outputs
for tx_hash, tx in new_tx.items():
mpa = self.mempool_addresses.get(tx_hash, {})
out_values = []
for x in tx.get('outputs'):
addr = x.get('address', '')
out_values.append((addr, x['value']))
if not addr:
continue
v = mpa.get(addr, 0)
v += x['value']
mpa[addr] = v
touched_addresses.add(addr)
self.mempool_addresses[tx_hash] = mpa
self.mempool_values[tx_hash] = out_values
# check all inputs
for tx_hash, tx in new_tx.items():
mpa = self.mempool_addresses.get(tx_hash, {})
for x in tx.get('inputs'):
mpv = self.mempool_values.get(x.get('prevout_hash'))
if mpv:
addr, value = mpv[x.get('prevout_n')]
else:
txi = (x.get('prevout_hash') +
int_to_hex(x.get('prevout_n'), 4)).decode('hex')
try:
addr = self.storage.get_address(txi)
value = self.storage.get_utxo_value(addr, txi)
except:
print_log(
"utxo not in database; postponing mempool update")
return
if not addr:
continue
v = mpa.get(addr, 0)
v -= value
mpa[addr] = v
touched_addresses.add(addr)
self.mempool_addresses[tx_hash] = mpa
# remove deprecated entries from mempool_addresses
for tx_hash, addresses in self.mempool_addresses.items():
if tx_hash not in self.mempool_hashes:
self.mempool_addresses.pop(tx_hash)
self.mempool_values.pop(tx_hash)
for addr in addresses:
touched_addresses.add(addr)
# remove deprecated entries from mempool_hist
new_mempool_hist = {}
for addr in self.mempool_hist.keys():
h = self.mempool_hist[addr]
hh = []
for tx_hash, delta in h:
if tx_hash in self.mempool_addresses:
hh.append((tx_hash, delta))
if hh:
new_mempool_hist[addr] = hh
# add new transactions to mempool_hist
for tx_hash in new_tx.keys():
addresses = self.mempool_addresses[tx_hash]
for addr, delta in addresses.items():
h = new_mempool_hist.get(addr, [])
if (tx_hash, delta) not in h:
h.append((tx_hash, delta))
new_mempool_hist[addr] = h
with self.mempool_lock:
self.mempool_hist = new_mempool_hist
# invalidate cache for touched addresses
for addr in touched_addresses:
self.invalidate_cache(addr)
t1 = time.time()
if t1 - t0 > 1:
print_log('mempool_update', t1 - t0, len(self.mempool_hashes),
len(self.mempool_hist))
def process(self, request, cache_only=False):
message_id = request['id']
method = request['method']
params = request.get('params', [])
result = None
error = None
if method == 'blockchain.numblocks.subscribe':
result = self.storage.height
elif method == 'blockchain.headers.subscribe':
result = self.header
elif method == 'blockchain.address.subscribe':
address = str(params[0])
result = self.get_status(address, cache_only)
elif method == 'blockchain.address.get_history':
address = str(params[0])
result = self.get_history(address, cache_only)
elif method == 'blockchain.address.get_mempool':
address = str(params[0])
result = self.get_unconfirmed_history(address, cache_only)
elif method == 'blockchain.address.get_balance':
address = str(params[0])
confirmed = self.storage.get_balance(address)
unconfirmed = self.get_unconfirmed_value(address)
result = {'confirmed': confirmed, 'unconfirmed': unconfirmed}
elif method == 'blockchain.address.get_proof':
address = str(params[0])
result = self.storage.get_proof(address)
elif method == 'blockchain.address.listunspent':
address = str(params[0])
result = self.storage.listunspent(address)
elif method == 'blockchain.utxo.get_address':
txid = str(params[0])
pos = int(params[1])
txi = (txid + int_to_hex(pos, 4)).decode('hex')
result = self.storage.get_address(txi)
elif method == 'blockchain.block.get_header':
if cache_only:
result = -1
else:
height = int(params[0])
result = self.get_header(height)
elif method == 'blockchain.block.get_chunk':
if cache_only:
result = -1
else:
index = int(params[0])
result = self.get_chunk(index)
elif method == 'blockchain.transaction.broadcast':
try:
txo = self.bitcoind('sendrawtransaction', params)
print_log("sent tx:", txo)
result = txo
except BaseException, e:
result = str(e) # do not send an error
print_log("error:", result, params)
def __init__(self, config, shared):
Processor.__init__(self)
self.mtimes = {} # monitoring
self.shared = shared
self.config = config
self.up_to_date = False
self.watch_lock = threading.Lock()
self.watch_blocks = []
self.watch_headers = []
self.watched_addresses = {}
self.history_cache = {}
self.chunk_cache = {}
self.cache_lock = threading.Lock()
self.headers_data = ''
self.headers_path = config.get('leveldb', 'path_fulltree')
self.mempool_values = {}
self.mempool_addresses = {}
self.mempool_hist = {}
self.mempool_hashes = set([])
self.mempool_lock = threading.Lock()
self.address_queue = Queue()
try:
self.test_reorgs = config.getboolean(
'leveldb', 'test_reorgs') # simulate random blockchain reorgs
except:
self.test_reorgs = False
self.storage = Storage(config, shared, self.test_reorgs)
self.dblock = threading.Lock()
self.bitcoind_url = 'http://%s:%s@%s:%s/' % (
config.get('bitcoind', 'user'), config.get('bitcoind', 'password'),
config.get('bitcoind', 'host'), config.get('bitcoind', 'port'))
while True:
try:
self.bitcoind('getinfo')
break
except:
print_log('cannot contact litecoind...')
time.sleep(5)
continue
self.sent_height = 0
self.sent_header = None
# catch_up headers
self.init_headers(self.storage.height)
threading.Timer(0, lambda: self.catch_up(sync=False)).start()
while not shared.stopped() and not self.up_to_date:
try:
time.sleep(1)
except:
print "keyboard interrupt: stopping threads"
shared.stop()
sys.exit(0)
print_log("Blockchain is up to date.")
self.memorypool_update()
print_log("Memory pool initialized.")
self.timer = threading.Timer(10, self.main_iteration)
self.timer.start()
def process(self, request, cache_only=False):
message_id = request['id']
method = request['method']
params = request.get('params', ())
result = None
error = None
if method == 'blockchain.numblocks.subscribe':
result = self.storage.height
elif method == 'blockchain.headers.subscribe':
result = self.header
elif method == 'blockchain.address.subscribe':
address = str(params[0])
result = self.get_status(address, cache_only)
elif method == 'blockchain.address.get_history':
address = str(params[0])
result = self.get_history(address, cache_only)
elif method == 'blockchain.address.get_mempool':
address = str(params[0])
result = self.get_unconfirmed_history(address)
elif method == 'blockchain.address.get_balance':
address = str(params[0])
confirmed = self.storage.get_balance(address)
unconfirmed = self.get_unconfirmed_value(address)
result = { 'confirmed':confirmed, 'unconfirmed':unconfirmed }
elif method == 'blockchain.address.get_proof':
address = str(params[0])
result = self.storage.get_proof(address)
elif method == 'blockchain.address.listunspent':
address = str(params[0])
result = self.storage.listunspent(address)
elif method == 'blockchain.utxo.get_address':
txid = str(params[0])
pos = int(params[1])
txi = (txid + int_to_hex4(pos)).decode('hex')
result = self.storage.get_address(txi)
elif method == 'blockchain.block.get_header':
if cache_only:
result = -1
else:
height = int(params[0])
result = self.get_header(height)
elif method == 'blockchain.block.get_chunk':
if cache_only:
result = -1
else:
index = int(params[0])
result = self.get_chunk(index)
elif method == 'blockchain.transaction.broadcast':
try:
txo = self.bitcoind('sendrawtransaction', params)
print_log("sent tx:", txo)
result = txo
except BaseException, e:
error = e.args[0]
if error["code"] == -26:
# If we return anything that's not the transaction hash,
# it's considered an error message
message = error["message"]
if "non-mandatory-script-verify-flag" in message:
result = "Your client produced a transaction that is not accepted by the Litecoin network any more. Please upgrade to Electrum 2.5.1 or newer\n"
else:
result = "The transaction was rejected by network rules.(" + message + ")\n" \
"[" + params[0] + "]"
else:
result = error["message"] # do send an error
print_log("error:", result)
def __init__(self, config, shared, test_reorgs):
self.shared = shared
self.hash_list = {}
self.parents = {}
self.skip_batch = {}
self.test_reorgs = test_reorgs
# init path
self.dbpath = config.get('leveldb', 'path')
if not os.path.exists(self.dbpath):
os.mkdir(self.dbpath)
try:
self.db_utxo = DB(self.dbpath, 'utxo',
config.getint('leveldb', 'utxo_cache'))
self.db_hist = DB(self.dbpath, 'hist',
config.getint('leveldb', 'hist_cache'))
self.db_addr = DB(self.dbpath, 'addr',
config.getint('leveldb', 'addr_cache'))
self.db_undo = DB(self.dbpath, 'undo', None)
except:
logger.error('db init', exc_info=True)
self.shared.stop()
try:
self.last_hash, self.height, db_version = ast.literal_eval(
self.db_undo.get('height'))
except:
print_log('Initializing database')
self.height = 0
self.last_hash = GENESIS_HASH
self.pruning_limit = config.getint('leveldb', 'pruning_limit')
db_version = DB_VERSION
self.put_node('', Node.from_dict({}))
# check version
if db_version != DB_VERSION:
print_log(
"Your database '%s' is deprecated. Please create a new database"
% self.dbpath)
self.shared.stop()
return
# pruning limit
try:
self.pruning_limit = ast.literal_eval(self.db_undo.get('limit'))
except:
self.pruning_limit = config.getint('leveldb', 'pruning_limit')
self.db_undo.put('version', repr(self.pruning_limit))
# reorg limit
try:
self.reorg_limit = ast.literal_eval(
self.db_undo.get('reorg_limit'))
except:
self.reorg_limit = config.getint('leveldb', 'reorg_limit')
self.db_undo.put('reorg_limit', repr(self.reorg_limit))
# compute root hash
root_node = self.get_node('')
self.root_hash, coins = root_node.get_hash('', None)
# print stuff
print_log("Database version %d." % db_version)
print_log("Pruning limit for spent outputs is %d." %
self.pruning_limit)
print_log("Reorg limit is %d blocks." % self.reorg_limit)
print_log("Blockchain height", self.height)
print_log("UTXO tree root hash:", self.root_hash.encode('hex'))
print_log("Coins in database:", coins)
def get_history(self, addr, cache_only=False):
# todo: make this more efficient. it iterates over txpoints multiple times
with self.cache_lock:
cached_version = self.tx_cache.get(addr)
if cached_version is not None:
return cached_version
if cache_only:
return -1
version, binaddr = decode_check_address(addr)
if binaddr is None:
return None
dbhash = self.binin(binaddr)
rows = []
rows += self.get_address_out_rows(dbhash)
rows += self.get_address_in_rows(dbhash)
txpoints = []
known_tx = []
for row in rows:
try:
nTime, chain_id, height, is_in, blk_hash, tx_hash, tx_id, pos, value = row
except:
print_log("cannot unpack row", row)
break
tx_hash = self.hashout_hex(tx_hash)
txpoints.append({
"timestamp": int(nTime),
"height": int(height),
"is_input": int(is_in),
"block_hash": self.hashout_hex(blk_hash),
"tx_hash": tx_hash,
"tx_id": int(tx_id),
"index": int(pos),
"value": int(value),
})
known_tx.append(tx_hash)
# todo: sort them really...
txpoints = sorted(txpoints, key=operator.itemgetter("timestamp"))
# read memory pool
rows = []
rows += self.get_address_in_rows_memorypool(dbhash)
rows += self.get_address_out_rows_memorypool(dbhash)
address_has_mempool = False
for row in rows:
is_in, tx_hash, tx_id, pos, value = row
tx_hash = self.hashout_hex(tx_hash)
if tx_hash in known_tx:
continue
# discard transactions that are too old
if self.last_tx_id - tx_id > 50000:
print_log("discarding tx id", tx_id)
continue
# this means that pending transactions were added to the db, even if they are not returned by getmemorypool
address_has_mempool = True
#print_log("mempool", tx_hash)
txpoints.append({
"timestamp": 0,
"height": 0,
"is_input": int(is_in),
"block_hash": 'mempool',
"tx_hash": tx_hash,
"tx_id": int(tx_id),
"index": int(pos),
"value": int(value),
})
for txpoint in txpoints:
tx_id = txpoint['tx_id']
txinputs = []
inrows = self.get_tx_inputs(tx_id)
for row in inrows:
_hash = self.binout(row[6])
if not _hash:
#print_log("WARNING: missing tx_in for tx", tx_id, addr)
continue
address = hash_to_address(chr(self.addrtype), _hash)
txinputs.append(address)
txpoint['inputs'] = txinputs
txoutputs = []
outrows = self.get_tx_outputs(tx_id)
for row in outrows:
_hash = self.binout(row[6])
if not _hash:
#print_log("WARNING: missing tx_out for tx", tx_id, addr)
continue
address = hash_to_address(chr(self.addrtype), _hash)
txoutputs.append(address)
txpoint['outputs'] = txoutputs
# for all unspent inputs, I want their scriptpubkey. (actually I could deduce it from the address)
if not txpoint['is_input']:
# detect if already redeemed...
for row in outrows:
if row[6] == dbhash:
break
else:
raise
#row = self.get_tx_output(tx_id,dbhash)
# pos, script, value, o_hash, o_id, o_pos, binaddr = row
# if not redeemed, we add the script
if row:
if not row[4]:
txpoint['raw_output_script'] = row[1]
txpoint.pop('tx_id')
txpoints = map(
lambda x: {
'tx_hash': x['tx_hash'],
'height': x['height']
}, txpoints)
out = []
for item in txpoints:
if item not in out:
out.append(item)
# cache result
## do not cache mempool results because statuses are ambiguous
#if not address_has_mempool:
with self.cache_lock:
self.tx_cache[addr] = out
return out
def main_iteration(self):
if self.shared.stopped():
print_log("Stopping timer")
return
with self.dblock:
t1 = time.time()
self.catch_up()
t2 = time.time()
self.memorypool_update()
if self.sent_height != self.storage.height:
self.sent_height = self.storage.height
for session in self.watch_blocks:
self.push_response(
session, {
'id': None,
'method': 'blockchain.numblocks.subscribe',
'params': [self.storage.height],
})
if self.sent_header != self.header:
print_log("blockchain: %d (%.3fs)" %
(self.storage.height, t2 - t1))
self.sent_header = self.header
for session in self.watch_headers:
self.push_response(
session, {
'id': None,
'method': 'blockchain.headers.subscribe',
'params': [self.header],
})
# Update status of masternodes.
masternodes_status = self.get_masternodes_status()
if self.sent_masternodes_status != masternodes_status:
for collateral, status in masternodes_status.items():
# Skip if a masternode didn't change.
if status == self.sent_masternodes_status.get(collateral):
continue
for session in self.watched_masternodes.get(collateral, []):
self.push_response(
session, {
'id': None,
'method': 'masternode.subscribe',
'params': [collateral],
'result': status,
})
self.sent_masternodes_status = masternodes_status
proposals_status = self.get_proposals_status()
if self.sent_proposals_status != proposals_status:
self.sent_proposals_status = proposals_status
for session in self.watch_proposals:
self.push_response(
session, {
'id': None,
'method': 'masternode.proposals.subscribe',
'params': [],
'result': proposals_status,
})
while True:
try:
addr, sessions = self.address_queue.get(False)
except:
break
status = self.get_status(addr)
for session in sessions:
self.push_response(
session, {
'id': None,
'method': 'blockchain.address.subscribe',
'params': [addr, status],
})
def catch_up(self, sync=True):
prev_root_hash = None
while not self.shared.stopped():
self.mtime('')
# are we done yet?
info = self.bitcoind('getinfo')
self.bitcoind_height = info.get('blocks')
bitcoind_block_hash = self.bitcoind('getblockhash',
[self.bitcoind_height])
if self.storage.last_hash == bitcoind_block_hash:
self.up_to_date = True
break
# fixme: this is unsafe, if we revert when the undo info is not yet written
revert = (random.randint(1, 100)
== 1) if self.test_reorgs else False
# not done..
self.up_to_date = False
try:
next_block_hash = self.bitcoind('getblockhash',
[self.storage.height + 1])
next_block = self.getfullblock(next_block_hash)
except BaseException, e:
revert = True
next_block = self.getfullblock(self.storage.last_hash)
self.mtime('daemon')
if not revert and (next_block.get('previousblockhash')
== self.storage.last_hash):
prev_root_hash = self.storage.get_root_hash()
self.import_block(next_block, next_block_hash,
self.storage.height + 1, sync)
self.storage.height = self.storage.height + 1
self.write_header(self.block2header(next_block), sync)
self.storage.last_hash = next_block_hash
self.mtime('import')
if self.storage.height % 1000 == 0 and not sync:
t_daemon = self.mtimes.get('daemon')
t_import = self.mtimes.get('import')
print_log(
"catch_up: block %d (%.3fs %.3fs)" %
(self.storage.height, t_daemon, t_import),
self.storage.get_root_hash().encode('hex'))
self.mtimes['daemon'] = 0
self.mtimes['import'] = 0
else:
# revert current block
block = self.getfullblock(self.storage.last_hash)
print_log("blockchain reorg", self.storage.height,
block.get('previousblockhash'),
self.storage.last_hash)
self.import_block(block,
self.storage.last_hash,
self.storage.height,
sync,
revert=True)
self.pop_header()
self.flush_headers()
self.storage.height -= 1
# read previous header from disk
self.header = self.read_header(self.storage.height)
self.storage.last_hash = self.hash_header(self.header)
if prev_root_hash:
assert prev_root_hash == self.storage.get_root_hash()
prev_root_hash = None
address = params[0]
result = self.store.get_history(address, cache_only)
except Exception, e:
error = str(e) + ': ' + address
print_log("error:", error)
elif method == 'blockchain.block.get_header':
if cache_only:
result = -1
else:
try:
height = params[0]
result = self.store.get_block_header(height)
except Exception, e:
error = str(e) + ': %d' % height
print_log("error:", error)
elif method == 'blockchain.block.get_chunk':
if cache_only:
result = -1
else:
try:
index = params[0]
result = self.store.get_chunk(index)
except Exception, e:
error = str(e) + ': %d' % index
print_log("error:", error)
elif method == 'blockchain.transaction.broadcast':
txo = self.store.send_tx(params[0])
print_log("sent tx:", txo)
from processor import Dispatcher, print_log
from backends.irc import ServerProcessor
from transports.stratum_tcp import TcpServer
from transports.stratum_http import HttpServer
backend_name = config.get('server', 'backend')
if backend_name == 'libbitcoin':
from backends.libbitcoin import BlockchainProcessor
elif backend_name == 'leveldb':
from backends.bitcoind import BlockchainProcessor
else:
print "Unknown backend '%s' specified\n" % backend_name
sys.exit(1)
print "\n\n\n\n\n"
print_log("Starting Electrum server on", host)
# Create hub
dispatcher = Dispatcher(config)
shared = dispatcher.shared
# handle termination signals
import signal
def handler(signum = None, frame = None):
print_log('Signal handler called with signal', signum)
shared.stop()
for sig in [signal.SIGTERM, signal.SIGHUP, signal.SIGQUIT]:
signal.signal(sig, handler)
# Create and register processors