def getBlockHeaderHash(mptr: mmap, start: int):
seek = start + 8
mptr.seek(seek) ## ignore magic number and block size
block_header = mptr.read(80)
block_header_hash = hashlib.sha256(
hashlib.sha256(block_header).digest()).digest()
return bytes.decode(binascii.hexlify(block_header_hash[::-1]))
def getBlockPreHeader(mptr: mmap):
block_pre_header = {}
block_pre_header['magic_number'] = bytes.decode(
binascii.hexlify(mptr.read(4)[::-1]))
# logging.debug('magic_number = %s' % block_pre_header['magic_number'])
block_pre_header['block_length'] = int(
binascii.hexlify(mptr.read(4)[::-1]), 16)
return block_pre_header
def getBlock(mptr: mmap, start: int):
block = {}
mptr.seek(start + 88)
txn_count = getTransactionCount(mptr)
txn_list = []
txn_list.append(getCoinbaseTransaction(mptr))
for index in range(1, txn_count):
txn = getTransaction(mptr)
txn_list.append(txn)
block['txn_list'] = txn_list
return block
def _ensure_length(t: int, file: BinaryIO, file_mmap: mmap) -> mmap:
if len(file_mmap) > t:
return file_mmap
size = len(file_mmap)
add_size = (t - size) * 2 + int(size / 2)
file_mmap.flush()
file_mmap.close()
file.seek(size)
file.write(b'\0' * add_size)
file.flush()
ans = mmap.mmap(file.fileno(), 0)
assert (len(ans) > t)
return ans
def getBlock(mptr: mmap, start: int):
global g_block_header_hash, g_txn_index
block = {}
while mptr.read(4) == b'\x00\x00\x00\x00':
start += 4
logging.debug('skipped')
pass
# while True:
# if bytes.decode(binascii.hexlify(mptr.read(4)[::-1])) == '00000000':
# start += 4
# else:
# break
block['block_header_hash'] = getBlockHeaderHash(mptr, start)
g_block_header_hash = block['block_header_hash']
mptr.seek(start) ## ignore magic number and block size
block['block_pre_header'] = getBlockPreHeader(mptr)
# logging.debug('blockfile index = %d' % g_blockfile_index)
# logging.debug('block index = %d' % g_block_index)
# logging.debug('block_header_hash = %s' % g_block_header_hash)
assert block['block_pre_header']['magic_number'] == '00000000' or block[
'block_pre_header']['magic_number'] == 'd9b4bef9'
# if block['block_pre_header']['magic_number'] == '00000000':
# mptr.seek(start)
# raise EOFError
block['block_header'] = getBlockHeader(mptr)
block['txn_count'] = getTransactionCount(mptr)
txn_list = []
txn_list.append(getCoinbaseTransaction(mptr))
for index in range(1, block['txn_count']):
g_txn_index = index
txn = getTransaction(mptr)
txn_list.append(txn)
block['txn_list'] = txn_list
g_genesis_flag = False
return block
def getCountBytes(mptr: mmap):
mptr_read = mptr.read(1)
count_bytes = mptr_read
txn_size = int(binascii.hexlify(mptr_read), 16)
if txn_size < 0xfd:
return count_bytes
elif txn_size == 0xfd:
mptr_read = mptr.read(2)
count_bytes += mptr_read
txn_size = int(binascii.hexlify(mptr_read[::-1]), 16)
return count_bytes
elif txn_size == 0xfe:
mptr_read = mptr.read(4)
count_bytes += mptr_read
txn_size = int(binascii.hexlify(mptr_read[::-1]), 16)
return count_bytes
else:
mptr_read = mptr.read(8)
count_bytes += mptr_read
txn_size = int(binascii.hexlify(mptr_read[::-1]), 16)
return count_bytes
def getBlock(mptr: mmap, start: int):
global g_txn_index
prev_block_header_hash = getPrevBlockHeaderHash(mptr, start)
start += 80
mptr.seek(start) ## skip block header
txn_count = getTransactionCount(mptr)
print('transaction count = %d' % txn_count)
logging.debug('transaction count = %d' % txn_count)
txn_list = []
txn_list.append(getCoinbaseTransaction(mptr))
for index in range(1, txn_count):
g_txn_index = index
txn = getTransaction(mptr)
txn_list.append(txn)
block = {}
block['block'] = txn_list
g_genesis_flag = False
return prev_block_header_hash, block
def seek_payload(m_space: mmap):
cursor = 0
header = ''
regex = r"(#.*\n)*(P6|P3)(\n|\s){0,2}(#.*\n)*(\d+)(\n|\s){0,2}(#.*\n)*(\d+)(\n|\s){0,2}(#.*\n)*(255|1023)(\n|\s){0,2}(#.*\n)*"
while not re.match(regex, header, re.MULTILINE):
try:
header += str(m_space.readline(), encoding='utf-8')
# Esta condicion se cumple cuando se llego al final del la memoria
# mapeada y no se es capaz de avanzar.
if cursor == m_space.tell():
raise errores.FormatIdentifierNotFound
cursor = m_space.tell()
except errores.FormatIdentifierNotFound:
print("El archivo no contiene la cabecera de una archivo PPM")
sys.exit(1)
except Exception as e:
print(f'Error inesperado leyendo el archivo. Error {e}')
sys.exit(1)
return cursor, header
def getBlockHeader(mptr: mmap):
block_header = {}
block_header['block_version'] = int(binascii.hexlify(mptr.read(4)[::-1]),
16)
block_header['prev_block_hash'] = bytes.decode(
binascii.hexlify(mptr.read(32)[::-1]))
block_header['merkle_tree_root'] = bytes.decode(
binascii.hexlify(mptr.read(32)[::-1]))
block_header['timestamp'] = int(binascii.hexlify(mptr.read(4)[::-1]), 16)
block_header['date_time'] = datetime.datetime.fromtimestamp(
block_header['timestamp']).strftime('%Y-%m-%d %H:%M:%S')
block_header['bits'] = bytes.decode(binascii.hexlify(mptr.read(4)[::-1]))
block_header['nounce'] = bytes.decode(binascii.hexlify(mptr.read(4)[::-1]))
print('block_header = %s' % block_header)
return block_header
def getTransaction(mptr: mmap):
global g_blockfile_index, g_txn_index, g_block_header_hash, g_block_index
txn = {}
mptr_read = mptr.read(4)
raw_txn = mptr_read
txn['version'] = int(binascii.hexlify(mptr_read[::-1]), 16)
mptr_read = getCountBytes(mptr)
input_count = getCount(mptr_read)
if input_count == 0:
# post segwit
txn['is_segwit'] = bool(int(binascii.hexlify(mptr.read(1)), 16))
mptr_read = getCountBytes(mptr)
txn['input_count'] = getCount(mptr_read)
else:
txn['input_count'] = input_count
raw_txn += mptr_read
txn['input'] = []
for index in range(txn['input_count']):
txn_input = {}
mptr_read = mptr.read(32)
raw_txn += mptr_read
txn_input['prev_txn_hash'] = bytes.decode(
binascii.hexlify(mptr_read[::-1]))
mptr_read = mptr.read(4)
raw_txn += mptr_read
txn_input['prev_txn_out_index'] = int(
binascii.hexlify(mptr_read[::-1]), 16)
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
txn_input['scriptsig_size'] = getCount(mptr_read)
mptr_read = mptr.read(txn_input['scriptsig_size'])
raw_txn += mptr_read
txn_input['scriptsig'] = bytes.decode(binascii.hexlify(mptr_read))
mptr_read = mptr.read(4)
raw_txn += mptr_read
txn_input['sequence'] = int(binascii.hexlify(mptr_read[::-1]), 16)
txn['input'].append(txn_input)
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
txn['out_count'] = getCount(mptr_read)
txn['out'] = []
for index in range(txn['out_count']):
txn_out = {}
mptr_read = mptr.read(8)
raw_txn += mptr_read
txn_out['_satoshis'] = int(binascii.hexlify(mptr_read[::-1]), 16)
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
txn_out['scriptpubkey_size'] = getCount(mptr_read)
mptr_read = mptr.read(txn_out['scriptpubkey_size'])
raw_txn += mptr_read
txn_out['scriptpubkey'] = bytes.decode(binascii.hexlify(mptr_read))
txn['out'].append(txn_out)
if 'is_segwit' in txn and txn['is_segwit'] == True:
for index in range(txn['input_count']):
mptr_read = getCountBytes(mptr)
txn['input'][index]['witness_count'] = getCount(mptr_read)
txn['input'][index]['witness'] = []
for inner_index in range(txn['input'][index]['witness_count']):
txn_witness = {}
mptr_read = getCountBytes(mptr)
txn_witness['size'] = getCount(mptr_read)
txn_witness['witness'] = bytes.decode(
binascii.hexlify(mptr.read(txn_witness['size'])))
txn['input'][index]['witness'].append(txn_witness)
mptr_read = mptr.read(4)
raw_txn += mptr_read
txn['locktime'] = int(binascii.hexlify(mptr_read[::-1]), 16)
txn['txn_hash'] = getTxnHash(raw_txn)
logging.debug(json.dumps(txn, indent=4))
logging.debug('raw_txn_str = %s' % bytes.decode(binascii.hexlify(raw_txn)))
logging.debug('raw_txn_bytes = %s' % raw_txn)
# check_raw_txn = rpc_connection.getrawtransaction(txn['txn_hash'])
# logging.debug('blockfile index = %d' % g_blockfile_index)
# logging.debug('block index = %d' % g_block_index)
# logging.debug('txn index = %d' % g_txn_index)
# logging.debug('block_header_hash = %s' % g_block_header_hash)
# logging.debug('checked raw txn = %s' % check_raw_txn)
# logging.debug('txn_hash = %s' % txn['txn_hash'])
# logging.debug('raw_txn = %s' % bytes.decode(binascii.hexlify(raw_txn)))
return txn
def getCoinbaseTransaction(mptr: mmap):
global g_blockfile_index, g_txn_index, g_block_header_hash, g_block_index, g_genesis_flag
txn = {}
mptr_read = mptr.read(4)
raw_txn = mptr_read
txn['version'] = int(binascii.hexlify(mptr_read[::-1]), 16)
mptr_read = getCountBytes(mptr)
input_count = getCount(mptr_read)
if input_count == 0:
# post segwit
txn['is_segwit'] = bool(int(binascii.hexlify(mptr.read(1)), 16))
mptr_read = getCountBytes(mptr)
txn['input_count'] = getCount(mptr_read)
else:
txn['input_count'] = input_count # this will be 1
raw_txn += mptr_read
txn['input'] = []
for index in range(txn['input_count']):
txn_input = {}
mptr_read = mptr.read(32)
raw_txn += mptr_read
txn_input['prev_txn_hash'] = bytes.decode(
binascii.hexlify(mptr_read[::-1]))
mptr_read = mptr.read(4)
raw_txn += mptr_read
txn_input['prev_txn_out_index'] = int(
binascii.hexlify(mptr_read[::-1]), 16)
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
txn_input['coinbase_data_size'] = getCount(mptr_read)
fptr1 = mptr.tell()
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
txn_input['coinbase_data_bytes_in_height'] = getCount(mptr_read)
mptr_read = mptr.read(txn_input['coinbase_data_bytes_in_height'])
raw_txn += mptr_read
txn_input['coinbase_data_block_height'] = int(
binascii.hexlify(mptr_read[::-1]), 16)
fptr2 = mptr.tell()
arbitrary_data_size = txn_input['coinbase_data_size'] - (fptr2 - fptr1)
mptr_read = mptr.read(arbitrary_data_size)
raw_txn += mptr_read
txn_input['coinbase_arbitrary_data'] = bytes.decode(
binascii.hexlify(mptr_read[::-1]))
mptr_read = mptr.read(4)
raw_txn += mptr_read
txn_input['sequence'] = int(binascii.hexlify(mptr_read[::-1]), 16)
txn['input'].append(txn_input)
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
txn['out_count'] = getCount(mptr_read)
txn['out'] = []
for index in range(txn['out_count']):
txn_out = {}
mptr_read = mptr.read(8)
raw_txn += mptr_read
txn_out['satoshis'] = int(binascii.hexlify(mptr_read[::-1]), 16)
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
txn_out['scriptpubkey_size'] = getCount(mptr_read)
mptr_read = mptr.read(txn_out['scriptpubkey_size'])
raw_txn += mptr_read
txn_out['scriptpubkey'] = bytes.decode(binascii.hexlify(mptr_read))
txn['out'].append(txn_out)
if 'is_segwit' in txn and txn['is_segwit'] == True:
for index in range(txn['input_count']):
count_bytes = getCountBytes(mptr)
txn['input'][index]['witness_count'] = getCount(count_bytes)
txn['input'][index]['witness'] = []
for inner_index in range(txn['input'][index]['witness_count']):
txn_witness = {}
count_bytes = getCountBytes(mptr)
txn_witness['size'] = getCount(count_bytes)
txn_witness['witness'] = bytes.decode(
binascii.hexlify(mptr.read(txn_witness['size'])))
txn['input'][index]['witness'].append(txn_witness)
mptr_read = mptr.read(4)
raw_txn += mptr_read
txn['locktime'] = int(binascii.hexlify(mptr_read[::-1]), 16)
txn['txn_hash'] = getTxnHash(raw_txn)
# logging.debug(json.dumps(txn, indent=4))
# logging.debug('raw_txn = %s' % bytes.decode(binascii.hexlify(raw_txn)))
# if g_genesis_flag is False:
# check_raw_txn = rpc_connection.getrawtransaction(txn['txn_hash'])
# logging.debug('blockfile index = %d' % g_blockfile_index)
# logging.debug('block index = %d' % g_block_index)
# logging.debug('txn index = %d' % g_txn_index)
# logging.debug('block_header_hash = %s' % g_block_header_hash)
# logging.debug('checked raw txn = %s' % check_raw_txn)
# logging.debug('txn_hash = %s' % txn['txn_hash'])
return txn
def getCoinbaseTransaction(mptr: mmap):
global g_txn_index, g_block_header_hash, g_block_index, g_genesis_flag
txn = {}
mptr_read = mptr.read(4)
raw_txn = mptr_read
mptr_read = getCountBytes(mptr)
input_count = getCount(mptr_read)
is_segwit = False
if input_count == 0:
# post segwit
is_segwit = bool(int(binascii.hexlify(mptr.read(1)), 16))
mptr_read = getCountBytes(mptr)
input_count = getCount(mptr_read)
raw_txn += mptr_read
txn['input'] = []
for index in range(input_count):
txn_input = {}
mptr_read = mptr.read(32)
raw_txn += mptr_read
mptr_read = mptr.read(4)
raw_txn += mptr_read
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
coinbase_data_size = getCount(mptr_read)
fptr1 = mptr.tell()
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
coinbase_data_bytes_in_height = getCount(mptr_read)
mptr_read = mptr.read(coinbase_data_bytes_in_height)
raw_txn += mptr_read
fptr2 = mptr.tell()
arbitrary_data_size = coinbase_data_size - (fptr2 - fptr1)
mptr_read = mptr.read(arbitrary_data_size)
raw_txn += mptr_read
mptr_read = mptr.read(4)
raw_txn += mptr_read
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
out_count = getCount(mptr_read)
txn['out'] = []
for index in range(out_count):
txn_out = {}
txn_out['index'] = index
mptr_read = mptr.read(8)
raw_txn += mptr_read
txn_out['btc'] = round(
int(binascii.hexlify(mptr_read[::-1]), 16) / 10**8, 8)
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
scriptpubkey_size = getCount(mptr_read)
mptr_read = mptr.read(scriptpubkey_size)
raw_txn += mptr_read
script_type = getScriptType(mptr_read)
if script_type == "P2PKH":
txn_out['address'] = getAddressFromP2PKH(mptr_read)
elif script_type == "P2SH":
txn_out['address'] = getAddressFromP2SH(mptr_read)
elif script_type == "P2WPKH":
txn_out['address'] = getAddressFromP2WPKH(mptr_read)
elif script_type == "P2WSH":
txn_out['address'] = getAddressFromP2WSH(mptr_read)
txn_out['scriptpubkey'] = bytes.decode(binascii.hexlify(mptr_read))
if txn_out['btc'] != 0.0:
txn['out'].append(txn_out)
if is_segwit == True:
for index in range(input_count):
count_bytes = getCountBytes(mptr)
witness_count = getCount(count_bytes)
for inner_index in range(witness_count):
count_bytes = getCountBytes(mptr)
txn_witness_size = getCount(count_bytes)
txn_witness_witness = bytes.decode(
binascii.hexlify(mptr.read(txn_witness_size)))
mptr_read = mptr.read(4)
raw_txn += mptr_read
locktime = int(binascii.hexlify(mptr_read[::-1]), 16)
txn['txn_hash'] = getTxnHash(raw_txn)
logging.debug(json.dumps(txn, indent=4))
return txn
def getPrevBlockHeaderHash(mptr: mmap, start: int):
seek = start + 4 ## ignore block version
mptr.seek(seek)
prev_block_hash = mptr.read(32)
return prev_block_hash
def getTransaction(mptr: mmap):
global g_txn_index, g_block_header_hash, g_block_index
txn = {}
mptr_read = mptr.read(4)
raw_txn = mptr_read
mptr_read = getCountBytes(mptr)
input_count = getCount(mptr_read)
is_segwit = False
if input_count == 0:
# post segwit
is_segwit = bool(int(binascii.hexlify(mptr.read(1)), 16))
mptr_read = getCountBytes(mptr)
input_count = getCount(mptr_read)
raw_txn += mptr_read
txn['input'] = []
for index in range(input_count):
txn_input = {}
mptr_read = mptr.read(32)
raw_txn += mptr_read
txn_input['prev_txn_hash'] = bytes.decode(
binascii.hexlify(mptr_read[::-1]))
mptr_read = mptr.read(4)
raw_txn += mptr_read
txn_input['prev_txn_out_index'] = int(
binascii.hexlify(mptr_read[::-1]), 16)
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
scriptsig_size = getCount(mptr_read)
mptr_read = mptr.read(scriptsig_size)
raw_txn += mptr_read
# scriptsig = bytes.decode(binascii.hexlify(mptr_read))
mptr_read = mptr.read(4)
raw_txn += mptr_read
# sequence = int(binascii.hexlify(mptr_read[::-1]), 16)
txn['input'].append(txn_input)
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
out_count = getCount(mptr_read)
txn['out'] = []
for index in range(out_count):
txn_out = {}
txn_out['index'] = index
mptr_read = mptr.read(8)
raw_txn += mptr_read
txn_out['btc'] = round(
int(binascii.hexlify(mptr_read[::-1]), 16) / 10**8, 8)
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
scriptpubkey_size = getCount(mptr_read)
mptr_read = mptr.read(scriptpubkey_size)
raw_txn += mptr_read
script_type = getScriptType(mptr_read)
if script_type == "P2PKH":
txn_out['address'] = getAddressFromP2PKH(mptr_read)
elif script_type == "P2SH":
txn_out['address'] = getAddressFromP2SH(mptr_read)
elif script_type == "P2WPKH":
txn_out['address'] = getAddressFromP2WPKH(mptr_read)
elif script_type == "P2WSH":
txn_out['address'] = getAddressFromP2WSH(mptr_read)
txn_out['scriptpubkey'] = bytes.decode(binascii.hexlify(mptr_read))
txn['out'].append(txn_out)
if is_segwit == True:
for index in range(input_count):
mptr_read = getCountBytes(mptr)
witness_count = getCount(mptr_read)
for inner_index in range(witness_count):
mptr_read = getCountBytes(mptr)
txn_witness_size = getCount(mptr_read)
txn_witness_witness = bytes.decode(
binascii.hexlify(mptr.read(txn_witness_size)))
mptr_read = mptr.read(4)
raw_txn += mptr_read
locktime = int(binascii.hexlify(mptr_read[::-1]), 16)
txn['txn_hash'] = getTxnHash(raw_txn)
logging.debug(json.dumps(txn, indent=4))
return txn
def getCoinbaseTransaction(mptr: mmap):
txn = {}
raw_txn = mptr.read(4)
mptr_read = getCountBytes(mptr)
input_count = getCount(mptr_read)
if input_count == 0:
# post segwit
is_segwit = bool(int(binascii.hexlify(mptr.read(1)), 16))
mptr_read = getCountBytes(mptr)
input_count = getCount(mptr_read)
raw_txn += mptr_read
txn['input'] = []
for index in range(input_count):
txn_input = {}
mptr_read = mptr.read(32)
raw_txn += mptr_read
txn_input['prev_txn_hash'] = bytes.decode(
binascii.hexlify(mptr_read[::-1]))
mptr_read = mptr.read(4)
raw_txn += mptr_read
txn_input['prev_txn_out_index'] = int(
binascii.hexlify(mptr_read[::-1]), 16)
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
coinbase_data_size = getCount(mptr_read)
fptr1 = mptr.tell()
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
raw_txn += mptr.read(getCount(mptr_read))
fptr2 = mptr.tell()
arbitrary_data_size = coinbase_data_size - (fptr2 - fptr1)
raw_txn += mptr.read(arbitrary_data_size)
raw_txn += mptr.read(4)
txn['input'].append(txn_input)
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
out_count = getCount(mptr_read)
txn['out'] = []
for index in range(out_count):
txn_out = {}
mptr_read = mptr.read(8)
raw_txn += mptr_read
txn_out['satoshis'] = int(binascii.hexlify(mptr_read[::-1]), 16)
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
scriptpubkey_size = getCount(mptr_read)
mptr_read = mptr.read(scriptpubkey_size)
raw_txn += mptr_read
txn_out['scriptpubkey'] = bytes.decode(binascii.hexlify(mptr_read))
txn['out'].append(txn_out)
if 'is_segwit' in txn and is_segwit == True:
for index in range(input_count):
count_bytes = getCountBytes(mptr)
witness_count = getCount(count_bytes)
txn['input'][index]['witness'] = []
for inner_index in range(witness_count):
txn_witness = {}
count_bytes = getCountBytes(mptr)
witness_size = getCount(count_bytes)
txn_witness['witness'] = bytes.decode(
binascii.hexlify(mptr.read(witness_size)))
txn['input'][index]['witness'].append(txn_witness)
mptr_read = mptr.read(4)
raw_txn += mptr_read
txn['txn_hash'] = getTxnHash(raw_txn)
return txn
def getTransaction(mptr: mmap):
txn_id = set()
out = {}
inputs = {}
txn = {}
raw_txn = mptr.read(4)
mptr_read = getCountBytes(mptr)
input_count = getCount(mptr_read)
if input_count == 0:
# post segwit
is_segwit = bool(int(binascii.hexlify(mptr.read(1)), 16))
mptr_read = getCountBytes(mptr)
input_count = getCount(mptr_read)
raw_txn += mptr_read
txn['input'] = []
input_list = []
for index in range(input_count):
txn_input = {}
mptr_read = mptr.read(32)
raw_txn += mptr_read
txn_input['prev_txn_hash'] = bytes.decode(
binascii.hexlify(mptr_read[::-1]))
txn_id.add(txn_input['prev_txn_hash'])
mptr_read = mptr.read(4)
raw_txn += mptr_read
txn_input['prev_txn_out_index'] = int(
binascii.hexlify(mptr_read[::-1]), 16)
out_id = "%s_%d" % (txn_input['prev_txn_hash'],
txn_input['prev_txn_out_index'])
if out_id not in out:
out[out_id] = {}
out[out_id]['index'] = txn_input['prev_txn_out_index']
out[out_id]['satoshis'] = -1
input_list.append(out_id)
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
scriptsig_size = getCount(mptr_read)
mptr_read = mptr.read(scriptsig_size)
raw_txn += mptr_read
txn_input['scriptsig'] = bytes.decode(binascii.hexlify(mptr_read))
raw_txn += mptr.read(4)
txn['input'].append(txn_input)
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
out_count = getCount(mptr_read)
txn['out'] = []
for index in range(out_count):
out_list = []
txn_out = {}
mptr_read = mptr.read(8)
raw_txn += mptr_read
txn_out['satoshis'] = int(binascii.hexlify(mptr_read[::-1]), 16)
out_list.append(txn_out['satoshis'])
mptr_read = getCountBytes(mptr)
raw_txn += mptr_read
scriptpubkey_size = getCount(mptr_read)
mptr_read = mptr.read(scriptpubkey_size)
raw_txn += mptr_read
txn_out['scriptpubkey'] = bytes.decode(binascii.hexlify(mptr_read))
txn['out'].append(txn_out)
if 'is_segwit' in txn and is_segwit == True:
for index in range(input_count):
mptr_read = getCountBytes(mptr)
witness_count = getCount(mptr_read)
txn['input'][index]['witness'] = []
for inner_index in range(witness_count):
txn_witness = {}
mptr_read = getCountBytes(mptr)
witness_size = getCount(mptr_read)
txn_witness['witness'] = bytes.decode(
binascii.hexlify(mptr.read(witness_size)))
txn['input'][index]['witness'].append(txn_witness)
raw_txn += mptr.read(4)
txn['txn_hash'] = getTxnHash(raw_txn)
txn_id.add(txn['txn_hash'])
inputs[txn['txn_hash']] = input_list
for index in range(len(out_list)):
out_id = "%s_%d" % (txn['txn_hash'], index)
if out_id not in out:
out[out_id] = {}
out[out_id]['index'] = index
out[out_id]['satoshis'] = out_list[index]
print(json.dumps(txn, indent=4))
neo4j_import = os.path.join(os.path.sep, 'var', 'lib', 'neo4j', 'import')
txn_node_path = os.path.join(neo4j_import, 'txn_node.csv')
with open(txn_node_path, 'wt') as txn_file:
print('name', file=txn_file)
print('\n'.join(txn_id), file=txn_file)
out_node_path = os.path.join(neo4j_import, 'out_node.csv')
with open(out_node_path, 'wt') as out_file:
out_list = [{
'id': key,
'name': value['index'],
'satoshis': value['satoshis']
} for key, value in out.items()]
print('out_node: %s' % out_list)
keys = out_list[0].keys()
dict_writer = csv.DictWriter(out_file, keys)
dict_writer.writeheader()
dict_writer.writerows(out_list)
txn_out_rel_path = os.path.join(neo4j_import, 'txn_out_rel.csv')
with open(txn_out_rel_path, 'wt') as txn_out_file:
txn_out_list = [{
'from_node': 'transaction',
'from_key': key.split('_')[0],
'to_node': 'out',
'to_key': key
} for key in out.keys()]
print('txn_out_rel: %s' % txn_out_list)
keys = txn_out_list[0].keys()
dict_writer = csv.DictWriter(txn_out_file, keys)
dict_writer.writeheader()
dict_writer.writerows(txn_out_list)
input_txn_rel_path = os.path.join(neo4j_import, 'input_txn_rel.csv')
with open('input_txn_rel', 'wt') as out_txn_file:
out_txn_list = [{
'from_node': 'out',
'from_key': value,
'satoshis': out[value]['satoshis'],
'to_node': 'transaction',
'to_key': txn['txn_hash']
} for value in inputs[txn['txn_hash']]]
print('input_txn_rel: %s' % out_txn_list)
keys = out_txn_list[0].keys()
dict_writer = csv.DictWriter(out_txn_file, keys)
dict_writer.writeheader()
dict_writer.writerows(out_txn_list)
exit()
# node
cypher = 'USING PERIODIC COMMIT '
cypher += 'LOAD CSV WITH HEADERS FROM "file:///' + + '" AS row '
if 'source_reference' in columns:
cypher += 'MERGE (n:' + node + ' { source_reference:row.source_reference })'
else:
cypher += 'MERGE (n:' + node + ' { name:row.name })'
cypher += ' ON CREATE SET n = { ' + attribute_string + ' }'
cypher += ' ON MATCH SET n += { ' + attribute_string + ' }'
try:
response = graph.run(cypher)
except ProtocolError:
raise CustomException(
{
'error_message':
'Unable to establish connection with Neo4j Bolt Server',
'host': os.getenv('NEXT_GEN_TOOLING_SERVER_HOSTNAME'),
'BOLT_PORT': int(os.getenv('BOLT_PORT'))
}, 530)
except ConstraintError as e:
raise CustomException({'error_message': str(e)}, 545)
resp_value = response.stats()
# look at the stats() dictionary returned for all the queryable values in the response
nodes_added = resp_value.get('labels_added')
## rel
# cypher = 'USING PERIODIC COMMIT '
# cypher += 'LOAD CSV WITH HEADERS FROM "file:///' + filename + '" AS row '
# # let's match the from node first
# if "glob_source" in dir(builtins):
# cypher += 'MATCH (fn:' + from_node +":"+ builtins.glob_source + ' {' + from_attr + ':row.from_key}) '
# else:
# cypher += 'MATCH (fn:' + from_node + ' {' + from_attr + ':row.from_key}) '
# # let's match the to node now
# if "glob_source" in dir(builtins):
# cypher += 'MATCH (tn:' + to_node +":"+ builtins.glob_source + ' {' + to_attr + ':row.to_key}) '
# else:
# cypher += 'MATCH (tn:' + to_node + ' {' + to_attr + ':row.to_key}) '
#
# #todo Relationship direction ALWAYS goes (from_node)-[]->(to_node) - do we need a 'direction' passed in?
# # let's setup the relationship creation (always use unique so relationships are not duplicated)
# cypher += 'CREATE UNIQUE (fn)-[:' + relationship + ']->(tn)'
# try:
# response = graph.run(cypher)
# except ProtocolError:
# raise CustomException({'error_message':'Unable to establish connection with Neo4j Bolt Server', 'host': os.getenv('NEXT_GEN_TOOLING_SERVER_HOSTNAME'), 'BOLT_PORT': int(os.getenv('BOLT_PORT'))}, 530)
# except ConstraintError:
# raise CustomException({'error_message':str(e)}, 545)
# logging.info(cypher)
# resp_value = response.stats()
# rels_added = resp_value.get('relationships_created')
# check_raw_txn = rpc_connection.getrawtransaction(txn['txn_hash'])
# print('checked raw txn = %s' % check_raw_txn)
# print('txn_hash = %s' % txn['txn_hash'])
return txn