def get_block_by_hash(self,
block_hash: str) -> Union[Dict[str, str], None]:
"""
Retrieves block information specified by its hash.
Args:
block_hash: Block hash.
Returns:
Dictionary representing the blockchain block.
"""
block_index = db_get_wrapper(self.db,
b'hash-block-' + block_hash.encode())
if block_index is None:
LOG.info('Block of specified hash not found.')
return None
raw_block = db_get_wrapper(self.db, b'block-' + block_index)
block = coder.decode_block(raw_block)
block['number'] = block_index.decode()
transaction_hashes = block['transactions'].split('+')
transactions = [] # type: List[Dict[str, Any]]
if transaction_hashes == ['']:
block['transactions'] = transactions
return block
for tx_hash in transaction_hashes:
transaction = self.get_transaction_by_hash(tx_hash)
transactions.append(transaction) # type: ignore
block['transactions'] = transactions
return block
def get_transactions_of_block_by_hash(
self, block_hash: str) -> Union[List[Dict[str, Any]], None]:
"""
Gets list of transactions belonging to specified block.
Args:
block_hash: hash of the block.
Returns:
List of specified block transactions.
"""
block_index = db_get_wrapper(self.db,
b'hash-block-' + block_hash.encode())
if block_index is None:
LOG.info('Block of specified hash not found.')
return None
raw_block = db_get_wrapper(self.db, b'block-' + block_index)
block = coder.decode_block(raw_block)
transaction_hashes = block['transactions'].split('+')
transactions = [] # type: List[Dict[str, Any]]
if transaction_hashes == ['']:
block['transactions'] = transactions
return []
for tx_hash in transaction_hashes:
transaction = self.get_transaction_by_hash(tx_hash)
transactions.append(transaction) # type: ignore
return transactions
def get_transaction_by_hash(self, tx_hash) -> Union[Dict[str, Any], None]:
"""
Retrieves transaction specified by its hash.
Args:
tx_hash: Hash of the transaction.
Returns:
The desired transaction.
"""
raw_tx = db_get_wrapper(self.db, b'transaction-' + tx_hash.encode())
if raw_tx is None:
LOG.info('Transaction of given hash not found.')
return None
transaction = coder.decode_transaction(raw_tx)
transaction['hash'] = tx_hash
internal_tx_indexes = [] # type: List[Any]
if transaction['internalTxIndex'] > 0:
prefix = 'associated-data-' + tx_hash + '-tit-'
print('OOO')
internal_tx_indexes = db_iter_wrapper(self.db, prefix)
print('OOOO')
internal_transactions = []
for tx_index in internal_tx_indexes:
tx_decoded = tx_index.decode()
raw_tx = db_get_wrapper(self.db,
b'internal-tx-' + tx_decoded.encode())
int_tx = coder.decode_internal_tx(raw_tx)
internal_transactions.append(int_tx)
transaction.pop('internalTxIndex', None)
transaction['internalTransactions'] = internal_transactions
return transaction
def expand_tokens(self, tokens: Dict,
token_txs: List) -> Tuple[Dict, Dict]:
"""
Find all relevant tokens from DB and reject not-found transactions.
Args:
tokens: Tokens gathered so far (in this batch).
token_txs: Token transactions to get other token info.
Returns:
Updated token list.
"""
full_tokens = {}
filtered_txs = {}
for token_tx in token_txs:
data = db_get_wrapper(
self.db, b'token-' + token_tx['tokenAddress'].encode())
if data is not None:
db_token = coder.decode_token(data)
db_token['transactions'] = []
full_tokens[token_tx['tokenAddress']] = db_token
self._highest_token_tx += 1
filtered_txs[self._highest_token_tx] = token_tx
elif token_tx['tokenAddress'] in tokens:
full_tokens[token_tx['tokenAddress']] = tokens[
token_tx['tokenAddress']]
self._highest_token_tx += 1
filtered_txs[self._highest_token_tx] = token_tx
for token in tokens:
if token not in full_tokens:
full_tokens[token] = tokens[token]
return (full_tokens, filtered_txs)
def get_transactions_of_block_by_index(
self, block_index: str) -> Union[List[Dict[str, Any]], None]:
"""
Gets list of transactions belonging to specified block.
Args:
block_index: index of the block.
Returns:
List of specified block transactions.
"""
raw_block = db_get_wrapper(self.db, b'block-' + block_index.encode())
block = coder.decode_block(raw_block)
transaction_hashes = block['transactions'].split('+')
transactions = [] # type: List[Dict[str, Any]]
if transaction_hashes == ['']:
block['transactions'] = transactions
return []
for tx_hash in transaction_hashes:
transaction = self.get_transaction_by_hash(tx_hash)
transactions.append(transaction) # type: ignore
return transactions
def get_token(self, addr: str, time_from: int, time_to: int,
no_tx_list: int) -> Union[Dict[str, Any], None]:
"""
Get informtion about a token based on its contract address.
Args:
addr: Token address.
time_from: Beginning datetime to take transactions from.
time_to: Ending datetime to take transactions from.
no_tx_list: Maximum transactions to return.
Returns:
Information about a token.
"""
raw_token = db_get_wrapper(self.db, b'token-' + addr.encode())
if raw_token is None:
return None
token = coder.decode_token(raw_token)
token['address'] = addr
token_tx_indexes = [] # type: List[bytes]
if token['txIndex'] > 0:
prefix = 'associated-data-' + addr + '-tt-'
token_tx_indexes = db_iter_wrapper(self.db, prefix)
found_txs = 0
token_txs = []
for token_tx_index in token_tx_indexes:
if found_txs >= no_tx_list:
break
tx_decoded = token_tx_index.decode()
tx_index, timestamp = tx_decoded.split('-')
if (time_from <= int(timestamp) and time_to >= int(timestamp)):
raw_tx = db_get_wrapper(self.db,
b'token-tx-' + tx_index.encode())
token_tx = coder.decode_token_tx(raw_tx)
token_txs.append(token_tx)
found_txs += 1
token.pop('txIndex', None)
token['tokenTransactions'] = token_txs
return token
def get_block_hash_by_index(self, block_index: str) -> Union[str, None]:
"""
Retrieves block hash by its index.
Args:
block_index: Index of a block.
Returns:
Block hash.
"""
raw_block = db_get_wrapper(self.db, b'block-' + block_index.encode())
if raw_block is None:
return None
block = coder.decode_block(raw_block)
return block['hash']
def get_balance(self, addr: str) -> Union[str, None]:
"""
Get balance of an address.
Args:
addr: Requested address.
Returns:
Current balance of an address.
"""
raw_address = db_get_wrapper(self.db, b'address-' + addr.encode())
if raw_address is None:
return None
address = coder.decode_address(raw_address)
return address['balance']
def _update_db_balances(self, addr_balances: Dict) -> None:
"""
Updates balances of Ethereum addresses in the LevelDB database in batches.
Args:
addr_balances: Dictionary containing 'address: balance' entries.
"""
address_objects = {}
for address in addr_balances:
raw_addr = db_get_wrapper(self.db,
b'address-' + str(address).encode())
if raw_addr is None:
continue
address_objects[address] = coder.decode_address(raw_addr)
address_objects[address]['balance'] = addr_balances[address]
self.db_lock.acquire()
wb = rocksdb.WriteBatch()
for address in address_objects:
address_value = coder.encode_address(address_objects[address])
wb.put(b'address-' + str(address).encode(), address_value)
self.db.write(wb)
self.db_lock.release()
def get_blocks_by_indexes(
self, index_start: str,
index_end: str) -> Union[List[Dict[str, Any]], None]:
"""
Retrieves a list of blocks specified by an index range.
Args:
index_start: First index.
index_end: Last index.
Returns:
A list of blocks.
"""
blocks = []
for block_index in range(int(index_start), int(index_end) + 1):
raw_block = db_get_wrapper(self.db,
b'block-' + str(block_index).encode())
block = coder.decode_block(raw_block)
block['number'] = block_index
transaction_hashes = block['transactions'].split('+')
transactions = [] # type: List[Dict[str, Any]]
if transaction_hashes == ['']:
block['transactions'] = transactions
blocks.append(block)
continue
for tx_hash in transaction_hashes:
transaction = self.get_transaction_by_hash(tx_hash)
transactions.append(transaction) # type: ignore
block['transactions'] = transactions
blocks.append(block)
return blocks
def fill_addresses(self, addresses: Dict, transactions: Dict, tokens: Dict,
token_txs: List) -> Tuple[Dict, Dict]:
"""
Fill addresses with transaction information.
Args:
addresses: Currently processed addresses.
transactions: Currently processed transactions.
tokens: Currently processed tokens.
token_txs: Currently processed token transactions.
Returns:
Addresses with new information.
"""
LOG.info('Filling addresses.')
updated_tokens, filtered_token_txs = self.expand_tokens(
tokens, token_txs)
addresses, updated_tokens = self.fill_addresses_tokens(
addresses, updated_tokens, filtered_token_txs)
addresses_encode = {}
addresses_write_dict = {}
for addr_hash, addr_dict in addresses.items():
existing_data = db_get_wrapper(self.db,
b'address-' + addr_hash.encode())
# Address not yet in records
if existing_data is not None:
existing_address = coder.decode_address(existing_data)
last_input_tx_index = int(existing_address['inputTxIndex'])
last_output_tx_index = int(existing_address['outputTxIndex'])
last_input_token_tx_index = int(
existing_address['inputTokenTxIndex'])
last_output_token_tx_index = int(
existing_address['outputTokenTxIndex'])
last_input_int_tx_index = int(
existing_address['inputIntTxIndex'])
last_output_int_tx_index = int(
existing_address['outputIntTxIndex'])
last_mined_block_index = int(existing_address['minedIndex'])
else:
last_input_tx_index = 0
last_output_tx_index = 0
last_input_token_tx_index = 0
last_output_token_tx_index = 0
last_input_int_tx_index = 0
last_output_int_tx_index = 0
last_mined_block_index = 0
address_encode = {}
if existing_data is not None:
address_encode['tokenContract'] = existing_address[
'tokenContract']
if addr_hash in updated_tokens:
updated_tokens[addr_hash]['type'] = existing_address[
'tokenContract']
else:
if 'tokenContract' in addr_dict:
address_encode['tokenContract'] = addr_dict[
'tokenContract']
if addr_hash in updated_tokens:
updated_tokens[addr_hash]['type'] = addr_dict[
'tokenContract']
else:
address_encode['tokenContract'] = 'False'
address_encode['balance'] = 'null'
if existing_data is not None:
address_encode['code'] = existing_address['code']
else:
address_encode['code'] = addr_dict['code']
for input_tx in addr_dict['newInputTxs']:
last_input_tx_index += 1
addresses_write_dict[addr_hash + '-i-' +
str(last_input_tx_index)] = (
str(input_tx[0]) + '-' +
str(input_tx[1]) + '-' +
str(input_tx[2]))
for output_tx in addr_dict['newOutputTxs']:
last_output_tx_index += 1
addresses_write_dict[addr_hash + '-o-' +
str(last_output_tx_index)] = (
str(output_tx[0]) + '-' +
str(output_tx[1]) + '-' +
str(output_tx[2]))
for mined_hash in addr_dict['mined']:
last_mined_block_index += 1
addresses_write_dict[addr_hash + '-b-' +
str(last_mined_block_index)] = mined_hash
address_encode['inputTxIndex'] = last_input_tx_index
address_encode['outputTxIndex'] = last_output_tx_index
address_encode['inputTokenTxIndex'] = last_input_token_tx_index
address_encode['outputTokenTxIndex'] = last_output_token_tx_index
address_encode['inputIntTxIndex'] = last_input_int_tx_index
address_encode['outputIntTxIndex'] = last_output_int_tx_index
address_encode['minedIndex'] = last_mined_block_index
addresses_encode[addr_hash] = address_encode
# Also add token information to the addresses.
addresses_encode, updated_tokens, addresses_write_dict = self.fill_addrs_token_txs(
addresses, addresses_encode, updated_tokens, addresses_write_dict)
# Also add internal transactions to addresses
addresses_encode, addresses_write_dict = self.fill_addrs_int_txs(
addresses, addresses_encode, addresses_write_dict)
return (addresses_encode, addresses_write_dict, updated_tokens,
filtered_token_txs)
def get_blocks_by_datetime(
self, limit: int, block_start: int,
block_end: int) -> Union[List[Dict[str, Any]], None]:
"""
Retrieves multiple blocks based on specified datetime range.
Args:
limit: Maximum blocks to gather.
block_start: Beginning datetime.
block_end: End datetime.
Returns:
List of block dictionaries.
"""
block_indexes = [] # type: List[int]
blocks = [] # type: List[Dict[str, Any]]
retry_counter = 0
while True:
try:
it = self.db.iteritems()
it.seek(b'timestamp-block-' + str(block_start).encode())
counter = 0
while True:
data = it.get()
timestamp = int(data[0].decode().split('-')[-1])
block_index = int(data[1].decode())
it.__next__()
if timestamp < block_start:
continue
if timestamp > block_end:
break
block_indexes.append(block_index)
counter += 1
if (counter >= limit and limit > 0):
break
break
except rocksdb.errors.RocksIOError as e:
if retry_counter >= 10:
LOG.info('Too many failed retries. Stopping.')
raise e
if 'No such file or directory' in str(e):
LOG.info('DB lookup failed. Retrying.')
sleep(2)
retry_counter += 1
if block_indexes == []:
return None
# Since DB is working with string-numbers things might be kind of tricky
block_indexes.sort()
for block_index in range(block_indexes[0], block_indexes[-1] + 1):
raw_block = db_get_wrapper(self.db,
b'block-' + str(block_index).encode())
block = coder.decode_block(raw_block)
block['number'] = block_index
transaction_hashes = block['transactions'].split('+')
transactions = [] # type: List[Dict[str, Any]]
if transaction_hashes == ['']:
block['transactions'] = transactions
blocks.append(block)
continue
for tx_hash in transaction_hashes:
transaction = self.get_transaction_by_hash(tx_hash)
transactions.append(transaction) # type: ignore
block['transactions'] = transactions
blocks.append(block)
return blocks
def get_address(self, addr: str, time_from: int, time_to: int,
val_from: int, val_to: int,
no_tx_list: int) -> Union[List[Dict[str, Any]], None]:
"""
Get information of an address, with the possibility of filtering/limiting transactions.
Args:
addr: Ethereum address.
time_from: Beginning datetime to take transactions from.
time_to: Ending datetime to take transactions from.
val_from: Minimum transferred currency of the transactions.
val_to: Maximum transferred currency of transactions.
no_tx_list: Maximum transactions to return.
Returns:
Address information along with its transactions.
"""
raw_address = db_get_wrapper(self.db, b'address-' + addr.encode())
if raw_address is None:
return None
address = coder.decode_address(raw_address)
if address['code'] != '0x':
raw_code = db_get_wrapper(
self.db, b'address-contract-' + address['code'].encode())
address['code'] = raw_code.decode()
input_transactions, output_transactions = (
self.get_transactions_of_address(addr, time_from, time_to,
val_from, val_to, no_tx_list,
True))
address.pop('inputTxIndex', None)
address.pop('outputTxIndex', None)
address['inputTransactions'] = input_transactions
address['outputTransactions'] = output_transactions
input_int_transactions, output_int_transactions = (
self.get_internal_txs_of_address(addr, time_from, time_to,
val_from, val_to, no_tx_list,
True))
address.pop('inputIntTxIndex', None)
address.pop('outputIntTxIndex', None)
address['inputInternalTransactions'] = input_int_transactions
address['outputInternalTransactions'] = output_int_transactions
mined_hashes = [] # type: List[bytes]
if address['minedIndex'] > 0:
prefix = 'associated-data-' + addr + '-b-'
mined_hashes = db_iter_wrapper(self.db, prefix)
address.pop('minedIndex', None)
address['mined'] = list(map(lambda x: x.decode(), mined_hashes))
input_token_txs, output_token_txs = (self.get_token_txs_of_address(
addr, time_from, time_to, no_tx_list, True))
address.pop('inputTokenTxIndex', None)
address.pop('outputTokenTxIndex', None)
address['inputTokenTransactions'] = input_token_txs
address['outputTokenTransactions'] = output_token_txs
return address
def get_token_txs_of_address(self,
addr: str,
time_from: int,
time_to: int,
no_tx_list: int,
internal=False) -> Any:
"""
Get token txs of specified address, with filtering by time and transferred capital.
Args:
addr: Ethereum address.
time_from: Beginning datetime to take transactions from.
time_to: Ending datetime to take transactions from.
no_tx_list: Maximum transactions to return.
internal: Whether this method was called internally.
Returns:
List of token transactions of an address.
"""
raw_address = db_get_wrapper(self.db, b'address-' + addr.encode())
if raw_address is None:
return None
address = coder.decode_address(raw_address)
input_token_txs = []
output_token_txs = []
input_token_tx_indexes = [] # type: List[bytes]
output_token_tx_indexes = [] # type: List[bytes]
if address['inputTokenTxIndex'] > 0:
prefix = 'associated-data-' + addr + '-ti-'
input_token_tx_indexes = db_iter_wrapper(self.db, prefix)
if address['outputTokenTxIndex'] > 0:
prefix = 'associated-data-' + addr + '-to-'
output_token_tx_indexes = db_iter_wrapper(self.db, prefix)
found_txs = 0
for token_tx_index in input_token_tx_indexes:
if found_txs >= no_tx_list:
break
tx_decoded = token_tx_index.decode()
tx_index, timestamp = tx_decoded.split('-')
if (time_from <= int(timestamp) and time_to >= int(timestamp)):
raw_tx = db_get_wrapper(self.db,
b'token-tx-' + tx_index.encode())
token_tx = coder.decode_token_tx(raw_tx)
input_token_txs.append(token_tx)
found_txs += 1
for token_tx_index in output_token_tx_indexes:
if found_txs >= no_tx_list:
break
tx_decoded = token_tx_index.decode()
tx_index, timestamp = tx_decoded.split('-')
if (time_from <= int(timestamp) and time_to >= int(timestamp)):
raw_tx = db_get_wrapper(self.db,
b'token-tx-' + tx_index.encode())
token_tx = coder.decode_token_tx(raw_tx)
output_token_txs.append(token_tx)
found_txs += 1
if internal:
return (input_token_txs, output_token_txs)
else:
return input_token_txs + output_token_txs
def get_transactions_of_address(self,
addr: str,
time_from: int,
time_to: int,
val_from: int,
val_to: int,
no_tx_list: int,
internal=False) -> Any:
"""
Get transactions of specified address, with filtering by time and transferred capital.
Args:
addr: Ethereum address.
time_from: Beginning datetime to take transactions from.
time_to: Ending datetime to take transactions from.
val_from: Minimum transferred currency of the transactions.
val_to: Maximum transferred currency of transactions.
no_tx_list: Maximum transactions to return.
internal: Whether this method was called internally.
Returns:
List of address transactions.
"""
raw_address = db_get_wrapper(self.db, b'address-' + addr.encode())
if raw_address is None:
return None
address = coder.decode_address(raw_address)
input_tx_hashes = [] # type: List[bytes]
output_tx_hashes = [] # type: List[bytes]
if address['inputTxIndex'] > 0:
prefix = 'associated-data-' + addr + '-i-'
input_tx_hashes = db_iter_wrapper(self.db, prefix)
if address['outputTxIndex'] > 0:
prefix = 'associated-data-' + addr + '-o-'
output_tx_hashes = db_iter_wrapper(self.db, prefix)
found_txs = 0
input_transactions = []
for tx_data in input_tx_hashes:
if found_txs >= no_tx_list:
break
tx_decoded = tx_data.decode()
tx_hash, value, timestamp = tx_decoded.split('-')
if (time_from <= int(timestamp) and time_to >= int(timestamp)
and val_from <= int(value) and val_to >= int(value)):
transaction = self.get_transaction_by_hash(tx_hash)
transaction.pop('internalTransactions', None) # type: ignore
input_transactions.append(transaction)
found_txs += 1
output_transactions = []
for tx_data in output_tx_hashes:
if found_txs >= no_tx_list:
break
tx_decoded = tx_data.decode()
tx_hash, value, timestamp = tx_decoded.split('-')
if (time_from <= int(timestamp) and time_to >= int(timestamp)
and val_from <= int(value) and val_to >= int(value)):
transaction = self.get_transaction_by_hash(tx_hash)
transaction.pop('internalTransactions', None) # type: ignore
output_transactions.append(transaction)
found_txs += 1
if internal:
return (input_transactions, output_transactions)
else:
return input_transactions + output_transactions
