def add_transaction(cls, transaction_data, block_number, iso_timestamp):
"""
Creates a new transaction object from data received from JSON-RPC call
eth_getBlockByNumber.
:param dict transaction_data: data received from JSON RPC call
:param datetime iso_timestamp: timestamp when the block containing the transaction was mined
:param int block_number: block number of the block where this transaction was included
"""
try:
receiver = to_checksum_address(transaction_data['to'])
except TypeError:
receiver = None
transaction = cls(block_number=block_number,
nonce=to_int(transaction_data['nonce']),
transaction_hash=to_hex(transaction_data['hash']),
sender=to_checksum_address(transaction_data['from']),
start_gas=to_int(transaction_data['gas']),
value=int(str(to_int(transaction_data['value']))),
receiver=receiver,
data=transaction_data['input'],
gas_price=str(to_int(transaction_data['gasPrice'])),
timestamp=iso_timestamp,
transaction_index=to_int(
transaction_data['transactionIndex']))
return transaction
def add_receipt(cls, receipt_data, block_number, timestamp):
"""
Creates a new receipt object from data received from JSON-RPC call
eth_getTransactionReceipt.
:param dict receipt_data: receipt data received from the JSON RPC callable
:param int timestamp: timestamp of the block where this transaction was included
:param int block_number: block number of the block where this transaction was included
"""
if block_number > constants.FORK_BLOCK_NUMBER['Byzantium']:
status = bool(to_int(receipt_data['status']))
else:
status = None
receipt = cls(
transaction_hash=to_hex(receipt_data['transactionHash']),
status=status,
gas_used=to_int(receipt_data['gasUsed']),
cumulative_gas_used=to_int(receipt_data['cumulativeGasUsed']),
contract_address=receipt_data['contractAddress'],
block_number=block_number,
timestamp=timestamp,
transaction_index=to_int(receipt_data['transactionIndex']))
return receipt
def add_log(cls, log_data, block_number, iso_timestamp):
"""
Creates a new log object from data received from JSON-RPC call
eth_getTransactionReceipt.
:param dict log_data: data received from receipt JSON RPC call
:param int block_number: block number of the block containing the log
:param datetime iso_timestamp: timestamp when the block containing the transaction was mined
"""
topics_count = len(log_data['topics'])
log = cls(transaction_hash=to_hex(log_data['transactionHash']),
transaction_index=to_int(log_data['transactionIndex']),
topics_count=topics_count,
address=to_checksum_address(log_data['address']),
log_index=to_int(log_data['logIndex']),
data=log_data['data'],
block_number=block_number,
timestamp=iso_timestamp,
topic_1='',
topic_2='',
topic_3='',
topic_4='')
if topics_count == 0:
logger.warn('No topics present')
elif topics_count == 1:
log.topic_1 = to_hex(log_data['topics'][0])
elif topics_count == 2:
log.topic_1 = to_hex(log_data['topics'][0])
log.topic_2 = to_hex(log_data['topics'][1])
elif topics_count == 3:
log.topic_1 = to_hex(log_data['topics'][0])
log.topic_2 = to_hex(log_data['topics'][1])
log.topic_3 = to_hex(log_data['topics'][2])
elif topics_count == 4:
log.topic_1 = to_hex(log_data['topics'][0])
log.topic_2 = to_hex(log_data['topics'][1])
log.topic_3 = to_hex(log_data['topics'][2])
log.topic_4 = to_hex(log_data['topics'][3])
else:
logger.error('More than 4 topics are not possible')
logger.debug("tx_hash: {}, log_index: {}".format(
log.transaction_hash, log.log_index))
return log
def test_conversion_round_trip(value):
intermediate_value = to_hex(value)
result_value = to_int(hexstr=intermediate_value)
error_msg = "Expected: {0!r}, Result: {1!r}, Intermediate: {2!r}".format(
value,
result_value,
intermediate_value,
)
assert result_value == value, error_msg
def test_conversion_round_trip(value):
intermediate_value = to_hex(value)
result_value = to_int(hexstr=intermediate_value)
error_msg = "Expected: {0!r}, Result: {1!r}, Intermediate: {2!r}".format(
value,
result_value,
intermediate_value,
)
assert result_value == value, error_msg
def add_block(cls, block_data, iso_timestamp):
"""
Creates a new block object from data received from JSON-RPC call
eth_getBlockByNumber.
:param dict block_data: data received from the JSON RPC call
:param datetime iso_timestamp: timestamp when the block was mined
"""
block = cls(block_hash=to_hex(block_data['hash']),
parent_hash=to_hex(block_data['parentHash']),
difficulty=to_int(block_data['difficulty']),
block_number=to_int(block_data['number']),
gas_used=to_int(block_data['gasUsed']),
miner=to_checksum_address(block_data['miner']),
timestamp=iso_timestamp,
sha3uncles=to_hex(block_data['sha3Uncles']),
extra_data=to_hex(block_data['extraData']),
gas_limit=to_int(block_data['gasLimit']),
transaction_count=len(block_data['transactions']),
uncle_count=len(block_data['uncles']))
return block
def new_blocks():
"""
Celery beat task which runs every second to get new blocks.
:param block_filter: block filter as described in session.py
"""
current_session = get_current_session()
logger.debug("Reached at new blocks to get block hashes")
block_hashes = current_session.block_filter.get_new_entries()
for block_hash in block_hashes:
block_data = current_session.w3.eth.getBlock(block_hash)
block_number = to_int(block_data['number'])
BlockTaskMeta.add_block_task_meta(task_name='new_blocks',
state='WAITING',
block_number=block_number,
block_hash=to_hex(block_hash))
logger.info(block_hashes)
def to_standard_signature_bytes(ethereum_signature_bytes):
rs = ethereum_signature_bytes[:-1]
v = to_int(ethereum_signature_bytes[-1])
standard_v = to_standard_v(v)
return rs + to_bytes(standard_v)
def add_block_number(block_number):
"""
Adds the block, transactions, uncles, logs and traces of a given block
number into the db_session
:param int block_number: The block number to add to the database
"""
current_session = get_current_session()
block_task_meta = BlockTaskMeta.update_block_task_meta_from_block_number(
block_number=block_number, state='STARTED')
# getting the block_data from the node
block_data = current_session.w3.eth.getBlock(block_identifier=block_number,
full_transactions=True)
timestamp = to_int(block_data['timestamp'])
iso_timestamp = datetime.utcfromtimestamp(timestamp).isoformat()
block = Blocks.add_block(block_data=block_data,
iso_timestamp=iso_timestamp)
block_hash = block.block_hash
if current_session.settings.PARSE_TRACE and \
current_session.settings.PARSE_STATE_DIFF and \
block_number != 0:
block_trace_list = current_session.w3.parity.\
traceReplayBlockTransactions(block_number,
mode=['trace', 'stateDiff'])
elif block_number != 0:
if current_session.settings.PARSE_TRACE:
block_trace_list = current_session.w3.parity.\
traceReplayBlockTransactions(block_number,
mode=['trace'])
if current_session.settings.PARSE_STATE_DIFF:
block_trace_list = current_session.w3.parity.\
traceReplayBlockTransactions(block_number,
mode=['stateDiff'])
# added the block data in the db session
with current_session.db_session_scope():
current_session.db_session.add(block)
uncle_hashes = block_data['uncles']
uncle_list = []
for i in range(0, len(uncle_hashes)):
# Unfortunately there is no command eth_getUncleByHash
uncle_data = current_session.w3.eth.getUncleByBlock(
block_number, i)
uncle = Uncles.add_uncle(uncle_data=uncle_data,
block_number=block_number,
iso_timestamp=iso_timestamp)
current_session.db_session.add(uncle)
uncle_list.append(uncle)
transaction_list = block_data['transactions']
# loop to get the transaction, receipts, logs and traces of the block
for index, transaction_data in enumerate(transaction_list):
transaction = Transactions.add_transaction(
transaction_data,
block_number=block_number,
iso_timestamp=iso_timestamp)
# added the transaction in the db session
current_session.db_session.add(transaction)
receipt_data = current_session.w3.eth.getTransactionReceipt(
transaction_data['hash'])
receipt = Receipts.add_receipt(receipt_data,
block_number=block_number,
timestamp=iso_timestamp)
current_session.db_session.add(receipt)
fees = int(transaction.gas_price) * int(receipt.gas_used)
log_list = receipt_data['logs']
Logs.add_log_list(current_session=current_session,
log_list=log_list,
block_number=block_number,
timestamp=transaction.timestamp)
if current_session.settings.PARSE_TRACE:
trace_list = block_trace_list[index]['trace']
Traces.add_trace_list(
current_session=current_session,
trace_list=trace_list,
transaction_hash=transaction.transaction_hash,
transaction_index=transaction.transaction_index,
block_number=transaction.block_number,
timestamp=transaction.timestamp)
if current_session.settings.PARSE_STATE_DIFF:
state_diff_dict = block_trace_list[index]['stateDiff']
if state_diff_dict is not None:
StateDiff.add_state_diff_dict(
current_session=current_session,
state_diff_dict=state_diff_dict,
transaction_hash=transaction.transaction_hash,
transaction_index=transaction.transaction_index,
block_number=transaction.block_number,
timestamp=transaction.timestamp,
miner=block.miner,
fees=fees)
if block_number == 0:
StateDiff.parse_genesis_rewards(current_session=current_session,
block=block)
else:
StateDiff.add_mining_rewards(current_session=current_session,
block=block,
uncle_list=uncle_list)
# updating the meta info table
MetaInfo.set_last_pushed_block(current_session, block_number)
logger.info("Commiting block: {} to sql".format(block_number))
return block_hash
def add_block_number(block_number, ether_sql_session):
"""
Adds the block, transactions, uncles, logs and traces of a given block
number into the db_session
:param int block_number: The block number to add to the database
"""
# getting the block_data from the node
block_data = ether_sql_session.w3.eth.getBlock(
block_identifier=block_number, full_transactions=True)
timestamp = to_int(block_data['timestamp'])
iso_timestamp = datetime.utcfromtimestamp(timestamp).isoformat()
block = Blocks.add_block(block_data=block_data,
iso_timestamp=iso_timestamp)
ether_sql_session.db_session.add(
block) # added the block data in the db session
logger.debug('Reached this spot')
transaction_list = block_data['transactions']
# loop to get the transaction, receipts, logs and traces of the block
for transaction_data in transaction_list:
transaction = Transactions.add_transaction(transaction_data,
block_number=block_number,
iso_timestamp=iso_timestamp)
# added the transaction in the db session
ether_sql_session.db_session.add(transaction)
receipt_data = ether_sql_session.w3.eth.getTransactionReceipt(
transaction_data['hash'])
receipt = Receipts.add_receipt(receipt_data,
block_number=block_number,
timestamp=iso_timestamp)
ether_sql_session.db_session.add(
receipt) # added the receipt in the database
logs_list = receipt_data['logs']
for dict_log in logs_list:
log = Logs.add_log(dict_log,
block_number=block_number,
iso_timestamp=iso_timestamp)
ether_sql_session.db_session.add(
log) # adding the log in db session
if ether_sql_session.settings.PARSE_TRACE:
dict_trace_list = ether_sql_session.w3.parity.traceTransaction(
to_hex(transaction_data['hash']))
if dict_trace_list is not None:
for dict_trace in dict_trace_list:
trace = Traces.add_trace(dict_trace,
block_number=block_number,
timestamp=iso_timestamp)
ether_sql_session.db_session.add(
trace) # added the trace in the db session
uncle_list = block_data['uncles']
for i in range(0, len(uncle_list)):
# Unfortunately there is no command eth_getUncleByHash
uncle_data = ether_sql_session.w3.eth.getUncleByBlock(block_number, i)
uncle = Uncles.add_uncle(uncle_data=uncle_data,
block_number=block_number,
iso_timestamp=iso_timestamp)
ether_sql_session.db_session.add(uncle)
return ether_sql_session
