def scrape_block_range(ctx, start_block_number, end_block_number, mode,
fill_gaps):
"""
Pushes the data between start_block_number and end_block_number in the
database. If no values are provided, the start_block_number is the last
block_number+1 in sql and end_block_number is the current block_number in
node. Also checks for missing blocks and adds them to the list of required
block numbers
:param int start_block_number: starting block number of scraping
:param int end_block_number: end block number of scraping
:param str mode: Mode of data sync 'parallel' or single
:param bool fill_gaps: If switched on instructs to also fill missing blocks
"""
current_session = get_current_session()
with current_session.db_session_scope():
sql_block_number = Blocks.get_max_block_number()
if end_block_number is None:
end_block_number = current_session.w3.eth.blockNumber
logger.debug(end_block_number)
if start_block_number is None:
if sql_block_number is None:
start_block_number = 0
elif sql_block_number == end_block_number:
start_block_number = sql_block_number
else:
start_block_number = sql_block_number + 1
logger.debug(start_block_number)
# casting numbers to integers
if start_block_number == end_block_number:
list_block_numbers = []
else:
start_block_number = int(start_block_number)
end_block_number = int(end_block_number)
list_block_numbers = list(
range(start_block_number, end_block_number + 1))
if fill_gaps and start_block_number != 0:
missing_blocks = Blocks.missing_blocks(sql_block_number)
logger.debug(missing_blocks)
logger.info('{} missing blocks detected'.format(len(missing_blocks)))
for missing in missing_blocks:
logger.debug(missing.block_number)
list_block_numbers.append(missing.block_number)
logger.debug(list_block_numbers)
if len(list_block_numbers) == 0:
logger.warning('No blocks pushed in database')
if mode == 'parallel':
scrape_blocks(list_block_numbers=list_block_numbers, mode=mode)
elif mode == 'single':
scrape_blocks(list_block_numbers=list_block_numbers, mode=mode)
else:
raise ValueError('The mode: {} is not recognized'.format(mode))
def fill_missing_blocks():
session = get_current_session()
runner = CliRunner()
runner.invoke(cli, ['--settings', session.setting_name,
'scrape_block_range',
'--end_block_number', 10,
'--fill_gaps'])
assert len(Blocks.missing_blocks(10)) == 0
def final_missing_blocks(list_blocks=list_final_missing_blocks):
final_missing_blocks = Blocks.missing_blocks(max_block_number=15)
assert len(final_missing_blocks) == len(list_blocks)
for index, result in enumerate(final_missing_blocks):
assert result.block_number == list_blocks[index]
def initial_missing_blocks(list_blocks=list_initial_missing_blocks):
initial_missing_blocks = Blocks.missing_blocks()
assert len(initial_missing_blocks) == len(list_blocks)
for index, result in enumerate(initial_missing_blocks):
assert result.block_number == list_blocks[index]
def add_block_number(block_number):
"""
Adds the block, transactions, uncles, logs and traces of a given block
number into the db_session
:param int block_number: The block number to add to the database
"""
current_session = get_current_session()
block_task_meta = BlockTaskMeta.update_block_task_meta_from_block_number(
block_number=block_number, state='STARTED')
# getting the block_data from the node
block_data = current_session.w3.eth.getBlock(block_identifier=block_number,
full_transactions=True)
timestamp = to_int(block_data['timestamp'])
iso_timestamp = datetime.utcfromtimestamp(timestamp).isoformat()
block = Blocks.add_block(block_data=block_data,
iso_timestamp=iso_timestamp)
block_hash = block.block_hash
if current_session.settings.PARSE_TRACE and \
current_session.settings.PARSE_STATE_DIFF and \
block_number != 0:
block_trace_list = current_session.w3.parity.\
traceReplayBlockTransactions(block_number,
mode=['trace', 'stateDiff'])
elif block_number != 0:
if current_session.settings.PARSE_TRACE:
block_trace_list = current_session.w3.parity.\
traceReplayBlockTransactions(block_number,
mode=['trace'])
if current_session.settings.PARSE_STATE_DIFF:
block_trace_list = current_session.w3.parity.\
traceReplayBlockTransactions(block_number,
mode=['stateDiff'])
# added the block data in the db session
with current_session.db_session_scope():
current_session.db_session.add(block)
uncle_hashes = block_data['uncles']
uncle_list = []
for i in range(0, len(uncle_hashes)):
# Unfortunately there is no command eth_getUncleByHash
uncle_data = current_session.w3.eth.getUncleByBlock(
block_number, i)
uncle = Uncles.add_uncle(uncle_data=uncle_data,
block_number=block_number,
iso_timestamp=iso_timestamp)
current_session.db_session.add(uncle)
uncle_list.append(uncle)
transaction_list = block_data['transactions']
# loop to get the transaction, receipts, logs and traces of the block
for index, transaction_data in enumerate(transaction_list):
transaction = Transactions.add_transaction(
transaction_data,
block_number=block_number,
iso_timestamp=iso_timestamp)
# added the transaction in the db session
current_session.db_session.add(transaction)
receipt_data = current_session.w3.eth.getTransactionReceipt(
transaction_data['hash'])
receipt = Receipts.add_receipt(receipt_data,
block_number=block_number,
timestamp=iso_timestamp)
current_session.db_session.add(receipt)
fees = int(transaction.gas_price) * int(receipt.gas_used)
log_list = receipt_data['logs']
Logs.add_log_list(current_session=current_session,
log_list=log_list,
block_number=block_number,
timestamp=transaction.timestamp)
if current_session.settings.PARSE_TRACE:
trace_list = block_trace_list[index]['trace']
Traces.add_trace_list(
current_session=current_session,
trace_list=trace_list,
transaction_hash=transaction.transaction_hash,
transaction_index=transaction.transaction_index,
block_number=transaction.block_number,
timestamp=transaction.timestamp)
if current_session.settings.PARSE_STATE_DIFF:
state_diff_dict = block_trace_list[index]['stateDiff']
if state_diff_dict is not None:
StateDiff.add_state_diff_dict(
current_session=current_session,
state_diff_dict=state_diff_dict,
transaction_hash=transaction.transaction_hash,
transaction_index=transaction.transaction_index,
block_number=transaction.block_number,
timestamp=transaction.timestamp,
miner=block.miner,
fees=fees)
if block_number == 0:
StateDiff.parse_genesis_rewards(current_session=current_session,
block=block)
else:
StateDiff.add_mining_rewards(current_session=current_session,
block=block,
uncle_list=uncle_list)
# updating the meta info table
MetaInfo.set_last_pushed_block(current_session, block_number)
logger.info("Commiting block: {} to sql".format(block_number))
return block_hash
def blockNumber(ctx):
""" Gives the current highest block in database"""
click.echo(Blocks.get_max_block_number())
def add_block_number(block_number, ether_sql_session):
"""
Adds the block, transactions, uncles, logs and traces of a given block
number into the db_session
:param int block_number: The block number to add to the database
"""
# getting the block_data from the node
block_data = ether_sql_session.w3.eth.getBlock(
block_identifier=block_number, full_transactions=True)
timestamp = to_int(block_data['timestamp'])
iso_timestamp = datetime.utcfromtimestamp(timestamp).isoformat()
block = Blocks.add_block(block_data=block_data,
iso_timestamp=iso_timestamp)
ether_sql_session.db_session.add(
block) # added the block data in the db session
logger.debug('Reached this spot')
transaction_list = block_data['transactions']
# loop to get the transaction, receipts, logs and traces of the block
for transaction_data in transaction_list:
transaction = Transactions.add_transaction(transaction_data,
block_number=block_number,
iso_timestamp=iso_timestamp)
# added the transaction in the db session
ether_sql_session.db_session.add(transaction)
receipt_data = ether_sql_session.w3.eth.getTransactionReceipt(
transaction_data['hash'])
receipt = Receipts.add_receipt(receipt_data,
block_number=block_number,
timestamp=iso_timestamp)
ether_sql_session.db_session.add(
receipt) # added the receipt in the database
logs_list = receipt_data['logs']
for dict_log in logs_list:
log = Logs.add_log(dict_log,
block_number=block_number,
iso_timestamp=iso_timestamp)
ether_sql_session.db_session.add(
log) # adding the log in db session
if ether_sql_session.settings.PARSE_TRACE:
dict_trace_list = ether_sql_session.w3.parity.traceTransaction(
to_hex(transaction_data['hash']))
if dict_trace_list is not None:
for dict_trace in dict_trace_list:
trace = Traces.add_trace(dict_trace,
block_number=block_number,
timestamp=iso_timestamp)
ether_sql_session.db_session.add(
trace) # added the trace in the db session
uncle_list = block_data['uncles']
for i in range(0, len(uncle_list)):
# Unfortunately there is no command eth_getUncleByHash
uncle_data = ether_sql_session.w3.eth.getUncleByBlock(block_number, i)
uncle = Uncles.add_uncle(uncle_data=uncle_data,
block_number=block_number,
iso_timestamp=iso_timestamp)
ether_sql_session.db_session.add(uncle)
return ether_sql_session
def get_state_at_block(cls, block_number=None):
"""
Updates the state with either specified `block_number` or the maximum
``block_number`` in the database. Also, checks if there are any missing
blocks in the database, if yes then stops the calculation of state
prematurely.
: param int block_number: Block number of desired state, if none then
constructs the state for the highest available state_diff
"""
# checks if there are any gaps in the block_numbers
if block_number is None:
block_number = Blocks.get_max_block_number()
missing_block_numbers = Blocks.missing_blocks(block_number)
if len(missing_block_numbers) > 0:
raise MissingBlocksError('Cannot construct state at block {}, \
{} blocks are missing'.format(block_number,
len(missing_block_numbers)))
if MetaInfo.get_current_state_block == block_number:
logger.info('State is already at block {}'.format(block_number))
return
current_session = get_current_session()
with current_session.db_session_scope():
current_session.db_session.query(Storage).delete()
current_session.db_session.query(cls).delete()
# query to get the balance
query_balance = current_session.db_session.query(
StateDiff.address,
func.sum(StateDiff.balance_diff).label('balance'),
func.sum(StateDiff.nonce_diff).label('nonce')).\
filter(StateDiff.block_number <= block_number).\
group_by(StateDiff.address)
subquery_balance = query_balance.subquery()
# query to get the code
row_number_column = func.row_number().over(
partition_by=StateDiff.address,
order_by=[StateDiff.block_number.desc(),
StateDiff.transaction_index.desc()])\
.label('row_number')
query_code = current_session.db_session.query(
StateDiff.address.label('address'),
StateDiff.code_to.label('code'))
query_code = query_code.add_column(row_number_column)
query_code = query_code.filter(
or_(StateDiff.code_from != None, StateDiff.code_to != None))
query_code = query_code.filter(
StateDiff.block_number <= block_number)
query_code = query_code.from_self().filter(row_number_column == 1)
query_code = query_code.filter(StateDiff.code_to != '0x')
subquery_code = query_code.subquery()
# joining the two queries
query_state = current_session.db_session.query(
subquery_balance.c.address, subquery_balance.c.balance,
subquery_balance.c.nonce, subquery_code.c.code)
query_state = query_state.outerjoin(
subquery_code,
subquery_balance.c.address == subquery_code.c.address)
# updating the state table with query results
for row in query_state:
state = cls.add_state(address=row.address,
balance=row.balance,
nonce=row.nonce,
code=row.code)
current_session.db_session.add(state)
# update the meta_info.current_state_block
MetaInfo.set_current_state_block(current_session, block_number)
with current_session.db_session_scope():
# update the storage table
Storage.get_storage_at_block(current_session, block_number)