def fill_missing_blocks():
session = get_current_session()
runner = CliRunner()
runner.invoke(cli, ['--settings', session.setting_name,
'scrape_block_range',
'--end_block_number', 10,
'--fill_gaps'])
assert len(Blocks.missing_blocks(10)) == 0
def scrape_block_range(ctx, start_block_number, end_block_number, mode,
fill_gaps):
"""
Pushes the data between start_block_number and end_block_number in the
database. If no values are provided, the start_block_number is the last
block_number+1 in sql and end_block_number is the current block_number in
node. Also checks for missing blocks and adds them to the list of required
block numbers
:param int start_block_number: starting block number of scraping
:param int end_block_number: end block number of scraping
:param str mode: Mode of data sync 'parallel' or single
:param bool fill_gaps: If switched on instructs to also fill missing blocks
"""
current_session = get_current_session()
with current_session.db_session_scope():
sql_block_number = Blocks.get_max_block_number()
if end_block_number is None:
end_block_number = current_session.w3.eth.blockNumber
logger.debug(end_block_number)
if start_block_number is None:
if sql_block_number is None:
start_block_number = 0
elif sql_block_number == end_block_number:
start_block_number = sql_block_number
else:
start_block_number = sql_block_number + 1
logger.debug(start_block_number)
# casting numbers to integers
if start_block_number == end_block_number:
list_block_numbers = []
else:
start_block_number = int(start_block_number)
end_block_number = int(end_block_number)
list_block_numbers = list(
range(start_block_number, end_block_number + 1))
if fill_gaps and start_block_number != 0:
missing_blocks = Blocks.missing_blocks(sql_block_number)
logger.debug(missing_blocks)
logger.info('{} missing blocks detected'.format(len(missing_blocks)))
for missing in missing_blocks:
logger.debug(missing.block_number)
list_block_numbers.append(missing.block_number)
logger.debug(list_block_numbers)
if len(list_block_numbers) == 0:
logger.warning('No blocks pushed in database')
if mode == 'parallel':
scrape_blocks(list_block_numbers=list_block_numbers, mode=mode)
elif mode == 'single':
scrape_blocks(list_block_numbers=list_block_numbers, mode=mode)
else:
raise ValueError('The mode: {} is not recognized'.format(mode))
def final_missing_blocks(list_blocks=list_final_missing_blocks):
final_missing_blocks = Blocks.missing_blocks(max_block_number=15)
assert len(final_missing_blocks) == len(list_blocks)
for index, result in enumerate(final_missing_blocks):
assert result.block_number == list_blocks[index]
def initial_missing_blocks(list_blocks=list_initial_missing_blocks):
initial_missing_blocks = Blocks.missing_blocks()
assert len(initial_missing_blocks) == len(list_blocks)
for index, result in enumerate(initial_missing_blocks):
assert result.block_number == list_blocks[index]
def get_state_at_block(cls, block_number=None):
"""
Updates the state with either specified `block_number` or the maximum
``block_number`` in the database. Also, checks if there are any missing
blocks in the database, if yes then stops the calculation of state
prematurely.
: param int block_number: Block number of desired state, if none then
constructs the state for the highest available state_diff
"""
# checks if there are any gaps in the block_numbers
if block_number is None:
block_number = Blocks.get_max_block_number()
missing_block_numbers = Blocks.missing_blocks(block_number)
if len(missing_block_numbers) > 0:
raise MissingBlocksError('Cannot construct state at block {}, \
{} blocks are missing'.format(block_number,
len(missing_block_numbers)))
if MetaInfo.get_current_state_block == block_number:
logger.info('State is already at block {}'.format(block_number))
return
current_session = get_current_session()
with current_session.db_session_scope():
current_session.db_session.query(Storage).delete()
current_session.db_session.query(cls).delete()
# query to get the balance
query_balance = current_session.db_session.query(
StateDiff.address,
func.sum(StateDiff.balance_diff).label('balance'),
func.sum(StateDiff.nonce_diff).label('nonce')).\
filter(StateDiff.block_number <= block_number).\
group_by(StateDiff.address)
subquery_balance = query_balance.subquery()
# query to get the code
row_number_column = func.row_number().over(
partition_by=StateDiff.address,
order_by=[StateDiff.block_number.desc(),
StateDiff.transaction_index.desc()])\
.label('row_number')
query_code = current_session.db_session.query(
StateDiff.address.label('address'),
StateDiff.code_to.label('code'))
query_code = query_code.add_column(row_number_column)
query_code = query_code.filter(
or_(StateDiff.code_from != None, StateDiff.code_to != None))
query_code = query_code.filter(
StateDiff.block_number <= block_number)
query_code = query_code.from_self().filter(row_number_column == 1)
query_code = query_code.filter(StateDiff.code_to != '0x')
subquery_code = query_code.subquery()
# joining the two queries
query_state = current_session.db_session.query(
subquery_balance.c.address, subquery_balance.c.balance,
subquery_balance.c.nonce, subquery_code.c.code)
query_state = query_state.outerjoin(
subquery_code,
subquery_balance.c.address == subquery_code.c.address)
# updating the state table with query results
for row in query_state:
state = cls.add_state(address=row.address,
balance=row.balance,
nonce=row.nonce,
code=row.code)
current_session.db_session.add(state)
# update the meta_info.current_state_block
MetaInfo.set_current_state_block(current_session, block_number)
with current_session.db_session_scope():
# update the storage table
Storage.get_storage_at_block(current_session, block_number)