def verify_pushed_block_56160_contents():
current_session = get_current_session()
with current_session.db_session_scope():
block_task_meta = current_session.db_session.query(BlockTaskMeta).first()
task_id = block_task_meta.task_id
task_completed = AsyncResult(task_id).get()
verify_block_56160_contents()
def get_max_block_number(cls):
current_session = get_current_session()
with current_session.db_session_scope():
max_block_number = current_session.db_session.query(
func.max(cls.block_number)).scalar()
return max_block_number
def verify_block_range_56160_56170():
session = get_current_session()
with session.db_session_scope():
logger.debug('Total blocks {}'.format(session.db_session.query(Blocks).count()))
assert session.db_session.query(Blocks).count() == 11
assert session.db_session.query(Transactions).count() == 3
assert session.db_session.query(Receipts).count() == 3
assert session.db_session.query(Logs).count() == 1
assert session.db_session.query(Uncles).count() == 1
# assert session.db_session.query(MetaInfo).count() == 1
if session.settings.PARSE_TRACE:
assert session.db_session.query(Traces).count() == 3
if session.settings.PARSE_STATE_DIFF:
assert session.db_session.query(StateDiff).count() == 20
assert session.db_session.query(StorageDiff).count() == 2
assert session.db_session.query(StateDiff).filter_by(
state_diff_type='miner').count() == 11
assert session.db_session.query(StateDiff).filter_by(
state_diff_type='fees').count() == 3
assert session.db_session.query(StateDiff).filter_by(
state_diff_type='sender').count() == 3
assert session.db_session.query(StateDiff).filter_by(
state_diff_type='uncle').count() == 1
def on_failure(self, exc, task_id, args, kwargs, einfo):
current_session = get_current_session()
with current_session.db_session_scope():
block_task_meta = BlockTaskMeta.get_block_task_meta_from_task_id(
task_id)
block_task_meta.state = 'FAILURE'
current_session.db_session.add(block_task_meta)
def drop_tables(ctx):
""" Alias for 'alembic downgrade base'.
Downgrade to no database tables
"""
current_session = get_current_session()
command.downgrade(setup_alembic_config(url=current_session.url),
revision='base', sql=False, tag=None)
def scrape_blocks(list_block_numbers, mode):
"""
Function which starts scrapping data from the node and pushes it into
the sql database
:param list list_block_numbers: List of block numbers to push in the database
:param str mode: Mode to be used weather parallel or single
"""
task_list = []
current_session = get_current_session()
for block_number in list_block_numbers:
logger.debug('Adding block: {}'.format(block_number))
if mode == 'parallel':
r = add_block_number.delay(block_number)
task_list.append(r)
block_task_meta = BlockTaskMeta(task_id=r.id,
task_name='add_block_number',
state='PENDING',
block_number=block_number)
with current_session.db_session_scope():
current_session.db_session.add(block_task_meta)
elif mode == 'single':
add_block_number(block_number)
else:
raise ValueError('Mode {} is unavailable'.format(mode))
return task_list
def verify_block_range_56160_56170():
session = get_current_session()
with session.db_session_scope():
logger.debug('Total blocks {}'.format(session.db_session.query(Blocks).count()))
assert session.db_session.query(Blocks).count() == 11
assert session.db_session.query(Transactions).count() == 3
assert session.db_session.query(Receipts).count() == 3
assert session.db_session.query(Logs).count() == 1
assert session.db_session.query(Uncles).count() == 1
# assert session.db_session.query(MetaInfo).count() == 1
number_of_rows_in_meta_info = session.db_session.\
query(MetaInfo).count()
meta_info_properties_in_sql = session.db_session.\
query(MetaInfo).first().to_dict()
assert number_of_rows_in_meta_info == 1
print(meta_info_properties_in_sql)
assert meta_info_properties_in_sql == EXPECTED_META_INFO
if session.settings.PARSE_TRACE:
assert session.db_session.query(Traces).count() == 3
if session.settings.PARSE_STATE_DIFF:
assert session.db_session.query(StateDiff).count() == 20
assert session.db_session.query(StorageDiff).count() == 2
assert session.db_session.query(StateDiff).filter_by(
state_diff_type='miner').count() == 11
assert session.db_session.query(StateDiff).filter_by(
state_diff_type='fees').count() == 3
assert session.db_session.query(StateDiff).filter_by(
state_diff_type='sender').count() == 3
assert session.db_session.query(StateDiff).filter_by(
state_diff_type='uncle').count() == 1
def upgrade_tables(ctx):
""" Alias for 'alembic upgrade head'.
Upgrade to latest model version
"""
current_session = get_current_session()
command.upgrade(setup_alembic_config(url=current_session.url),
revision='head', sql=False, tag=None)
def verify_block_56160_contents():
# comparing values of blocks
session = get_current_session()
with session.db_session_scope():
number_of_rows_in_meta_info = session.db_session.\
query(MetaInfo).count()
meta_info_properties_in_sql = session.db_session.\
query(MetaInfo).first().to_dict()
assert number_of_rows_in_meta_info == 1
print(meta_info_properties_in_sql)
assert meta_info_properties_in_sql == EXPECTED_META_INFO
block_properties_in_sql = session.db_session.\
query(Blocks).filter_by(block_number=56160).first().to_dict()
assert block_properties_in_sql == EXPECTED_BLOCK_PROPERTIES
# comparing values of uncles
uncle_properties_in_sql = session.db_session.\
query(Uncles).filter_by(current_blocknumber=56160).first().to_dict()
assert uncle_properties_in_sql == EXPECTED_UNCLE_PROPERTIES
# comparing values of transactions
transaction_properties_in_sql = session.db_session.\
query(Transactions).filter_by(block_number=56160).first().to_dict()
assert transaction_properties_in_sql == EXPEXTED_TRANSACTION_PROPERTIES
# comparing values of receipts
receipt_properties_in_sql = session.db_session.\
query(Receipts).filter_by(block_number=56160).first().to_dict()
assert receipt_properties_in_sql == EXPECTED_RECEIPT_PROPERTIES
# comparing values of logs
log_properties_in_sql = session.db_session.\
query(Logs).filter_by(block_number=56160).first().to_dict()
assert log_properties_in_sql == EXPECTED_LOG_PROPERTIES
# comparing values of traces
if session.settings.PARSE_TRACE:
trace_properties_in_sql = session.\
db_session.query(Traces).filter_by(block_number=56160).first().\
to_dict()
assert trace_properties_in_sql == EXPECTED_TRACE_PROPERTIES
# comparing values of states
if session.settings.PARSE_STATE_DIFF:
# comparing values if state diffs
state_diff_property_in_sql = session.\
db_session.query(StateDiff).filter_by(block_number=56160).all()
for i in range(0, len(state_diff_property_in_sql)):
assert state_diff_property_in_sql[i].to_dict() == \
EXPECTED_STATE_DIFF_PROPERTIES[i]
# comparing values of storage_diffs
storage_diff_property_in_sql = session.\
db_session.query(StorageDiff).filter_by(block_number=56160).all()
for i in range(0, len(storage_diff_property_in_sql)):
assert storage_diff_property_in_sql[i].to_dict() == \
EXPECTED_STORAGE_DIFF_PROPERTIES[i]
def fill_missing_blocks():
session = get_current_session()
runner = CliRunner()
runner.invoke(cli, ['--settings', session.setting_name,
'scrape_block_range',
'--end_block_number', 10,
'--fill_gaps'])
assert len(Blocks.missing_blocks(10)) == 0
def get_blocks_to_be_pushed_in_queue(cls, current_session):
current_session = get_current_session()
current_eth_blocknumber = current_session.w3.eth.blockNumber
block_lag = current_session.settings.BLOCK_LAG
query = current_session.db_session.query(cls.block_number).filter(
and_(cls.state == 'WAITING',
cls.block_number < current_eth_blocknumber - block_lag))
return query.from_self().distinct()
def push_initial_blocks_in_queue():
current_session = get_current_session()
push_blocks_in_queue()
with current_session.db_session_scope():
block_task_meta = current_session.db_session.query(BlockTaskMeta).first()
assert block_task_meta.block_number == BASE_BLOCK
assert block_task_meta.block_hash == EXPECTED_BLOCK_HASHES[BASE_BLOCK]
assert block_task_meta.state == 'PENDING'
def close_celery_session(**kwargs):
current_session = get_current_session()
try:
current_session.db_session.close
except AttributeError:
logger.debug('db_session attribute does not exist')
finally:
pass
def scrape_block_range(ctx, start_block_number, end_block_number, mode,
fill_gaps):
"""
Pushes the data between start_block_number and end_block_number in the
database. If no values are provided, the start_block_number is the last
block_number+1 in sql and end_block_number is the current block_number in
node. Also checks for missing blocks and adds them to the list of required
block numbers
:param int start_block_number: starting block number of scraping
:param int end_block_number: end block number of scraping
:param str mode: Mode of data sync 'parallel' or single
:param bool fill_gaps: If switched on instructs to also fill missing blocks
"""
current_session = get_current_session()
with current_session.db_session_scope():
sql_block_number = Blocks.get_max_block_number()
if end_block_number is None:
end_block_number = current_session.w3.eth.blockNumber
logger.debug(end_block_number)
if start_block_number is None:
if sql_block_number is None:
start_block_number = 0
elif sql_block_number == end_block_number:
start_block_number = sql_block_number
else:
start_block_number = sql_block_number + 1
logger.debug(start_block_number)
# casting numbers to integers
if start_block_number == end_block_number:
list_block_numbers = []
else:
start_block_number = int(start_block_number)
end_block_number = int(end_block_number)
list_block_numbers = list(
range(start_block_number, end_block_number + 1))
if fill_gaps and start_block_number != 0:
missing_blocks = Blocks.missing_blocks(sql_block_number)
logger.debug(missing_blocks)
logger.info('{} missing blocks detected'.format(len(missing_blocks)))
for missing in missing_blocks:
logger.debug(missing.block_number)
list_block_numbers.append(missing.block_number)
logger.debug(list_block_numbers)
if len(list_block_numbers) == 0:
logger.warning('No blocks pushed in database')
if mode == 'parallel':
scrape_blocks(list_block_numbers=list_block_numbers, mode=mode)
elif mode == 'single':
scrape_blocks(list_block_numbers=list_block_numbers, mode=mode)
else:
raise ValueError('The mode: {} is not recognized'.format(mode))
def migrate(ctx, message):
""" Alias for 'alembic revision --autogenerate'
Run this command after changing sql tables
"""
current_session = get_current_session()
if message is None:
click.echo(ctx.get_help())
else:
command.revision(setup_alembic_config(url=current_session.url),
message=message, autogenerate=True, sql=None)
def on_success(self, retval, task_id, args, kwargs):
current_session = get_current_session()
with current_session.db_session_scope():
block_task_meta = BlockTaskMeta.get_block_task_meta_from_task_id(
task_id)
for i_block_task_meta in block_task_meta:
if i_block_task_meta.block_hash == retval or i_block_task_meta is None:
i_block_task_meta.state = 'SUCCESS'
current_session.db_session.add(i_block_task_meta)
else:
i_block_task_meta.state = 'FORKED'
current_session.db_session.add(i_block_task_meta)
def remove_block_number(block_number):
"""
Removes the block, transactions, uncles, logs and traces of a given block
number into the database to perform chain reorgs.
:param int block_number: The block number to add to the database
"""
current_session = get_current_session()
with current_session.db_session_scope():
current_session.db_session.query(Blocks).\
filter_by(block_number=block_number).delete()
logger.info("Removed block {}".format(block_number))
def update_block_task_meta_from_block_number(cls, block_number, **kwargs):
current_session = get_current_session()
with current_session.db_session_scope():
block_task_meta = current_session.db_session.query(cls).\
filter_by(block_number=block_number)
logger.debug(
'Updating task meta of block {0}'.format(block_number))
for i_block_task_meta in block_task_meta:
for key, value in kwargs.items():
setattr(i_block_task_meta, key, value)
logger.debug('Updated task meta {}'.format(
i_block_task_meta.to_dict()))
def verify_removed_block_range_56160_56170():
for i in range(56160, 56171):
remove_block_number(i)
session = get_current_session()
with session.db_session_scope():
assert session.db_session.query(Blocks).count() == 0
assert session.db_session.query(Transactions).count() == 0
assert session.db_session.query(Receipts).count() == 0
assert session.db_session.query(Logs).count() == 0
assert session.db_session.query(Uncles).count() == 0
assert session.db_session.query(Traces).count() == 0
assert session.db_session.query(StateDiff).count() == 0
assert session.db_session.query(StorageDiff).count() == 0
def add_block_task_meta(cls,
block_number,
task_name,
state,
task_id=None,
block_hash=None):
current_session = get_current_session()
block_task_meta = cls(task_name=task_name,
state=state,
block_number=block_number,
block_hash=block_hash,
task_id=task_id)
with current_session.db_session_scope():
current_session.db_session.add(block_task_meta)
def new_blocks():
"""
Celery beat task which runs every second to get new blocks.
:param block_filter: block filter as described in session.py
"""
current_session = get_current_session()
logger.debug("Reached at new blocks to get block hashes")
block_hashes = current_session.block_filter.get_new_entries()
for block_hash in block_hashes:
block_data = current_session.w3.eth.getBlock(block_hash)
block_number = to_int(block_data['number'])
BlockTaskMeta.add_block_task_meta(task_name='new_blocks',
state='WAITING',
block_number=block_number,
block_hash=to_hex(block_hash))
logger.info(block_hashes)
def export_to_csv_single_thread():
session = get_current_session()
directory = 'test_export'
call(["rm", "-rf", directory])
runner = CliRunner()
result = runner.invoke(cli, [
'--settings', session.setting_name, 'sql', 'export_to_csv',
'--directory', directory
])
assert result.exit_code == 0
# match the names of exported tables
tables_in_sql = list(base.metadata.tables)
files_in_directory = os.listdir(directory)
for sql_table in tables_in_sql:
assert sql_table + '.csv' in files_in_directory
call(["rm", "-rf", directory])
def add_dao_hardfork_state_diff():
current_session = get_current_session()
with current_session.db_session_scope():
for dict_account in DAO_DRAIN_LIST:
state_diff = StateDiff.add_state_diff(
balance_diff=dict_account['balance_diff'],
nonce_diff=None,
code_from=None,
code_to=None,
address=to_checksum_address(dict_account['address']),
transaction_hash=None,
transaction_index=None,
block_number=MAINNET_DAO_BLOCK,
timestamp=MAINNET_DAO_TIMESTAMP,
state_diff_type='dao-fork')
current_session.db_session.add(state_diff)
def match_state_dump_to_state_table(block_number):
current_session = get_current_session()
with open('tests/fixtures/balance/balance_{}.json'.format(block_number)) as data_file:
data = json.loads(data_file.read())
state = data['state']
with current_session.db_session_scope():
for address in state:
state_table_row = current_session.db_session.query(State).\
filter_by(address=to_checksum_address(address)).first()
assert state_table_row.balance == hex_to_integer(state[address]['balance'])
assert state_table_row.nonce == hex_to_integer(state[address]['nonce'])
if 'code' in state[address].keys():
assert state_table_row.code == "0x"+state[address]['code']
if state_table_row.storage is not None:
for storage in state_table_row.storage:
storage.storage = state[address]['storage'][storage.position]
def push_blocks_in_queue():
"""
Celery beat task which runs every 30 second to check for blocks which are
settings.BLOCK_LAG number of blocks behind the current ethereum client and
pushes the blocks in waiting to the queue.
"""
current_session = get_current_session()
blocks_in_waiting = BlockTaskMeta.get_blocks_to_be_pushed_in_queue()
for blocks in blocks_in_waiting:
block_number = int(blocks.block_number)
add_block_task = add_block_number.delay(block_number)
BlockTaskMeta.update_block_task_meta_from_block_number(
block_number=block_number,
task_id=add_block_task.id,
state='PENDING',
task_name='add_block_number')
def missing_blocks(cls, max_block_number=None):
"""
Return missing blocks in the blocks table between 0 to block_number
:param int max_block_number: Maximum block number that we want to find missing blocks from
"""
current_session = get_current_session()
if max_block_number is None:
max_block_number = cls.get_max_block_number()
logger.debug(max_block_number)
with current_session.db_session_scope():
stmt = current_session.db_session.query(
func.generate_series(0, max_block_number).label('i')).subquery()
joined = stmt.outerjoin(cls, cls.block_number == stmt.c.i)
missing_blocks = current_session.db_session.query(stmt.c.i.label('block_number')).\
select_from(joined).filter(cls.block_number == None).all()
return missing_blocks
def put_initial_blocks_in_waiting():
current_session = get_current_session()
put_blocks = range(BASE_BLOCK, BASE_BLOCK + current_session.settings.BLOCK_LAG)
for block_number in put_blocks:
block_data = current_session.w3.eth.getBlock(block_number)
block_hash = block_data['hash']
BlockTaskMeta.add_block_task_meta(task_name='new_blocks',
state='WAITING',
block_number=block_number,
block_hash=to_hex(block_hash))
verify_pushed_sql_contents()
with current_session.db_session_scope():
block_task_meta = current_session.db_session.query(BlockTaskMeta).first()
assert block_task_meta.block_number == BASE_BLOCK
assert block_task_meta.block_hash == EXPECTED_BLOCK_HASHES[BASE_BLOCK]
assert block_task_meta.state == 'WAITING'
def verify_pushed_sql_contents():
current_session = get_current_session()
with current_session.db_session_scope():
assert current_session.db_session.query(Blocks).count() == 0
assert current_session.db_session.query(Transactions).count() == 0
assert current_session.db_session.query(Receipts).count() == 0
assert current_session.db_session.query(Logs).count() == 0
assert current_session.db_session.query(Uncles).count() == 0
if current_session.settings.PARSE_TRACE:
assert current_session.db_session.query(Traces).count() == 0
if current_session.settings.PARSE_STATE_DIFF:
assert current_session.db_session.query(StateDiff).count() == 0
assert current_session.db_session.query(StorageDiff).count() == 0
assert current_session.db_session.query(BlockTaskMeta).count() == \
current_session.settings.BLOCK_LAG
def get_block_task_meta_from_block_hash(cls, block_hash):
current_session = get_current_session()
with current_session.db_session_scope():
return current_session.db_session.query(cls).\
filter_by(block_hash=block_hash)
def get_block_task_meta_from_task_id(cls, task_id):
current_session = get_current_session()
with current_session.db_session_scope():
return current_session.db_session.query(cls).\
filter_by(task_id=task_id)
