def verify_pushed_block_56160_contents(): current_session = get_current_session() with current_session.db_session_scope(): block_task_meta = current_session.db_session.query(BlockTaskMeta).first() task_id = block_task_meta.task_id task_completed = AsyncResult(task_id).get() verify_block_56160_contents()
def get_max_block_number(cls): current_session = get_current_session() with current_session.db_session_scope(): max_block_number = current_session.db_session.query( func.max(cls.block_number)).scalar() return max_block_number
def verify_block_range_56160_56170(): session = get_current_session() with session.db_session_scope(): logger.debug('Total blocks {}'.format(session.db_session.query(Blocks).count())) assert session.db_session.query(Blocks).count() == 11 assert session.db_session.query(Transactions).count() == 3 assert session.db_session.query(Receipts).count() == 3 assert session.db_session.query(Logs).count() == 1 assert session.db_session.query(Uncles).count() == 1 # assert session.db_session.query(MetaInfo).count() == 1 if session.settings.PARSE_TRACE: assert session.db_session.query(Traces).count() == 3 if session.settings.PARSE_STATE_DIFF: assert session.db_session.query(StateDiff).count() == 20 assert session.db_session.query(StorageDiff).count() == 2 assert session.db_session.query(StateDiff).filter_by( state_diff_type='miner').count() == 11 assert session.db_session.query(StateDiff).filter_by( state_diff_type='fees').count() == 3 assert session.db_session.query(StateDiff).filter_by( state_diff_type='sender').count() == 3 assert session.db_session.query(StateDiff).filter_by( state_diff_type='uncle').count() == 1
def on_failure(self, exc, task_id, args, kwargs, einfo): current_session = get_current_session() with current_session.db_session_scope(): block_task_meta = BlockTaskMeta.get_block_task_meta_from_task_id( task_id) block_task_meta.state = 'FAILURE' current_session.db_session.add(block_task_meta)
def drop_tables(ctx): """ Alias for 'alembic downgrade base'. Downgrade to no database tables """ current_session = get_current_session() command.downgrade(setup_alembic_config(url=current_session.url), revision='base', sql=False, tag=None)
def scrape_blocks(list_block_numbers, mode): """ Function which starts scrapping data from the node and pushes it into the sql database :param list list_block_numbers: List of block numbers to push in the database :param str mode: Mode to be used weather parallel or single """ task_list = [] current_session = get_current_session() for block_number in list_block_numbers: logger.debug('Adding block: {}'.format(block_number)) if mode == 'parallel': r = add_block_number.delay(block_number) task_list.append(r) block_task_meta = BlockTaskMeta(task_id=r.id, task_name='add_block_number', state='PENDING', block_number=block_number) with current_session.db_session_scope(): current_session.db_session.add(block_task_meta) elif mode == 'single': add_block_number(block_number) else: raise ValueError('Mode {} is unavailable'.format(mode)) return task_list
def verify_block_range_56160_56170(): session = get_current_session() with session.db_session_scope(): logger.debug('Total blocks {}'.format(session.db_session.query(Blocks).count())) assert session.db_session.query(Blocks).count() == 11 assert session.db_session.query(Transactions).count() == 3 assert session.db_session.query(Receipts).count() == 3 assert session.db_session.query(Logs).count() == 1 assert session.db_session.query(Uncles).count() == 1 # assert session.db_session.query(MetaInfo).count() == 1 number_of_rows_in_meta_info = session.db_session.\ query(MetaInfo).count() meta_info_properties_in_sql = session.db_session.\ query(MetaInfo).first().to_dict() assert number_of_rows_in_meta_info == 1 print(meta_info_properties_in_sql) assert meta_info_properties_in_sql == EXPECTED_META_INFO if session.settings.PARSE_TRACE: assert session.db_session.query(Traces).count() == 3 if session.settings.PARSE_STATE_DIFF: assert session.db_session.query(StateDiff).count() == 20 assert session.db_session.query(StorageDiff).count() == 2 assert session.db_session.query(StateDiff).filter_by( state_diff_type='miner').count() == 11 assert session.db_session.query(StateDiff).filter_by( state_diff_type='fees').count() == 3 assert session.db_session.query(StateDiff).filter_by( state_diff_type='sender').count() == 3 assert session.db_session.query(StateDiff).filter_by( state_diff_type='uncle').count() == 1
def upgrade_tables(ctx): """ Alias for 'alembic upgrade head'. Upgrade to latest model version """ current_session = get_current_session() command.upgrade(setup_alembic_config(url=current_session.url), revision='head', sql=False, tag=None)
def verify_block_56160_contents(): # comparing values of blocks session = get_current_session() with session.db_session_scope(): number_of_rows_in_meta_info = session.db_session.\ query(MetaInfo).count() meta_info_properties_in_sql = session.db_session.\ query(MetaInfo).first().to_dict() assert number_of_rows_in_meta_info == 1 print(meta_info_properties_in_sql) assert meta_info_properties_in_sql == EXPECTED_META_INFO block_properties_in_sql = session.db_session.\ query(Blocks).filter_by(block_number=56160).first().to_dict() assert block_properties_in_sql == EXPECTED_BLOCK_PROPERTIES # comparing values of uncles uncle_properties_in_sql = session.db_session.\ query(Uncles).filter_by(current_blocknumber=56160).first().to_dict() assert uncle_properties_in_sql == EXPECTED_UNCLE_PROPERTIES # comparing values of transactions transaction_properties_in_sql = session.db_session.\ query(Transactions).filter_by(block_number=56160).first().to_dict() assert transaction_properties_in_sql == EXPEXTED_TRANSACTION_PROPERTIES # comparing values of receipts receipt_properties_in_sql = session.db_session.\ query(Receipts).filter_by(block_number=56160).first().to_dict() assert receipt_properties_in_sql == EXPECTED_RECEIPT_PROPERTIES # comparing values of logs log_properties_in_sql = session.db_session.\ query(Logs).filter_by(block_number=56160).first().to_dict() assert log_properties_in_sql == EXPECTED_LOG_PROPERTIES # comparing values of traces if session.settings.PARSE_TRACE: trace_properties_in_sql = session.\ db_session.query(Traces).filter_by(block_number=56160).first().\ to_dict() assert trace_properties_in_sql == EXPECTED_TRACE_PROPERTIES # comparing values of states if session.settings.PARSE_STATE_DIFF: # comparing values if state diffs state_diff_property_in_sql = session.\ db_session.query(StateDiff).filter_by(block_number=56160).all() for i in range(0, len(state_diff_property_in_sql)): assert state_diff_property_in_sql[i].to_dict() == \ EXPECTED_STATE_DIFF_PROPERTIES[i] # comparing values of storage_diffs storage_diff_property_in_sql = session.\ db_session.query(StorageDiff).filter_by(block_number=56160).all() for i in range(0, len(storage_diff_property_in_sql)): assert storage_diff_property_in_sql[i].to_dict() == \ EXPECTED_STORAGE_DIFF_PROPERTIES[i]
def fill_missing_blocks(): session = get_current_session() runner = CliRunner() runner.invoke(cli, ['--settings', session.setting_name, 'scrape_block_range', '--end_block_number', 10, '--fill_gaps']) assert len(Blocks.missing_blocks(10)) == 0
def get_blocks_to_be_pushed_in_queue(cls, current_session): current_session = get_current_session() current_eth_blocknumber = current_session.w3.eth.blockNumber block_lag = current_session.settings.BLOCK_LAG query = current_session.db_session.query(cls.block_number).filter( and_(cls.state == 'WAITING', cls.block_number < current_eth_blocknumber - block_lag)) return query.from_self().distinct()
def push_initial_blocks_in_queue(): current_session = get_current_session() push_blocks_in_queue() with current_session.db_session_scope(): block_task_meta = current_session.db_session.query(BlockTaskMeta).first() assert block_task_meta.block_number == BASE_BLOCK assert block_task_meta.block_hash == EXPECTED_BLOCK_HASHES[BASE_BLOCK] assert block_task_meta.state == 'PENDING'
def close_celery_session(**kwargs): current_session = get_current_session() try: current_session.db_session.close except AttributeError: logger.debug('db_session attribute does not exist') finally: pass
def scrape_block_range(ctx, start_block_number, end_block_number, mode, fill_gaps): """ Pushes the data between start_block_number and end_block_number in the database. If no values are provided, the start_block_number is the last block_number+1 in sql and end_block_number is the current block_number in node. Also checks for missing blocks and adds them to the list of required block numbers :param int start_block_number: starting block number of scraping :param int end_block_number: end block number of scraping :param str mode: Mode of data sync 'parallel' or single :param bool fill_gaps: If switched on instructs to also fill missing blocks """ current_session = get_current_session() with current_session.db_session_scope(): sql_block_number = Blocks.get_max_block_number() if end_block_number is None: end_block_number = current_session.w3.eth.blockNumber logger.debug(end_block_number) if start_block_number is None: if sql_block_number is None: start_block_number = 0 elif sql_block_number == end_block_number: start_block_number = sql_block_number else: start_block_number = sql_block_number + 1 logger.debug(start_block_number) # casting numbers to integers if start_block_number == end_block_number: list_block_numbers = [] else: start_block_number = int(start_block_number) end_block_number = int(end_block_number) list_block_numbers = list( range(start_block_number, end_block_number + 1)) if fill_gaps and start_block_number != 0: missing_blocks = Blocks.missing_blocks(sql_block_number) logger.debug(missing_blocks) logger.info('{} missing blocks detected'.format(len(missing_blocks))) for missing in missing_blocks: logger.debug(missing.block_number) list_block_numbers.append(missing.block_number) logger.debug(list_block_numbers) if len(list_block_numbers) == 0: logger.warning('No blocks pushed in database') if mode == 'parallel': scrape_blocks(list_block_numbers=list_block_numbers, mode=mode) elif mode == 'single': scrape_blocks(list_block_numbers=list_block_numbers, mode=mode) else: raise ValueError('The mode: {} is not recognized'.format(mode))
def migrate(ctx, message): """ Alias for 'alembic revision --autogenerate' Run this command after changing sql tables """ current_session = get_current_session() if message is None: click.echo(ctx.get_help()) else: command.revision(setup_alembic_config(url=current_session.url), message=message, autogenerate=True, sql=None)
def on_success(self, retval, task_id, args, kwargs): current_session = get_current_session() with current_session.db_session_scope(): block_task_meta = BlockTaskMeta.get_block_task_meta_from_task_id( task_id) for i_block_task_meta in block_task_meta: if i_block_task_meta.block_hash == retval or i_block_task_meta is None: i_block_task_meta.state = 'SUCCESS' current_session.db_session.add(i_block_task_meta) else: i_block_task_meta.state = 'FORKED' current_session.db_session.add(i_block_task_meta)
def remove_block_number(block_number): """ Removes the block, transactions, uncles, logs and traces of a given block number into the database to perform chain reorgs. :param int block_number: The block number to add to the database """ current_session = get_current_session() with current_session.db_session_scope(): current_session.db_session.query(Blocks).\ filter_by(block_number=block_number).delete() logger.info("Removed block {}".format(block_number))
def update_block_task_meta_from_block_number(cls, block_number, **kwargs): current_session = get_current_session() with current_session.db_session_scope(): block_task_meta = current_session.db_session.query(cls).\ filter_by(block_number=block_number) logger.debug( 'Updating task meta of block {0}'.format(block_number)) for i_block_task_meta in block_task_meta: for key, value in kwargs.items(): setattr(i_block_task_meta, key, value) logger.debug('Updated task meta {}'.format( i_block_task_meta.to_dict()))
def verify_removed_block_range_56160_56170(): for i in range(56160, 56171): remove_block_number(i) session = get_current_session() with session.db_session_scope(): assert session.db_session.query(Blocks).count() == 0 assert session.db_session.query(Transactions).count() == 0 assert session.db_session.query(Receipts).count() == 0 assert session.db_session.query(Logs).count() == 0 assert session.db_session.query(Uncles).count() == 0 assert session.db_session.query(Traces).count() == 0 assert session.db_session.query(StateDiff).count() == 0 assert session.db_session.query(StorageDiff).count() == 0
def add_block_task_meta(cls, block_number, task_name, state, task_id=None, block_hash=None): current_session = get_current_session() block_task_meta = cls(task_name=task_name, state=state, block_number=block_number, block_hash=block_hash, task_id=task_id) with current_session.db_session_scope(): current_session.db_session.add(block_task_meta)
def new_blocks(): """ Celery beat task which runs every second to get new blocks. :param block_filter: block filter as described in session.py """ current_session = get_current_session() logger.debug("Reached at new blocks to get block hashes") block_hashes = current_session.block_filter.get_new_entries() for block_hash in block_hashes: block_data = current_session.w3.eth.getBlock(block_hash) block_number = to_int(block_data['number']) BlockTaskMeta.add_block_task_meta(task_name='new_blocks', state='WAITING', block_number=block_number, block_hash=to_hex(block_hash)) logger.info(block_hashes)
def export_to_csv_single_thread(): session = get_current_session() directory = 'test_export' call(["rm", "-rf", directory]) runner = CliRunner() result = runner.invoke(cli, [ '--settings', session.setting_name, 'sql', 'export_to_csv', '--directory', directory ]) assert result.exit_code == 0 # match the names of exported tables tables_in_sql = list(base.metadata.tables) files_in_directory = os.listdir(directory) for sql_table in tables_in_sql: assert sql_table + '.csv' in files_in_directory call(["rm", "-rf", directory])
def add_dao_hardfork_state_diff(): current_session = get_current_session() with current_session.db_session_scope(): for dict_account in DAO_DRAIN_LIST: state_diff = StateDiff.add_state_diff( balance_diff=dict_account['balance_diff'], nonce_diff=None, code_from=None, code_to=None, address=to_checksum_address(dict_account['address']), transaction_hash=None, transaction_index=None, block_number=MAINNET_DAO_BLOCK, timestamp=MAINNET_DAO_TIMESTAMP, state_diff_type='dao-fork') current_session.db_session.add(state_diff)
def match_state_dump_to_state_table(block_number): current_session = get_current_session() with open('tests/fixtures/balance/balance_{}.json'.format(block_number)) as data_file: data = json.loads(data_file.read()) state = data['state'] with current_session.db_session_scope(): for address in state: state_table_row = current_session.db_session.query(State).\ filter_by(address=to_checksum_address(address)).first() assert state_table_row.balance == hex_to_integer(state[address]['balance']) assert state_table_row.nonce == hex_to_integer(state[address]['nonce']) if 'code' in state[address].keys(): assert state_table_row.code == "0x"+state[address]['code'] if state_table_row.storage is not None: for storage in state_table_row.storage: storage.storage = state[address]['storage'][storage.position]
def push_blocks_in_queue(): """ Celery beat task which runs every 30 second to check for blocks which are settings.BLOCK_LAG number of blocks behind the current ethereum client and pushes the blocks in waiting to the queue. """ current_session = get_current_session() blocks_in_waiting = BlockTaskMeta.get_blocks_to_be_pushed_in_queue() for blocks in blocks_in_waiting: block_number = int(blocks.block_number) add_block_task = add_block_number.delay(block_number) BlockTaskMeta.update_block_task_meta_from_block_number( block_number=block_number, task_id=add_block_task.id, state='PENDING', task_name='add_block_number')
def missing_blocks(cls, max_block_number=None): """ Return missing blocks in the blocks table between 0 to block_number :param int max_block_number: Maximum block number that we want to find missing blocks from """ current_session = get_current_session() if max_block_number is None: max_block_number = cls.get_max_block_number() logger.debug(max_block_number) with current_session.db_session_scope(): stmt = current_session.db_session.query( func.generate_series(0, max_block_number).label('i')).subquery() joined = stmt.outerjoin(cls, cls.block_number == stmt.c.i) missing_blocks = current_session.db_session.query(stmt.c.i.label('block_number')).\ select_from(joined).filter(cls.block_number == None).all() return missing_blocks
def put_initial_blocks_in_waiting(): current_session = get_current_session() put_blocks = range(BASE_BLOCK, BASE_BLOCK + current_session.settings.BLOCK_LAG) for block_number in put_blocks: block_data = current_session.w3.eth.getBlock(block_number) block_hash = block_data['hash'] BlockTaskMeta.add_block_task_meta(task_name='new_blocks', state='WAITING', block_number=block_number, block_hash=to_hex(block_hash)) verify_pushed_sql_contents() with current_session.db_session_scope(): block_task_meta = current_session.db_session.query(BlockTaskMeta).first() assert block_task_meta.block_number == BASE_BLOCK assert block_task_meta.block_hash == EXPECTED_BLOCK_HASHES[BASE_BLOCK] assert block_task_meta.state == 'WAITING'
def verify_pushed_sql_contents(): current_session = get_current_session() with current_session.db_session_scope(): assert current_session.db_session.query(Blocks).count() == 0 assert current_session.db_session.query(Transactions).count() == 0 assert current_session.db_session.query(Receipts).count() == 0 assert current_session.db_session.query(Logs).count() == 0 assert current_session.db_session.query(Uncles).count() == 0 if current_session.settings.PARSE_TRACE: assert current_session.db_session.query(Traces).count() == 0 if current_session.settings.PARSE_STATE_DIFF: assert current_session.db_session.query(StateDiff).count() == 0 assert current_session.db_session.query(StorageDiff).count() == 0 assert current_session.db_session.query(BlockTaskMeta).count() == \ current_session.settings.BLOCK_LAG
def get_block_task_meta_from_block_hash(cls, block_hash): current_session = get_current_session() with current_session.db_session_scope(): return current_session.db_session.query(cls).\ filter_by(block_hash=block_hash)
def get_block_task_meta_from_task_id(cls, task_id): current_session = get_current_session() with current_session.db_session_scope(): return current_session.db_session.query(cls).\ filter_by(task_id=task_id)