def stream_blocks(server, block_nums, start, end): """Stream blocks from steemd in JSON format \b Which Steemd: \b 1. CLI "--server" option if provided 2. ENV var "WEBSOCKET_URL" if provided 3. Default: "wss://steemit.com/wspa" \b Which Blocks To Output: \b - Stream blocks beginning with current block by omitting --start, --end, and BLOCKS - Fetch a range of blocks using --start and/or --end - Fetch list of blocks by passing BLOCKS a JSON array of block numbers (either filename or "-" for STDIN) Where To Output Blocks: \b 2. ENV var "BLOCKS_OUT" if provided 3. Default: STDOUT """ # Setup steemd source rpc = SteemNodeRPC(server) with click.open_file('-', 'w', encoding='utf8') as f: if block_nums: blocks = _stream_blocks(rpc, block_nums) elif start and end: blocks = _stream_blocks(rpc, range(start, end)) else: blocks = rpc.block_stream(start) json_blocks = map(json.dumps, blocks) for block in json_blocks: click.echo(block, file=f)
def _populate(database_url, steemd_http_url, steemd_websocket_url, max_procs, max_threads): # pylint: disable=too-many-locals, too-many-statements rpc = SimpleSteemAPIClient(steemd_http_url) engine_config = configure_engine(database_url) db_name = engine_config.url.database db_user_name = engine_config.url.username Session.configure(bind=engine_config.engine) session = Session() # [1/7] confirm db connectivity task_message = fmt_task_message( 'Confirm database connectivity', emoji_code_point=u'\U0001F4DE', counter=1) click.echo(task_message) url, table_count = test_connection(database_url) if url: success_msg = fmt_success_message( 'connected to %s and found %s tables', url.__repr__(), table_count) click.echo(success_msg) if not url: raise Exception('Unable to connect to database') del url del table_count # [2/7] kill existing db threads task_message = fmt_task_message( 'Killing active db threads', emoji_code_point='\U0001F4A5', counter=2) click.echo(task_message) all_procs, killed_procs = kill_db_processes(database_url, db_name, db_user_name) if len(killed_procs) > 0: success_msg = fmt_success_message('killed %s processes', len(killed_procs)) click.echo(success_msg) del all_procs del killed_procs # [3/7] init db if required task_message = fmt_task_message( 'Initialising db if required', emoji_code_point=u'\U0001F50C', counter=3) click.echo(task_message) init_tables(database_url, Base.metadata) # [4/7] find last irreversible block last_chain_block = rpc.last_irreversible_block_num() task_message = fmt_task_message( 'Finding highest blockchain block', emoji_code_point='\U0001F50E', counter=4) click.echo(task_message) success_msg = fmt_success_message( 'learned highest irreversible block is %s', last_chain_block) click.echo(success_msg) # [5/7] get missing block_nums task_message = fmt_task_message( 'Finding blocks missing from db', emoji_code_point=u'\U0001F52D', counter=5) click.echo(task_message) missing_block_nums_gen = Block.get_missing_block_num_iterator( session, last_chain_block, chunksize=100000) with click.progressbar( missing_block_nums_gen, label='Finding missing block_nums', color=True, show_eta=False, show_percent=False, empty_char='░', fill_char='█', show_pos=True, bar_template='%(bar)s %(info)s') as pbar: all_missing_block_nums = [] for missing_gen in pbar: all_missing_block_nums.extend(missing_gen()) success_msg = fmt_success_message('found %s missing blocks', len(all_missing_block_nums)) click.echo(success_msg) del missing_block_nums_gen del pbar session.invalidate() # [6/7] adding missing blocks task_message = fmt_task_message( 'Adding missing blocks to db, this may take a while', emoji_code_point=u'\U0001F4DD', counter=6) click.echo(task_message) max_workers = max_procs or os.cpu_count() or 1 chunksize = len(all_missing_block_nums) // max_workers if chunksize <= 0: chunksize = 1 map_func = partial( block_adder_process_worker, database_url, steemd_http_url, max_threads=max_threads) chunks = chunkify(all_missing_block_nums, 10000) with concurrent.futures.ProcessPoolExecutor( max_workers=max_workers) as executor: executor.map(map_func, chunks , chunksize=1) success_msg = fmt_success_message('added missing blocks') click.echo(success_msg) del all_missing_block_nums # [7/7] stream blocks task_message = fmt_task_message( 'Streaming blocks', emoji_code_point=u'\U0001F4DD', counter=7) click.echo(task_message) highest_db_block = Block.highest_block(session) ws_rpc = SteemNodeRPC(steemd_websocket_url) blocks = ws_rpc.block_stream(highest_db_block) add_blocks(blocks, session)