def stream_blocks(server, block_nums, start, end):
"""Stream blocks from steemd in JSON format
\b
Which Steemd:
\b
1. CLI "--server" option if provided
2. ENV var "WEBSOCKET_URL" if provided
3. Default: "wss://steemit.com/wspa"
\b
Which Blocks To Output:
\b
- Stream blocks beginning with current block by omitting --start, --end, and BLOCKS
- Fetch a range of blocks using --start and/or --end
- Fetch list of blocks by passing BLOCKS a JSON array of block numbers (either filename or "-" for STDIN)
Where To Output Blocks:
\b
2. ENV var "BLOCKS_OUT" if provided
3. Default: STDOUT
"""
# Setup steemd source
rpc = SteemNodeRPC(server)
with click.open_file('-', 'w', encoding='utf8') as f:
if block_nums:
blocks = _stream_blocks(rpc, block_nums)
elif start and end:
blocks = _stream_blocks(rpc, range(start, end))
else:
blocks = rpc.block_stream(start)
json_blocks = map(json.dumps, blocks)
for block in json_blocks:
click.echo(block, file=f)
def _populate(database_url, steemd_http_url, steemd_websocket_url, max_procs,
max_threads):
# pylint: disable=too-many-locals, too-many-statements
rpc = SimpleSteemAPIClient(steemd_http_url)
engine_config = configure_engine(database_url)
db_name = engine_config.url.database
db_user_name = engine_config.url.username
Session.configure(bind=engine_config.engine)
session = Session()
# [1/7] confirm db connectivity
task_message = fmt_task_message(
'Confirm database connectivity',
emoji_code_point=u'\U0001F4DE',
counter=1)
click.echo(task_message)
url, table_count = test_connection(database_url)
if url:
success_msg = fmt_success_message(
'connected to %s and found %s tables', url.__repr__(), table_count)
click.echo(success_msg)
if not url:
raise Exception('Unable to connect to database')
del url
del table_count
# [2/7] kill existing db threads
task_message = fmt_task_message(
'Killing active db threads', emoji_code_point='\U0001F4A5', counter=2)
click.echo(task_message)
all_procs, killed_procs = kill_db_processes(database_url, db_name,
db_user_name)
if len(killed_procs) > 0:
success_msg = fmt_success_message('killed %s processes',
len(killed_procs))
click.echo(success_msg)
del all_procs
del killed_procs
# [3/7] init db if required
task_message = fmt_task_message(
'Initialising db if required',
emoji_code_point=u'\U0001F50C',
counter=3)
click.echo(task_message)
init_tables(database_url, Base.metadata)
# [4/7] find last irreversible block
last_chain_block = rpc.last_irreversible_block_num()
task_message = fmt_task_message(
'Finding highest blockchain block',
emoji_code_point='\U0001F50E',
counter=4)
click.echo(task_message)
success_msg = fmt_success_message(
'learned highest irreversible block is %s', last_chain_block)
click.echo(success_msg)
# [5/7] get missing block_nums
task_message = fmt_task_message(
'Finding blocks missing from db',
emoji_code_point=u'\U0001F52D',
counter=5)
click.echo(task_message)
missing_block_nums_gen = Block.get_missing_block_num_iterator(
session, last_chain_block, chunksize=100000)
with click.progressbar(
missing_block_nums_gen,
label='Finding missing block_nums',
color=True,
show_eta=False,
show_percent=False,
empty_char='░',
fill_char='█',
show_pos=True,
bar_template='%(bar)s %(info)s') as pbar:
all_missing_block_nums = []
for missing_gen in pbar:
all_missing_block_nums.extend(missing_gen())
success_msg = fmt_success_message('found %s missing blocks',
len(all_missing_block_nums))
click.echo(success_msg)
del missing_block_nums_gen
del pbar
session.invalidate()
# [6/7] adding missing blocks
task_message = fmt_task_message(
'Adding missing blocks to db, this may take a while',
emoji_code_point=u'\U0001F4DD',
counter=6)
click.echo(task_message)
max_workers = max_procs or os.cpu_count() or 1
chunksize = len(all_missing_block_nums) // max_workers
if chunksize <= 0:
chunksize = 1
map_func = partial(
block_adder_process_worker,
database_url,
steemd_http_url,
max_threads=max_threads)
chunks = chunkify(all_missing_block_nums, 10000)
with concurrent.futures.ProcessPoolExecutor(
max_workers=max_workers) as executor:
executor.map(map_func, chunks , chunksize=1)
success_msg = fmt_success_message('added missing blocks')
click.echo(success_msg)
del all_missing_block_nums
# [7/7] stream blocks
task_message = fmt_task_message(
'Streaming blocks', emoji_code_point=u'\U0001F4DD', counter=7)
click.echo(task_message)
highest_db_block = Block.highest_block(session)
ws_rpc = SteemNodeRPC(steemd_websocket_url)
blocks = ws_rpc.block_stream(highest_db_block)
add_blocks(blocks, session)
