def task_stream_blocks(database_url, steemd_http_url, task_num=6):
task_message = fmt_task_message(
'Streaming blocks', emoji_code_point=u'\U0001F4DD', task_num=task_num)
click.echo(task_message)
with isolated_engine(database_url, pool_recycle=3600) as engine:
session = Session(bind=engine)
highest_db_block = Block.highest_block(session)
rpc = SimpleSteemAPIClient(steemd_http_url)
blocks = rpc.stream(highest_db_block)
blocks_to_add = []
for block in blocks:
try:
blocks_to_add.append(block)
add_blocks(blocks_to_add, session, insert=True)
except Exception as e:
logger.exception('failed to add block')
else:
blocks_to_add = []
def stream_blocks(url, block_nums, start, end):
"""Stream blocks from steemd in JSON format
\b
Which Steemd:
\b
1. CLI "--url" option if provided
2. ENV var "STEEMD_HTTP_URL" if provided
3. Default: "https://steemd.steemitdev.com"
\b
Which Blocks To Output:
\b
- Stream blocks beginning with current block by omitting --start, --end, and BLOCKS
- Fetch a range of blocks using --start and/or --end
- Fetch list of blocks by passing BLOCKS a JSON array of block numbers (either filename or "-" for STDIN)
Where To Output Blocks:
\b
2. ENV var "BLOCKS_OUT" if provided
3. Default: STDOUT
"""
# Setup steemd source
rpc = SimpleSteemAPIClient(url)
with click.open_file('-', 'w', encoding='utf8') as f:
if block_nums:
block_nums = json.load(block_nums)
blocks = _stream_blocks(rpc, block_nums)
elif start and end:
blocks = _stream_blocks(rpc, range(start, end))
else:
blocks = rpc.stream(start)
json_blocks = map(json.dumps, blocks)
for block in json_blocks:
click.echo(block, file=f)
