def task_get_last_irreversible_block(steemd_http_url, task_num=3):
rpc = SimpleSteemAPIClient(steemd_http_url)
last_chain_block = rpc.last_irreversible_block_num()
task_message = fmt_task_message(
'Finding highest blockchain block',
emoji_code_point='\U0001F50E',
task_num=task_num)
click.echo(task_message)
return last_chain_block
def get_random_operations(db, bottle, app, params):
"""Return a random sample of specified operation.
Args:
db ():
bottle ():
app ():
params ():
Returns:
"""
op_type = params.get('op_type')
op_count = params.get('count', 100)
op_class = app.config['sbds.tx_class_map'][op_type]
db_query_hard_limit = app.config['sbds.DB_QUERY_LIMIT']
results = db.query(op_class.block_num).limit(db_query_hard_limit).all()
if op_count > 1000:
op_count = 1000
if op_count > len(results):
op_count = len(results)
block_nums = [r[0] for r in random.sample(results, op_count)]
client = SimpleSteemAPIClient()
blocks = map(client.get_block, block_nums)
ops = extract_operations_from_blocks(blocks)
filtered_ops = filter(lambda op: op.get('type') == op_type, ops)
return list(filtered_ops)
def get_blocks_fast(start, end, chunksize, max_workers, url):
"""Request blocks from steemd in JSON format"""
rpc = SimpleSteemAPIClient(url)
if end == 0:
end = rpc.last_irreversible_block_num()
with click.open_file('-', 'w', encoding='utf8') as f:
blocks = _get_blocks_fast(start=start,
end=end,
chunksize=chunksize,
max_workers=max_workers,
rpc=rpc,
url=url)
json_blocks = map(json.dumps, blocks)
for block in json_blocks:
click.echo(block.encode('utf8'), file=f)
def task_stream_blocks(database_url, steemd_http_url, task_num=6):
task_message = fmt_task_message(
'Streaming blocks', emoji_code_point=u'\U0001F4DD', task_num=task_num)
click.echo(task_message)
with isolated_engine(database_url, pool_recycle=3600) as engine:
session = Session(bind=engine)
highest_db_block = Block.highest_block(session)
rpc = SimpleSteemAPIClient(steemd_http_url)
blocks = rpc.stream(highest_db_block)
blocks_to_add = []
for block in blocks:
try:
blocks_to_add.append(block)
add_blocks(blocks_to_add, session, insert=True)
except Exception as e:
logger.exception('failed to add block')
else:
blocks_to_add = []
def block_fetcher_thread_worker(rpc_url, block_nums, max_threads=None):
rpc = SimpleSteemAPIClient(rpc_url, return_with_args=True)
# pylint: disable=unused-variable
with concurrent.futures.ThreadPoolExecutor(
max_workers=max_threads) as executor:
for result, args in executor.map(rpc.get_block, block_nums):
# dont yield anything when we encounter a null output
# from an HTTP 503 error
if result:
yield result
def find_missing_blocks(ctx, url):
"""Return JSON array of block_nums from missing blocks"""
from sbds.storages.db.tables import Block
engine = ctx.obj['engine']
database_url = ctx.obj['database_url']
metadata = ctx.obj['metadata']
rpc = SimpleSteemAPIClient(url)
# init tables first
init_tables(database_url, metadata)
# configure session
Session.configure(bind=engine)
session = Session()
last_chain_block = rpc.last_irreversible_block_num()
click.echo(
json.dumps(
Block.find_missing(session, last_chain_block=last_chain_block)))
def stream_blocks(url, block_nums, start, end):
"""Stream blocks from steemd in JSON format
\b
Which Steemd:
\b
1. CLI "--url" option if provided
2. ENV var "STEEMD_HTTP_URL" if provided
3. Default: "https://steemd.steemitdev.com"
\b
Which Blocks To Output:
\b
- Stream blocks beginning with current block by omitting --start, --end, and BLOCKS
- Fetch a range of blocks using --start and/or --end
- Fetch list of blocks by passing BLOCKS a JSON array of block numbers (either filename or "-" for STDIN)
Where To Output Blocks:
\b
2. ENV var "BLOCKS_OUT" if provided
3. Default: STDOUT
"""
# Setup steemd source
rpc = SimpleSteemAPIClient(url)
with click.open_file('-', 'w', encoding='utf8') as f:
if block_nums:
block_nums = json.load(block_nums)
blocks = _stream_blocks(rpc, block_nums)
elif start and end:
blocks = _stream_blocks(rpc, range(start, end))
else:
blocks = rpc.stream(start)
json_blocks = map(json.dumps, blocks)
for block in json_blocks:
click.echo(block, file=f)
def _get_blocks_fast(start=None,
end=None,
chunksize=None,
max_workers=None,
rpc=None,
url=None):
extra = dict(start=start,
end=end,
chunksize=chunksize,
max_workers=max_workers,
rpc=rpc,
url=url)
logger.debug('get_blocks_fast', extra=extra)
rpc = rpc or SimpleSteemAPIClient(url)
with concurrent.futures.ThreadPoolExecutor(
max_workers=max_workers) as executor:
for i, chunk in enumerate(
chunkify(range(start, end), chunksize=chunksize), 1):
logger.debug('get_block_fast loop', extra=dict(chunk_count=i))
for b in executor.map(rpc.get_block, chunk):
# dont yield anything when we encounter a null output
# from an HTTP 503 error
if b:
yield b
def http_client(url='https://steemd.steemitdev.com', **kwargs):
return SimpleSteemAPIClient(url, **kwargs)
from sbds.storages.db.tables import TxEscrowApprove from sbds.storages.db.tables import TxEscrowDispute from sbds.storages.db.tables import TxEscrowRelease from sbds.storages.db.tables import TxEscrowTransfer from sbds.storages.db.tables import TxFeedPublish from sbds.storages.db.tables import TxLimitOrderCancel from sbds.storages.db.tables import TxLimitOrderCreate from sbds.storages.db.tables import TxPow from sbds.storages.db.tables import TxPow2 from sbds.storages.db.tables import TxRecoverAccount from sbds.storages.db.tables import TxRequestAccountRecovery from sbds.storages.db.tables import TxWithdrawVestingRoute from sbds.storages.db.tables import TxTransfer from sbds.storages.db.tables import TxTransferFromSavings from sbds.storages.db.tables import TxTransferToSavings from sbds.storages.db.tables import TxTransferToVesting from sbds.storages.db.tables import TxVote from sbds.storages.db.tables import TxWithdrawVesting from sbds.storages.db.tables import TxWitnessUpdate from sbds.storages.db.utils import configure_engine from sbds.http_client import SimpleSteemAPIClient db_url = os.environ['DATABASE_URL'] rpc_url = os.environ['STEEMD_HTTP_URL'] engine_config = configure_engine(db_url) engine = engine_config.engine session = Session(bind=engine) client = SimpleSteemAPIClient(url=rpc_url)
def block_fetcher_thread_worker(rpc_url, block_nums, max_threads=None):
rpc = SimpleSteemAPIClient(rpc_url, return_with_args=True)
# pylint: disable=unused-variable
for block in rpc.exec_multi_with_futures(
'get_block', block_nums, max_workers=max_threads):
yield block
def block_height(url):
rpc = SimpleSteemAPIClient(url)
click.echo(rpc.last_irreversible_block_num())
from sbds.server.methods import get_custom_json_by_tid
from sbds.server.methods import get_random_operation_block_nums
from sbds.server.methods import get_random_operations
logger = sbds.sbds_logging.getLogger(__name__)
app = bottle.Bottle()
app.config['sbds.RPC_URL'] = os.environ.get('STEEMD_HTTP_URL',
'https://steemd.steemitdev.com')
app.config['sbds.DATABASE_URL'] = os.environ.get('DATABASE_URL', 'sqlite:///')
app.config['sbds.MAX_BLOCK_NUM_DIFF'] = 10
app.config['sbds.MAX_DB_ROW_RESULTS'] = 100000
app.config['sbds.DB_QUERY_LIMIT'] = app.config['sbds.MAX_DB_ROW_RESULTS'] + 1
app.config['sbds.tx_class_map'] = tx_class_map
app.config['sbds.logger'] = logger
rpc = SimpleSteemAPIClient(app.config['sbds.RPC_URL'])
def get_db_plugin(database_url):
engine_config = configure_engine(database_url)
Session.configure(bind=engine_config.engine)
# pylint: disable=undefined-variable
return sqlalchemy.Plugin(
# SQLAlchemy engine created with create_engine function.
engine_config.engine,
# SQLAlchemy metadata, required only if create=True.
Base.metadata,
# Keyword used to inject session database in a route (default 'db').
from sbds.http_client import SimpleSteemAPIClient
from sbds.storages.db import Base, Session
from sbds.storages.db.tables.core import Block
from sbds.storages.db.tables.tx import tx_class_map
from sbds.storages.db.utils import configure_engine
MAX_DB_ROW_RESULTS = 100000
DB_QUERY_LIMIT = MAX_DB_ROW_RESULTS + 1
logger = sbds.logging.getLogger(__name__, level=logging.DEBUG)
rpc_url = os.environ.get('STEEMD_HTTP_URL', 'https://steemd.steemitdev.com')
database_url = os.environ.get('DATABASE_URL', 'sqlite:///')
rpc = SimpleSteemAPIClient(rpc_url)
database_url, url, engine_kwargs, engine = configure_engine(database_url,
echo=True)
Session.configure(bind=engine)
app = bottle.Bottle()
# pylint: disable=undefined-variable
plugin = sqlalchemy.Plugin(
engine, # SQLAlchemy engine created with create_engine function.
Base.metadata, # SQLAlchemy metadata, required only if create=True.
keyword='db',
# Keyword used to inject session database in a route (default 'db').
create=True,
# If it is true, execute `metadata.create_all(engine)` when plugin is applied (default False).
commit=False,
def _populate(database_url, steemd_http_url, steemd_websocket_url, max_procs,
max_threads):
# pylint: disable=too-many-locals, too-many-statements
rpc = SimpleSteemAPIClient(steemd_http_url)
engine_config = configure_engine(database_url)
db_name = engine_config.url.database
db_user_name = engine_config.url.username
Session.configure(bind=engine_config.engine)
session = Session()
# [1/7] confirm db connectivity
task_message = fmt_task_message(
'Confirm database connectivity',
emoji_code_point=u'\U0001F4DE',
counter=1)
click.echo(task_message)
url, table_count = test_connection(database_url)
if url:
success_msg = fmt_success_message(
'connected to %s and found %s tables', url.__repr__(), table_count)
click.echo(success_msg)
if not url:
raise Exception('Unable to connect to database')
del url
del table_count
# [2/7] kill existing db threads
task_message = fmt_task_message(
'Killing active db threads', emoji_code_point='\U0001F4A5', counter=2)
click.echo(task_message)
all_procs, killed_procs = kill_db_processes(database_url, db_name,
db_user_name)
if len(killed_procs) > 0:
success_msg = fmt_success_message('killed %s processes',
len(killed_procs))
click.echo(success_msg)
del all_procs
del killed_procs
# [3/7] init db if required
task_message = fmt_task_message(
'Initialising db if required',
emoji_code_point=u'\U0001F50C',
counter=3)
click.echo(task_message)
init_tables(database_url, Base.metadata)
# [4/7] find last irreversible block
last_chain_block = rpc.last_irreversible_block_num()
task_message = fmt_task_message(
'Finding highest blockchain block',
emoji_code_point='\U0001F50E',
counter=4)
click.echo(task_message)
success_msg = fmt_success_message(
'learned highest irreversible block is %s', last_chain_block)
click.echo(success_msg)
# [5/7] get missing block_nums
task_message = fmt_task_message(
'Finding blocks missing from db',
emoji_code_point=u'\U0001F52D',
counter=5)
click.echo(task_message)
missing_block_nums_gen = Block.get_missing_block_num_iterator(
session, last_chain_block, chunksize=100000)
with click.progressbar(
missing_block_nums_gen,
label='Finding missing block_nums',
color=True,
show_eta=False,
show_percent=False,
empty_char='░',
fill_char='█',
show_pos=True,
bar_template='%(bar)s %(info)s') as pbar:
all_missing_block_nums = []
for missing_gen in pbar:
all_missing_block_nums.extend(missing_gen())
success_msg = fmt_success_message('found %s missing blocks',
len(all_missing_block_nums))
click.echo(success_msg)
del missing_block_nums_gen
del pbar
session.invalidate()
# [6/7] adding missing blocks
task_message = fmt_task_message(
'Adding missing blocks to db, this may take a while',
emoji_code_point=u'\U0001F4DD',
counter=6)
click.echo(task_message)
max_workers = max_procs or os.cpu_count() or 1
chunksize = len(all_missing_block_nums) // max_workers
if chunksize <= 0:
chunksize = 1
map_func = partial(
block_adder_process_worker,
database_url,
steemd_http_url,
max_threads=max_threads)
chunks = chunkify(all_missing_block_nums, 10000)
with concurrent.futures.ProcessPoolExecutor(
max_workers=max_workers) as executor:
executor.map(map_func, chunks , chunksize=1)
success_msg = fmt_success_message('added missing blocks')
click.echo(success_msg)
del all_missing_block_nums
# [7/7] stream blocks
task_message = fmt_task_message(
'Streaming blocks', emoji_code_point=u'\U0001F4DD', counter=7)
click.echo(task_message)
highest_db_block = Block.highest_block(session)
ws_rpc = SteemNodeRPC(steemd_websocket_url)
blocks = ws_rpc.block_stream(highest_db_block)
add_blocks(blocks, session)
