def task_get_last_irreversible_block(steemd_http_url, task_num=3): rpc = SimpleSteemAPIClient(steemd_http_url) last_chain_block = rpc.last_irreversible_block_num() task_message = fmt_task_message( 'Finding highest blockchain block', emoji_code_point='\U0001F50E', task_num=task_num) click.echo(task_message) return last_chain_block
def get_random_operations(db, bottle, app, params): """Return a random sample of specified operation. Args: db (): bottle (): app (): params (): Returns: """ op_type = params.get('op_type') op_count = params.get('count', 100) op_class = app.config['sbds.tx_class_map'][op_type] db_query_hard_limit = app.config['sbds.DB_QUERY_LIMIT'] results = db.query(op_class.block_num).limit(db_query_hard_limit).all() if op_count > 1000: op_count = 1000 if op_count > len(results): op_count = len(results) block_nums = [r[0] for r in random.sample(results, op_count)] client = SimpleSteemAPIClient() blocks = map(client.get_block, block_nums) ops = extract_operations_from_blocks(blocks) filtered_ops = filter(lambda op: op.get('type') == op_type, ops) return list(filtered_ops)
def get_blocks_fast(start, end, chunksize, max_workers, url): """Request blocks from steemd in JSON format""" rpc = SimpleSteemAPIClient(url) if end == 0: end = rpc.last_irreversible_block_num() with click.open_file('-', 'w', encoding='utf8') as f: blocks = _get_blocks_fast(start=start, end=end, chunksize=chunksize, max_workers=max_workers, rpc=rpc, url=url) json_blocks = map(json.dumps, blocks) for block in json_blocks: click.echo(block.encode('utf8'), file=f)
def task_stream_blocks(database_url, steemd_http_url, task_num=6): task_message = fmt_task_message( 'Streaming blocks', emoji_code_point=u'\U0001F4DD', task_num=task_num) click.echo(task_message) with isolated_engine(database_url, pool_recycle=3600) as engine: session = Session(bind=engine) highest_db_block = Block.highest_block(session) rpc = SimpleSteemAPIClient(steemd_http_url) blocks = rpc.stream(highest_db_block) blocks_to_add = [] for block in blocks: try: blocks_to_add.append(block) add_blocks(blocks_to_add, session, insert=True) except Exception as e: logger.exception('failed to add block') else: blocks_to_add = []
def block_fetcher_thread_worker(rpc_url, block_nums, max_threads=None): rpc = SimpleSteemAPIClient(rpc_url, return_with_args=True) # pylint: disable=unused-variable with concurrent.futures.ThreadPoolExecutor( max_workers=max_threads) as executor: for result, args in executor.map(rpc.get_block, block_nums): # dont yield anything when we encounter a null output # from an HTTP 503 error if result: yield result
def find_missing_blocks(ctx, url): """Return JSON array of block_nums from missing blocks""" from sbds.storages.db.tables import Block engine = ctx.obj['engine'] database_url = ctx.obj['database_url'] metadata = ctx.obj['metadata'] rpc = SimpleSteemAPIClient(url) # init tables first init_tables(database_url, metadata) # configure session Session.configure(bind=engine) session = Session() last_chain_block = rpc.last_irreversible_block_num() click.echo( json.dumps( Block.find_missing(session, last_chain_block=last_chain_block)))
def stream_blocks(url, block_nums, start, end): """Stream blocks from steemd in JSON format \b Which Steemd: \b 1. CLI "--url" option if provided 2. ENV var "STEEMD_HTTP_URL" if provided 3. Default: "https://steemd.steemitdev.com" \b Which Blocks To Output: \b - Stream blocks beginning with current block by omitting --start, --end, and BLOCKS - Fetch a range of blocks using --start and/or --end - Fetch list of blocks by passing BLOCKS a JSON array of block numbers (either filename or "-" for STDIN) Where To Output Blocks: \b 2. ENV var "BLOCKS_OUT" if provided 3. Default: STDOUT """ # Setup steemd source rpc = SimpleSteemAPIClient(url) with click.open_file('-', 'w', encoding='utf8') as f: if block_nums: block_nums = json.load(block_nums) blocks = _stream_blocks(rpc, block_nums) elif start and end: blocks = _stream_blocks(rpc, range(start, end)) else: blocks = rpc.stream(start) json_blocks = map(json.dumps, blocks) for block in json_blocks: click.echo(block, file=f)
def _get_blocks_fast(start=None, end=None, chunksize=None, max_workers=None, rpc=None, url=None): extra = dict(start=start, end=end, chunksize=chunksize, max_workers=max_workers, rpc=rpc, url=url) logger.debug('get_blocks_fast', extra=extra) rpc = rpc or SimpleSteemAPIClient(url) with concurrent.futures.ThreadPoolExecutor( max_workers=max_workers) as executor: for i, chunk in enumerate( chunkify(range(start, end), chunksize=chunksize), 1): logger.debug('get_block_fast loop', extra=dict(chunk_count=i)) for b in executor.map(rpc.get_block, chunk): # dont yield anything when we encounter a null output # from an HTTP 503 error if b: yield b
def http_client(url='https://steemd.steemitdev.com', **kwargs): return SimpleSteemAPIClient(url, **kwargs)
from sbds.storages.db.tables import TxEscrowApprove from sbds.storages.db.tables import TxEscrowDispute from sbds.storages.db.tables import TxEscrowRelease from sbds.storages.db.tables import TxEscrowTransfer from sbds.storages.db.tables import TxFeedPublish from sbds.storages.db.tables import TxLimitOrderCancel from sbds.storages.db.tables import TxLimitOrderCreate from sbds.storages.db.tables import TxPow from sbds.storages.db.tables import TxPow2 from sbds.storages.db.tables import TxRecoverAccount from sbds.storages.db.tables import TxRequestAccountRecovery from sbds.storages.db.tables import TxWithdrawVestingRoute from sbds.storages.db.tables import TxTransfer from sbds.storages.db.tables import TxTransferFromSavings from sbds.storages.db.tables import TxTransferToSavings from sbds.storages.db.tables import TxTransferToVesting from sbds.storages.db.tables import TxVote from sbds.storages.db.tables import TxWithdrawVesting from sbds.storages.db.tables import TxWitnessUpdate from sbds.storages.db.utils import configure_engine from sbds.http_client import SimpleSteemAPIClient db_url = os.environ['DATABASE_URL'] rpc_url = os.environ['STEEMD_HTTP_URL'] engine_config = configure_engine(db_url) engine = engine_config.engine session = Session(bind=engine) client = SimpleSteemAPIClient(url=rpc_url)
def block_fetcher_thread_worker(rpc_url, block_nums, max_threads=None): rpc = SimpleSteemAPIClient(rpc_url, return_with_args=True) # pylint: disable=unused-variable for block in rpc.exec_multi_with_futures( 'get_block', block_nums, max_workers=max_threads): yield block
def block_height(url): rpc = SimpleSteemAPIClient(url) click.echo(rpc.last_irreversible_block_num())
from sbds.server.methods import get_custom_json_by_tid from sbds.server.methods import get_random_operation_block_nums from sbds.server.methods import get_random_operations logger = sbds.sbds_logging.getLogger(__name__) app = bottle.Bottle() app.config['sbds.RPC_URL'] = os.environ.get('STEEMD_HTTP_URL', 'https://steemd.steemitdev.com') app.config['sbds.DATABASE_URL'] = os.environ.get('DATABASE_URL', 'sqlite:///') app.config['sbds.MAX_BLOCK_NUM_DIFF'] = 10 app.config['sbds.MAX_DB_ROW_RESULTS'] = 100000 app.config['sbds.DB_QUERY_LIMIT'] = app.config['sbds.MAX_DB_ROW_RESULTS'] + 1 app.config['sbds.tx_class_map'] = tx_class_map app.config['sbds.logger'] = logger rpc = SimpleSteemAPIClient(app.config['sbds.RPC_URL']) def get_db_plugin(database_url): engine_config = configure_engine(database_url) Session.configure(bind=engine_config.engine) # pylint: disable=undefined-variable return sqlalchemy.Plugin( # SQLAlchemy engine created with create_engine function. engine_config.engine, # SQLAlchemy metadata, required only if create=True. Base.metadata, # Keyword used to inject session database in a route (default 'db').
from sbds.http_client import SimpleSteemAPIClient from sbds.storages.db import Base, Session from sbds.storages.db.tables.core import Block from sbds.storages.db.tables.tx import tx_class_map from sbds.storages.db.utils import configure_engine MAX_DB_ROW_RESULTS = 100000 DB_QUERY_LIMIT = MAX_DB_ROW_RESULTS + 1 logger = sbds.logging.getLogger(__name__, level=logging.DEBUG) rpc_url = os.environ.get('STEEMD_HTTP_URL', 'https://steemd.steemitdev.com') database_url = os.environ.get('DATABASE_URL', 'sqlite:///') rpc = SimpleSteemAPIClient(rpc_url) database_url, url, engine_kwargs, engine = configure_engine(database_url, echo=True) Session.configure(bind=engine) app = bottle.Bottle() # pylint: disable=undefined-variable plugin = sqlalchemy.Plugin( engine, # SQLAlchemy engine created with create_engine function. Base.metadata, # SQLAlchemy metadata, required only if create=True. keyword='db', # Keyword used to inject session database in a route (default 'db'). create=True, # If it is true, execute `metadata.create_all(engine)` when plugin is applied (default False). commit=False,
def _populate(database_url, steemd_http_url, steemd_websocket_url, max_procs, max_threads): # pylint: disable=too-many-locals, too-many-statements rpc = SimpleSteemAPIClient(steemd_http_url) engine_config = configure_engine(database_url) db_name = engine_config.url.database db_user_name = engine_config.url.username Session.configure(bind=engine_config.engine) session = Session() # [1/7] confirm db connectivity task_message = fmt_task_message( 'Confirm database connectivity', emoji_code_point=u'\U0001F4DE', counter=1) click.echo(task_message) url, table_count = test_connection(database_url) if url: success_msg = fmt_success_message( 'connected to %s and found %s tables', url.__repr__(), table_count) click.echo(success_msg) if not url: raise Exception('Unable to connect to database') del url del table_count # [2/7] kill existing db threads task_message = fmt_task_message( 'Killing active db threads', emoji_code_point='\U0001F4A5', counter=2) click.echo(task_message) all_procs, killed_procs = kill_db_processes(database_url, db_name, db_user_name) if len(killed_procs) > 0: success_msg = fmt_success_message('killed %s processes', len(killed_procs)) click.echo(success_msg) del all_procs del killed_procs # [3/7] init db if required task_message = fmt_task_message( 'Initialising db if required', emoji_code_point=u'\U0001F50C', counter=3) click.echo(task_message) init_tables(database_url, Base.metadata) # [4/7] find last irreversible block last_chain_block = rpc.last_irreversible_block_num() task_message = fmt_task_message( 'Finding highest blockchain block', emoji_code_point='\U0001F50E', counter=4) click.echo(task_message) success_msg = fmt_success_message( 'learned highest irreversible block is %s', last_chain_block) click.echo(success_msg) # [5/7] get missing block_nums task_message = fmt_task_message( 'Finding blocks missing from db', emoji_code_point=u'\U0001F52D', counter=5) click.echo(task_message) missing_block_nums_gen = Block.get_missing_block_num_iterator( session, last_chain_block, chunksize=100000) with click.progressbar( missing_block_nums_gen, label='Finding missing block_nums', color=True, show_eta=False, show_percent=False, empty_char='░', fill_char='█', show_pos=True, bar_template='%(bar)s %(info)s') as pbar: all_missing_block_nums = [] for missing_gen in pbar: all_missing_block_nums.extend(missing_gen()) success_msg = fmt_success_message('found %s missing blocks', len(all_missing_block_nums)) click.echo(success_msg) del missing_block_nums_gen del pbar session.invalidate() # [6/7] adding missing blocks task_message = fmt_task_message( 'Adding missing blocks to db, this may take a while', emoji_code_point=u'\U0001F4DD', counter=6) click.echo(task_message) max_workers = max_procs or os.cpu_count() or 1 chunksize = len(all_missing_block_nums) // max_workers if chunksize <= 0: chunksize = 1 map_func = partial( block_adder_process_worker, database_url, steemd_http_url, max_threads=max_threads) chunks = chunkify(all_missing_block_nums, 10000) with concurrent.futures.ProcessPoolExecutor( max_workers=max_workers) as executor: executor.map(map_func, chunks , chunksize=1) success_msg = fmt_success_message('added missing blocks') click.echo(success_msg) del all_missing_block_nums # [7/7] stream blocks task_message = fmt_task_message( 'Streaming blocks', emoji_code_point=u'\U0001F4DD', counter=7) click.echo(task_message) highest_db_block = Block.highest_block(session) ws_rpc = SteemNodeRPC(steemd_websocket_url) blocks = ws_rpc.block_stream(highest_db_block) add_blocks(blocks, session)