async def health(request): """Get hive health data. 500 if behind by more than 3 blocks.""" #pylint: disable=unused-argument is_syncer = Conf.get('sync_to_s3') max_head_age = (Conf.get('trail_blocks') + 3) * 3 state = await _head_state() if not state: status = 500 result = 'db not available' elif not is_syncer and state['db_head_age'] > max_head_age: status = 500 result = 'head block age (%s) > max (%s); head block num: %s' % ( state['db_head_age'], max_head_age, state['db_head_block']) else: status = 200 result = 'head block age is %d, head block num is %d' % ( state['db_head_age'], state['db_head_block']) return web.json_response( status=status, data=dict(state=state, result=result, status='OK' if status == 200 else 'WARN', sync_service=is_syncer, source_commit=os.environ.get('SOURCE_COMMIT'), schema_hash=os.environ.get('SCHEMA_HASH'), docker_tag=os.environ.get('DOCKER_TAG'), timestamp=datetime.utcnow().isoformat()))
def instance(cls): """Get a singleton, lazily initialized""" if not cls._instance: cls._instance = SteemClient(url=Conf.get('steemd_url'), max_batch=Conf.get('max_batch'), max_workers=Conf.get('max_workers')) return cls._instance
def run(cls): """Initialize state; setup/recovery checks; sync and runloop.""" # ensure db schema up to date, check app status DbState.initialize() # prefetch id->name memory map Accounts.load_ids() if DbState.is_initial_sync(): # resume initial sync cls.initial() DbState.finish_initial_sync() else: # recover from fork Blocks.verify_head() # perform cleanup if process did not exit cleanly CachedPost.recover_missing_posts() # debug mode: no sync, just stream if Conf.get('disable_sync'): return cls.listen() while True: # sync up to irreversible block cls.from_steemd() # take care of payout backlog CachedPost.dirty_paidouts(Blocks.head_date()) CachedPost.flush(trx=True) # listen for new blocks cls.listen()
def create_engine(echo=False): engine = sqlalchemy.create_engine( Conf.get('database_url'), isolation_level="READ UNCOMMITTED", # only works in mysql pool_recycle=3600, echo=echo) return engine
def listen(cls): trail_blocks = Conf.get('trail_blocks') assert trail_blocks >= 0 assert trail_blocks < 25 steemd = SteemClient.instance() hive_head = Blocks.head_num() for block in steemd.stream_blocks(hive_head + 1, trail_blocks, max_gap=40): start_time = time.perf_counter() query("START TRANSACTION") num = Blocks.process(block) follows = Follow.flush(trx=False) accts = Accounts.flush(trx=False, period=8) CachedPost.dirty_paidouts(block['timestamp']) cnt = CachedPost.flush(trx=False) query("COMMIT") ms = (time.perf_counter() - start_time) * 1000 print("[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits," "% 3d payouts,% 3d votes,% 3d accounts,% 3d follows --% 5dms%s" % (num, block['timestamp'], len(block['transactions']), cnt['insert'], cnt['update'], cnt['payout'], cnt['upvote'], accts, follows, int(ms), ' SLOW' if ms > 1000 else '')) # once per hour, update accounts if num % 1200 == 0: Accounts.dirty_oldest(10000) Accounts.flush(trx=True) #Accounts.update_ranks() # once a minute, update chain props if num % 20 == 0: cls._update_chain_state(steemd)
def create_engine(echo=False): """Create a new SA db engine. Use echo=True for ultra verbose.""" engine = sqlalchemy.create_engine( Conf.get('database_url'), isolation_level="READ UNCOMMITTED", # only works in mysql pool_recycle=3600, echo=echo) return engine
def listen(cls): """Live (block following) mode.""" trail_blocks = Conf.get('trail_blocks') assert trail_blocks >= 0 assert trail_blocks <= 100 # debug: no max gap if disable_sync in effect max_gap = None if Conf.get('disable_sync') else 100 steemd = SteemClient.instance() hive_head = Blocks.head_num() for block in steemd.stream_blocks(hive_head + 1, trail_blocks, max_gap): start_time = perf() query("START TRANSACTION") num = Blocks.process(block) follows = Follow.flush(trx=False) accts = Accounts.flush(trx=False, spread=8) CachedPost.dirty_paidouts(block['timestamp']) cnt = CachedPost.flush(trx=False) query("COMMIT") ms = (perf() - start_time) * 1000 log.info( "[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits," "% 3d payouts,% 3d votes,% 3d accts,% 3d follows --% 5dms%s", num, block['timestamp'], len(block['transactions']), cnt['insert'], cnt['update'], cnt['payout'], cnt['upvote'], accts, follows, int(ms), ' SLOW' if ms > 1000 else '') # once per hour, update accounts if num % 1200 == 0: Accounts.dirty_oldest(10000) Accounts.flush(trx=True) #Accounts.update_ranks() # once a minute, update chain props if num % 20 == 0: cls._update_chain_state(steemd)
async def health(request): """Get hive health data. 500 if behind by more than a few secs.""" #pylint: disable=unused-argument is_syncer = Conf.get('sync_to_s3') max_head_age = (Conf.get('trail_blocks') + 1) * 3 try: state = await hive_api.db_head_state() except OperationalError as e: if 'could not connect to server: Connection refused' in str(e): logging.error("hive /health could not connect to db") state = None else: raise e if not state: status = 500 result = 'db not available' elif not is_syncer and state['db_head_age'] > max_head_age: status = 500 result = 'head block age (%s) > max (%s); head block num: %s' % ( state['db_head_age'], max_head_age, state['db_head_block']) else: status = 200 result = 'head block age is %d, head block num is %d' % ( state['db_head_age'], state['db_head_block']) return web.json_response( status=status, data=dict(state=state, result=result, status='OK' if status == 200 else 'WARN', sync_service=is_syncer, source_commit=os.environ.get('SOURCE_COMMIT'), schema_hash=os.environ.get('SCHEMA_HASH'), docker_tag=os.environ.get('DOCKER_TAG'), timestamp=datetime.utcnow().isoformat()))
def run(): Conf.init_argparse() mode = '/'.join(Conf.get('mode')) if mode == 'server': run_server() elif mode == 'sync': run_sync() elif mode == 'status': print(DbState.status()) else: raise Exception("unknown run mode %s" % mode)
async def health(request): #pylint: disable=unused-argument state = await hive_api.db_head_state() max_head_age = (Conf.get('trail_blocks') + 1) * 3 if state['db_head_age'] > max_head_age: status = 500 result = 'head block age (%s) > max (%s); head block num: %s' % ( state['db_head_age'], max_head_age, state['db_head_block']) else: status = 200 result = 'head block age is %d, head block num is %d' % ( state['db_head_age'], state['db_head_block']) return web.json_response( status=status, data=dict(state=state, result=result, status='OK' if status == 200 else 'WARN', source_commit=os.environ.get('SOURCE_COMMIT'), schema_hash=os.environ.get('SCHEMA_HASH'), docker_tag=os.environ.get('DOCKER_TAG'), timestamp=datetime.utcnow().isoformat()))
def instance(cls): if not cls._instance: cls._instance = SteemClient(url=Conf.get('steemd_url'), max_batch=Conf.get('max_batch'), max_workers=Conf.get('max_workers')) return cls._instance