def reblog(cls, account, op_json, block_date): """Handle legacy 'reblog' op""" blogger = op_json['account'] author = op_json['author'] permlink = op_json['permlink'] if blogger != account: return # impersonation if not all(map(Accounts.exists, [author, blogger])): return post_id, depth = Posts.get_id_and_depth(author, permlink) if depth > 0: return # prevent comment reblogs if not post_id: log.debug("reblog: post not found: %s/%s", author, permlink) return if 'delete' in op_json and op_json['delete'] == 'delete': DB.query("DELETE FROM hive_reblogs WHERE account = :a AND " "post_id = :pid LIMIT 1", a=blogger, pid=post_id) if not DbState.is_initial_sync(): FeedCache.delete(post_id, Accounts.get_id(blogger)) else: sql = ("INSERT INTO hive_reblogs (account, post_id, created_at) " "VALUES (:a, :pid, :date) ON CONFLICT (account, post_id) DO NOTHING") DB.query(sql, a=blogger, pid=post_id, date=block_date) if not DbState.is_initial_sync(): FeedCache.insert(post_id, Accounts.get_id(blogger), block_date)
def run(cls): """Initialize state; setup/recovery checks; sync and runloop.""" # ensure db schema up to date, check app status DbState.initialize() # prefetch id->name memory map Accounts.load_ids() if DbState.is_initial_sync(): # resume initial sync cls.initial() DbState.finish_initial_sync() else: # recover from fork Blocks.verify_head() # perform cleanup if process did not exit cleanly CachedPost.recover_missing_posts() # debug mode: no sync, just stream if Conf.get('disable_sync'): return cls.listen() while True: # sync up to irreversible block cls.from_steemd() # take care of payout backlog CachedPost.dirty_paidouts(Blocks.head_date()) CachedPost.flush(trx=True) # listen for new blocks cls.listen()
def run_sync(): print("[HIVE] Welcome to hivemind") # make sure db schema is up to date, perform checks DbState.initialize() # prefetch id->name memory map Accounts.load_ids() if DbState.is_initial_sync(): # resume initial sync Sync.initial() DbState.finish_initial_sync() else: # recover from fork Blocks.verify_head() # perform cleanup in case process did not exit cleanly CachedPost.recover_missing_posts() while True: # sync up to irreversible block Sync.from_steemd() # take care of payout backlog CachedPost.dirty_paidouts(Blocks.head_date()) CachedPost.flush(trx=True) # start listening Sync.listen()
def run(self): """Initialize state; setup/recovery checks; sync and runloop.""" # ensure db schema up to date, check app status DbState.initialize() # prefetch id->name and id->rank memory maps Accounts.load_ids() Accounts.fetch_ranks() Community.recalc_pending_payouts() if DbState.is_initial_sync(): # resume initial sync self.initial() DbState.finish_initial_sync() else: # recover from fork Blocks.verify_head(self._steem) # perform cleanup if process did not exit cleanly CachedPost.recover_missing_posts(self._steem) #audit_cache_missing(self._db, self._steem) #audit_cache_deleted(self._db) self._update_chain_state() if self._conf.get('test_max_block'): # debug mode: partial sync return self.from_steemd() if self._conf.get('test_disable_sync'): # debug mode: no sync, just stream return self.listen() while True: # sync up to irreversible block self.from_steemd() # take care of payout backlog CachedPost.dirty_paidouts(Blocks.head_date()) CachedPost.flush(self._steem, trx=True) try: # listen for new blocks self.listen() except MicroForkException as e: # attempt to recover by restarting stream log.error("NOTIFYALERT microfork: %s", repr(e))
def insert(cls, post_id, account_id, created_at): """Inserts a [re-]post by an account into feed.""" assert not DbState.is_initial_sync(), 'writing to feed cache in sync' sql = """INSERT INTO hive_feed_cache (account_id, post_id, created_at) VALUES (:account_id, :id, :created_at) ON CONFLICT (account_id, post_id) DO NOTHING""" query(sql, account_id=account_id, id=post_id, created_at=created_at)
def op_transfer(cls, op, tx_idx, num, date): """Process raw transfer op; apply balance if valid post promote.""" NativeAd.check_ad_payment(op, date, num) record = cls._validated(op, tx_idx, num, date) if not record: return # add payment record sql = DB.build_insert('hive_payments', record, pk='id') DB.query(sql) # read current amount sql = "SELECT promoted FROM hive_posts WHERE id = :id" curr_amount = DB.query_one(sql, id=record['post_id']) new_amount = curr_amount + record['amount'] # update post record sql = "UPDATE hive_posts SET promoted = :val WHERE id = :id" DB.query(sql, val=new_amount, id=record['post_id']) # notify cached_post of new promoted balance, and trigger update if not DbState.is_initial_sync(): CachedPost.update_promoted_amount(record['post_id'], new_amount) author, permlink = cls._split_url(op['memo']) CachedPost.vote(author, permlink, record['post_id'])
def follow_op(cls, account, op_json, date): """Process an incoming follow op.""" op = cls._validated_op(account, op_json, date) if not op: return # perform delta check new_state = op['state'] old_state = cls._get_follow_db_state(op['flr'], op['flg']) if new_state == (old_state or 0): return # insert or update state if old_state is None: sql = """INSERT INTO hive_follows (follower, following, created_at, state) VALUES (:flr, :flg, :at, :state)""" else: sql = """UPDATE hive_follows SET state = :state WHERE follower = :flr AND following = :flg""" DB.query(sql, **op) # track count deltas if not DbState.is_initial_sync(): if new_state == 1: Follow.follow(op['flr'], op['flg']) if old_state == 1: Follow.unfollow(op['flr'], op['flg'])
def run(): """Run the proper routine as indicated by hive --mode argument.""" conf = Conf.init_argparse() Db.set_shared_instance(conf.db()) mode = conf.mode() if mode == 'server': from hive.server.serve import run_server run_server(conf=conf) elif mode == 'sync': from hive.indexer.sync import Sync Sync(conf=conf).run() elif mode == 'status': from hive.db.db_state import DbState print(DbState.status()) #elif mode == 'sync-profile': # from hive.indexer.sync import Sync # from hive.utils.profiler import Profiler # with Profiler(): # Sync(conf=conf).run() else: raise Exception("unknown run mode %s" % mode)
def insert(cls, op, date): """Inserts new post records.""" sql = """INSERT INTO hive_posts (is_valid, is_muted, parent_id, author, permlink, category, community_id, depth, created_at) VALUES (:is_valid, :is_muted, :parent_id, :author, :permlink, :category, :community_id, :depth, :date)""" sql += ";SELECT currval(pg_get_serial_sequence('hive_posts','id'))" post = cls._build_post(op, date) result = DB.query(sql, **post) post['id'] = int(list(result)[0][0]) cls._set_id(op['author'] + '/' + op['permlink'], post['id']) if not DbState.is_initial_sync(): if post['error']: author_id = Accounts.get_id(post['author']) Notify('error', dst_id=author_id, when=date, post_id=post['id'], payload=post['error']).write() CachedPost.insert(op['author'], op['permlink'], post['id']) if op['parent_author']: # update parent's child count CachedPost.recount(op['parent_author'], op['parent_permlink'], post['parent_id']) cls._insert_feed_cache(post)
def initial(self): """Initial sync routine.""" assert DbState.is_initial_sync(), "already synced" log.info("[INIT] *** Initial fast sync ***") self.from_steemd(is_initial_sync=True) if not can_continue_thread(): return
def delete(cls, op): pid, depth = cls.get_id_and_depth(op['author'], op['permlink']) query("UPDATE hive_posts SET is_deleted = '1' WHERE id = :id", id=pid) if not DbState.is_initial_sync(): CachedPost.delete(pid, op['author'], op['permlink']) if depth == 0: FeedCache.delete(pid)
def update(cls, op, date, pid): """Handle post updates. Here we could also build content diffs, but for now just used a signal to update cache record. """ # pylint: disable=unused-argument if not DbState.is_initial_sync(): CachedPost.update(op['author'], op['permlink'], pid)
def undelete(cls, op, date, pid): sql = """UPDATE hive_posts SET is_valid = :is_valid, is_deleted = '0', parent_id = :parent_id, category = :category, community = :community, depth = :depth WHERE id = :id""" post = cls._build_post(op, date, pid) DB.query(sql, **post) if not DbState.is_initial_sync(): CachedPost.undelete(pid, post['author'], post['permlink']) cls._insert_feed_cache(post)
def initial(cls): assert DbState.is_initial_sync(), "already synced" print("[INIT] *** Initial fast sync ***") cls.from_checkpoints() cls.from_steemd(is_initial_sync=True) print("[INIT] *** Initial cache build ***") # TODO: disable indexes during this process CachedPost.recover_missing_posts() FeedCache.rebuild()
def initial(self): """Initial sync routine.""" assert DbState.is_initial_sync(), "already synced" log.info("[INIT] *** Initial fast sync ***") self.from_checkpoints() self.from_steemd(is_initial_sync=True) log.info("[INIT] *** Initial cache build ***") CachedPost.recover_missing_posts(self._steem) FeedCache.rebuild() Follow.force_recount()
def delete(cls, post_id, account_id=None): """Remove a post from feed cache. If `account_id` is specified, we remove a single entry (e.g. a singular un-reblog). Otherwise, we remove all instances of the post (e.g. a post was deleted; its entry and all reblogs need to be removed. """ assert not DbState.is_initial_sync(), 'writing to feed cache in sync' sql = "DELETE FROM hive_feed_cache WHERE post_id = :id" if account_id: sql = sql + " AND account_id = :account_id" query(sql, account_id=account_id, id=post_id)
def sync_from_steemd(): is_initial_sync = DbState.is_initial_sync() steemd = get_adapter() lbound = Blocks.head_num() + 1 ubound = steemd.last_irreversible() if ubound <= lbound: return _abort = False try: print("[SYNC] start block %d, +%d to sync" % (lbound, ubound-lbound+1)) timer = Timer(ubound - lbound, entity='block', laps=['rps', 'wps']) while lbound < ubound: to = min(lbound + 1000, ubound) timer.batch_start() blocks = steemd.get_blocks_range(lbound, to) timer.batch_lap() Blocks.process_multi(blocks, is_initial_sync) timer.batch_finish(len(blocks)) date = blocks[-1]['timestamp'] print(timer.batch_status("[SYNC] Got block %d @ %s" % (to-1, date))) lbound = to except KeyboardInterrupt: traceback.print_exc() print("\n\n[SYNC] Aborted.. cleaning up..") _abort = True if not is_initial_sync: # Follows flushing may need to be moved closer to core (i.e. moved # into main block transactions). Important to keep in sync since # we need to prevent expensive recounts. This will fail if we aborted # in the middle of a transaction, meaning data loss. Better than # forcing it, however, since in-memory cache will be out of sync # with db state. Follow.flush(trx=True) # This flush is low importance; accounts are swept regularly. if not _abort: Accounts.flush(trx=True) # If this flush fails, all that could potentially be lost here is # edits and pre-payout votes. If the post has not been paid out yet, # then the worst case is it will be synced upon payout. If the post # is already paid out, worst case is to lose an edit. CachedPost.flush(trx=True) if _abort: print("[SYNC] Aborted") exit()
def run(): print("[HIVE] Welcome to hivemind") # make sure db schema is up to date, perform checks DbState.initialize() # prefetch id->name memory map Accounts.load_ids() if DbState.is_initial_sync(): print("[INIT] *** Initial fast sync ***") sync_from_checkpoints() sync_from_steemd() print("[INIT] *** Initial cache build ***") # todo: disable indexes during this process cache_missing_posts() FeedCache.rebuild() DbState.finish_initial_sync() else: # recover from fork Blocks.verify_head() # perform cleanup in case process did not exit cleanly cache_missing_posts() while True: # sync up to irreversible block sync_from_steemd() # take care of payout backlog CachedPost.dirty_paidouts(Blocks.head_date()) CachedPost.flush(trx=True) # start listening listen_steemd()
def delete(cls, op): """Marks a post record as being deleted.""" pid, depth = cls.get_id_and_depth(op['author'], op['permlink']) DB.query("UPDATE hive_posts SET is_deleted = '1' WHERE id = :id", id=pid) if not DbState.is_initial_sync(): CachedPost.delete(pid, op['author'], op['permlink']) if depth == 0: # TODO: delete from hive_reblogs -- otherwise feed cache gets populated with deleted posts somwrimas FeedCache.delete(pid) else: # force parent child recount when child is deleted prnt = cls._get_parent_by_child_id(pid) CachedPost.recount(prnt['author'], prnt['permlink'], prnt['id'])
def insert(cls, op, date): sql = """INSERT INTO hive_posts (is_valid, parent_id, author, permlink, category, community, depth, created_at) VALUES (:is_valid, :parent_id, :author, :permlink, :category, :community, :depth, :date)""" sql += ";SELECT currval(pg_get_serial_sequence('hive_posts','id'))" post = cls._build_post(op, date) result = DB.query(sql, **post) post['id'] = int(list(result)[0][0]) cls._set_id(op['author'] + '/' + op['permlink'], post['id']) if not DbState.is_initial_sync(): CachedPost.insert(op['author'], op['permlink'], post['id']) cls._insert_feed_cache(post)
def run(): Conf.init_argparse() mode = '/'.join(Conf.get('mode')) if mode == 'server': run_server() elif mode == 'sync': run_sync() elif mode == 'status': print(DbState.status()) else: raise Exception("unknown run mode %s" % mode)
def launch_mode(mode, conf): """Launch a routine as indicated by `mode`.""" if mode == 'server': from hive.server.serve import run_server run_server(conf=conf) elif mode == 'sync': from hive.indexer.sync import Sync Sync(conf=conf).run() elif mode == 'status': from hive.db.db_state import DbState print(DbState.status()) else: raise Exception("unknown run mode %s" % mode)
def run(): """Run the proper routine as indicated by hive --mode argument.""" Conf.init_argparse() mode = Conf.run_mode() if mode == 'server': run_server() elif mode == 'sync': Sync.run() elif mode == 'status': print(DbState.status()) else: raise Exception("unknown run mode %s" % mode)
def undelete(cls, op, date, pid): """Re-allocates an existing record flagged as deleted.""" sql = """UPDATE hive_posts SET is_valid = :is_valid, is_muted = :is_muted, is_deleted = '0', is_pinned = '0', parent_id = :parent_id, category = :category, community_id = :community_id, depth = :depth WHERE id = :id""" post = cls._build_post(op, date, pid) DB.query(sql, **post) if not DbState.is_initial_sync(): if post['error']: author_id = Accounts.get_id(post['author']) Notify('error', dst_id=author_id, when=date, post_id=post['id'], payload=post['error']).write() CachedPost.undelete(pid, post['author'], post['permlink'], post['category']) cls._insert_feed_cache(post)
def run(): """Run the proper routine as indicated by hive --mode argument.""" conf = Conf.init_argparse() Db.set_shared_instance(conf.db()) mode = '/'.join(conf.get('mode')) if mode == 'server': from hive.server.serve import run_server run_server(conf=conf) elif mode == 'sync': from hive.indexer.sync import Sync Sync(conf=conf).run() elif mode == 'status': from hive.db.db_state import DbState print(DbState.status()) else: raise Exception("unknown run mode %s" % mode)
def _notify(self, op, **kwargs): if DbState.is_initial_sync(): # TODO: set start date for notifs? # TODO: address other callers return dst_id = None score = 35 if self.account_id and not self.post_id: dst_id = self.account_id if not self._subscribed(self.account_id): score = 15 Notify(op, src_id=self.actor_id, dst_id=dst_id, post_id=self.post_id, when=self.date, community_id=self.community_id, score=score, **kwargs).write()
def process(self): """Applies a validated operation.""" assert self.valid, 'cannot apply invalid op' from hive.indexer.cached_post import CachedPost action = self.action params = dict( date=self.date, community=self.community, community_id=self.community_id, actor=self.actor, actor_id=self.actor_id, account=self.account, account_id=self.account_id, post_id=self.post_id, role_id=self.role_id, notes=self.notes, title=self.title, ) # Community-level commands if action == 'updateProps': bind = ', '.join([k + " = :" + k for k in list(self.props.keys())]) DB.query("UPDATE hive_communities SET %s WHERE id = :id" % bind, id=self.community_id, **self.props) self._notify('set_props', payload=json.dumps(read_key_dict(self.op, 'props'))) elif action == 'subscribe': DB.query( """INSERT INTO hive_subscriptions (account_id, community_id, created_at) VALUES (:actor_id, :community_id, :date)""", **params) DB.query( """UPDATE hive_communities SET subscribers = subscribers + 1 WHERE id = :community_id""", **params) self._notify('subscribe') elif action == 'unsubscribe': DB.query( """DELETE FROM hive_subscriptions WHERE account_id = :actor_id AND community_id = :community_id""", **params) DB.query( """UPDATE hive_communities SET subscribers = subscribers - 1 WHERE id = :community_id""", **params) # Account-level actions elif action == 'setRole': DB.query( """INSERT INTO hive_roles (account_id, community_id, role_id, created_at) VALUES (:account_id, :community_id, :role_id, :date) ON CONFLICT (account_id, community_id) DO UPDATE SET role_id = :role_id""", **params) self._notify('set_role', payload=Role(self.role_id).name) elif action == 'setUserTitle': DB.query( """INSERT INTO hive_roles (account_id, community_id, title, created_at) VALUES (:account_id, :community_id, :title, :date) ON CONFLICT (account_id, community_id) DO UPDATE SET title = :title""", **params) self._notify('set_label', payload=self.title) # Post-level actions elif action == 'mutePost': DB.query( """UPDATE hive_posts SET is_muted = '1' WHERE id = :post_id""", **params) self._notify('mute_post', payload=self.notes) if not DbState.is_initial_sync(): CachedPost.update(self.account, self.permlink, self.post_id) elif action == 'unmutePost': DB.query( """UPDATE hive_posts SET is_muted = '0' WHERE id = :post_id""", **params) self._notify('unmute_post', payload=self.notes) if not DbState.is_initial_sync(): CachedPost.update(self.account, self.permlink, self.post_id) elif action == 'pinPost': DB.query( """UPDATE hive_posts SET is_pinned = '1' WHERE id = :post_id""", **params) self._notify('pin_post', payload=self.notes) elif action == 'unpinPost': DB.query( """UPDATE hive_posts SET is_pinned = '0' WHERE id = :post_id""", **params) self._notify('unpin_post', payload=self.notes) elif action == 'flagPost': self._notify('flag_post', payload=self.notes) return True
async def db_head_state(): return DbState.status()
def delete(cls, post_id, account_id=None): assert not DbState.is_initial_sync(), 'writing to feed cache in sync' sql = "DELETE FROM hive_feed_cache WHERE post_id = :id" if account_id: sql = sql + " AND account_id = :account_id" query(sql, account_id=account_id, id=post_id)
def run(self): old_sig_int_handler = getsignal(SIGINT) old_sig_term_handler = getsignal(SIGTERM) set_handlers() Community.start_block = self._conf.get("community_start_block") # ensure db schema up to date, check app status DbState.initialize() if self._conf.get("log_explain_queries"): is_superuser = self._db.query_one("SELECT is_superuser()") assert is_superuser, 'The parameter --log_explain_queries=true can be used only when connect to the database with SUPERUSER privileges' _is_consistency = Blocks.is_consistency() if not _is_consistency: raise RuntimeError( "Fatal error related to `hive_blocks` consistency") show_info(self._db) paths = self._conf.get("mock_block_data_path") or [] for path in paths: self.load_mock_data(path) mock_vops_data_path = self._conf.get("mock_vops_data_path") if mock_vops_data_path: MockVopsProvider.load_block_data(mock_vops_data_path) # MockVopsProvider.print_data() # prefetch id->name and id->rank memory maps Accounts.load_ids() # community stats update_communities_posts_and_rank(self._db) last_imported_block = Blocks.head_num() hived_head_block = self._conf.get( 'test_max_block') or self._steem.last_irreversible() log.info("target_head_block : %s", hived_head_block) if DbState.is_initial_sync(): DbState.before_initial_sync(last_imported_block, hived_head_block) # resume initial sync self.initial() if not can_continue_thread(): restore_handlers() return current_imported_block = Blocks.head_num() # beacuse we cannot break long sql operations, then we back default CTRL+C # behavior for the time of post initial actions restore_handlers() try: DbState.finish_initial_sync(current_imported_block) except KeyboardInterrupt: log.info("Break finish initial sync") set_exception_thrown() return set_handlers() else: # recover from fork Blocks.verify_head(self._steem) self._update_chain_state() global trail_blocks trail_blocks = self._conf.get('trail_blocks') assert trail_blocks >= 0 assert trail_blocks <= 100