def _process(cls, block, is_initial_sync=False): """Process a single block. Assumes a trx is open.""" #pylint: disable=too-many-branches num = cls._push(block) date = block['timestamp'] account_names = set() json_ops = [] for tx_idx, tx in enumerate(block['transactions']): for operation in tx['operations']: op_type = operation['type'] op = operation['value'] # account ops if op_type == 'pow_operation': account_names.add(op['worker_account']) elif op_type == 'pow2_operation': account_names.add( op['work']['value']['input']['worker_account']) elif op_type == 'account_create_operation': account_names.add(op['new_account_name']) elif op_type == 'account_create_with_delegation_operation': account_names.add(op['new_account_name']) elif op_type == 'create_claimed_account_operation': account_names.add(op['new_account_name']) # account metadata updates elif op_type == 'account_update_operation': if not is_initial_sync: Accounts.dirty(op['account']) # full elif op_type == 'account_update2_operation': if not is_initial_sync: Accounts.dirty(op['account']) # full # post ops elif op_type == 'comment_operation': Posts.comment_op(op, date) if not is_initial_sync: Accounts.dirty(op['author']) # lite - stats elif op_type == 'delete_comment_operation': Posts.delete_op(op) elif op_type == 'vote_operation': if not is_initial_sync: Accounts.dirty(op['author']) # lite - rep Accounts.dirty(op['voter']) # lite - stats CachedPost.vote(op['author'], op['permlink'], None, op['voter']) # misc ops elif op_type == 'transfer_operation': Payments.op_transfer(op, tx_idx, num, date) elif op_type == 'custom_json_operation': json_ops.append(op) Accounts.register(account_names, date) # register any new names CustomOp.process_ops(json_ops, num, date) # follow/reblog/community ops return num
def _process(cls, block, is_initial_sync=False): """Process a single block. Assumes a trx is open.""" # pylint: disable=too-many-boolean-expressions,too-many-branches num = cls._push(block) date = block['timestamp'] account_names = set() comment_ops = [] json_ops = [] delete_ops = [] voted_authors = set() for tx_idx, tx in enumerate(block['transactions']): for operation in tx['operations']: if isinstance(operation, dict): op_type = operation['type'].split('_operation')[0] op = operation['value'] else: # pre-appbase-style. remove after deploy. #APPBASE op_type, op = operation # account ops if op_type == 'pow': account_names.add(op['worker_account']) elif op_type == 'pow2': # old style. remove after #APPBASE #account_names.add(op['work'][1]['input']['worker_account']) account_names.add( op['work']['value']['input']['worker_account']) elif op_type == 'account_create': account_names.add(op['new_account_name']) elif op_type == 'account_create_with_delegation': account_names.add(op['new_account_name']) # post ops elif op_type == 'comment': comment_ops.append(op) elif op_type == 'delete_comment': delete_ops.append(op) elif op_type == 'vote': if not is_initial_sync: CachedPost.vote(op['author'], op['permlink']) voted_authors.add( op['author']) # TODO: move to cachedpost # misc ops elif op_type == 'transfer': Payments.op_transfer(op, tx_idx, num, date) elif op_type == 'custom_json': json_ops.append(op) Accounts.register(account_names, date) # register any new names Accounts.dirty(voted_authors) # update rep of voted authors Posts.comment_ops(comment_ops, date) # handle inserts, edits Posts.delete_ops(delete_ops) # handle post deletion CustomOp.process_ops(json_ops, num, date) # follow/reblog/community ops return num
def _process(cls, block, is_initial_sync=False): """Process a single block. Assumes a trx is open.""" num = cls._push(block) date = block['timestamp'] account_names = set() comment_ops = [] json_ops = [] delete_ops = [] for tx_idx, tx in enumerate(block['transactions']): for operation in tx['operations']: op_type = operation['type'] op = operation['value'] # account ops if op_type == 'pow_operation': account_names.add(op['worker_account']) elif op_type == 'pow2_operation': account_names.add( op['work']['value']['input']['worker_account']) elif op_type == 'account_create_operation': account_names.add(op['new_account_name']) elif op_type == 'account_create_with_delegation_operation': account_names.add(op['new_account_name']) elif op_type == 'create_claimed_account_operation': account_names.add(op['new_account_name']) # post ops elif op_type == 'comment_operation': comment_ops.append(op) elif op_type == 'delete_comment_operation': delete_ops.append(op) elif op_type == 'vote_operation': if not is_initial_sync: CachedPost.vote(op['author'], op['permlink']) # misc ops elif op_type == 'transfer_operation': Payments.op_transfer(op, tx_idx, num, date) elif op_type == 'custom_json_operation': json_ops.append(op) Accounts.register(account_names, date) # register any new names Posts.comment_ops(comment_ops, date) # handle inserts, edits Posts.delete_ops(delete_ops) # handle post deletion CustomOp.process_ops(json_ops, num, date) # follow/reblog/community ops return num
def _process(cls, block, is_initial_sync=False): num = cls._push(block) date = block['timestamp'] account_names = set() comment_ops = [] json_ops = [] delete_ops = [] voted_authors = set() for tx_idx, tx in enumerate(block['transactions']): for operation in tx['operations']: op_type, op = operation # account ops if op_type == 'pow': account_names.add(op['worker_account']) elif op_type == 'pow2': account_names.add(op['work'][1]['input']['worker_account']) elif op_type == 'account_create': account_names.add(op['new_account_name']) elif op_type == 'account_create_with_delegation': account_names.add(op['new_account_name']) # post ops elif op_type == 'comment': comment_ops.append(op) elif op_type == 'delete_comment': delete_ops.append(op) elif op_type == 'vote': if not is_initial_sync: CachedPost.vote(op['author'], op['permlink']) voted_authors.add( op['author']) # TODO: move to cachedpost # misc ops elif op_type == 'transfer': Payments.op_transfer(op, tx_idx, num, date) elif op_type == 'custom_json': json_ops.append(op) Accounts.register(account_names, date) # register any new names Accounts.dirty(voted_authors) # update rep of voted authors Posts.comment_ops(comment_ops, date) # handle inserts, edits Posts.delete_ops(delete_ops) # handle post deletion CustomOp.process_ops(json_ops, num, date) # follow/reblog/community ops return num
def _check_migrations(cls): """Check current migration version and perform updates as needed.""" cls._ver = cls.db().query_one( "SELECT db_version FROM hive_state LIMIT 1") assert cls._ver is not None, 'could not load state record' if cls._ver == 0: raise Exception("dbv cannot be 0; reindex required") if cls._ver == 1: cls._set_ver(2) if cls._ver == 2: cls._set_ver(3) if cls._ver == 3: sql = """CREATE INDEX hive_accounts_ix3 ON hive_accounts USING btree (vote_weight, name varchar_pattern_ops)""" cls.db().query(sql) cls._set_ver(4) if cls._ver == 4: sql = """CREATE INDEX hive_follows_ix4 ON public.hive_follows USING btree (follower, following) WHERE state = 2;""" cls.db().query(sql) cls._set_ver(5) if cls._ver == 5: # recover acct names lost to issue #151 from hive.steem.client import SteemClient from hive.indexer.accounts import Accounts names = SteemClient().get_all_account_names() Accounts.load_ids() Accounts.register(names, '1970-01-01T00:00:00') Accounts.clear_ids() cls._set_ver(6) assert cls._ver == DB_VERSION, "migration missing or invalid DB_VERSION"
def _check_migrations(cls): """Check current migration version and perform updates as needed.""" #pylint: disable=line-too-long cls._ver = cls.db().query_one( "SELECT db_version FROM hive_state LIMIT 1") assert cls._ver is not None, 'could not load state record' if cls._ver == 0: raise Exception("dbv cannot be 0; reindex required") if cls._ver == 1: cls._set_ver(2) if cls._ver == 2: cls._set_ver(3) if cls._ver == 3: cls.db().query( "CREATE INDEX hive_accounts_ix3 ON hive_accounts (vote_weight, name varchar_pattern_ops)" ) cls._set_ver(4) if cls._ver == 4: cls.db().query( "CREATE INDEX hive_follows_ix4 ON hive_follows (follower, following) WHERE state = 2" ) cls._set_ver(5) if cls._ver == 5: # recover acct names lost to issue #151 from hive.steem.client import SteemClient from hive.indexer.accounts import Accounts names = SteemClient().get_all_account_names() Accounts.load_ids() Accounts.register(names, '1970-01-01T00:00:00') Accounts.clear_ids() cls._set_ver(6) if cls._ver == 6: cls.db().query("DROP INDEX hive_posts_cache_ix6") cls.db().query( "CREATE INDEX hive_posts_cache_ix6a ON hive_posts_cache (sc_trend, post_id) WHERE is_paidout = '0'" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix6b ON hive_posts_cache (post_id, sc_trend) WHERE is_paidout = '0'" ) cls.db().query("DROP INDEX hive_posts_cache_ix7") cls.db().query( "CREATE INDEX hive_posts_cache_ix7a ON hive_posts_cache (sc_hot, post_id) WHERE is_paidout = '0'" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix7b ON hive_posts_cache (post_id, sc_hot) WHERE is_paidout = '0'" ) cls._set_ver(7) if cls._ver == 7: cls.db().query( "CREATE INDEX hive_accounts_ix4 ON hive_accounts (id, name)") cls.db().query( "CREATE INDEX hive_accounts_ix5 ON hive_accounts (cached_at, name)" ) cls._set_ver(8) if cls._ver == 8: cls.db().query("DROP INDEX hive_follows_ix2") cls.db().query("DROP INDEX hive_follows_ix3") cls.db().query("DROP INDEX hive_follows_ix4") cls.db().query( "CREATE INDEX hive_follows_5a ON hive_follows (following, state, created_at, follower)" ) cls.db().query( "CREATE INDEX hive_follows_5b ON hive_follows (follower, state, created_at, following)" ) cls._set_ver(9) if cls._ver == 9: from hive.indexer.follow import Follow Follow.force_recount() cls._set_ver(10) if cls._ver == 10: cls.db().query( "CREATE INDEX hive_posts_cache_ix8 ON hive_posts_cache (category, payout, depth) WHERE is_paidout = '0'" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix9a ON hive_posts_cache (depth, payout, post_id) WHERE is_paidout = '0'" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix9b ON hive_posts_cache (category, depth, payout, post_id) WHERE is_paidout = '0'" ) cls._set_ver(11) if cls._ver == 11: cls.db().query("DROP INDEX hive_posts_ix1") cls.db().query("DROP INDEX hive_posts_ix2") cls.db().query( "CREATE INDEX hive_posts_ix3 ON hive_posts (author, depth, id) WHERE is_deleted = '0'" ) cls.db().query( "CREATE INDEX hive_posts_ix4 ON hive_posts (parent_id, id) WHERE is_deleted = '0'" ) cls._set_ver(12) reset_autovac(cls.db()) log.info("[HIVE] db version: %d", cls._ver) assert cls._ver == DB_VERSION, "migration missing or invalid DB_VERSION"
def process_block(block, is_initial_sync=False): date = block['timestamp'] block_id = block['block_id'] prev = block['previous'] block_num = int(block_id[:8], base=16) txs = block['transactions'] query( "INSERT INTO hive_blocks (num, hash, prev, txs, created_at) " "VALUES (:num, :hash, :prev, :txs, :date)", num=block_num, hash=block_id, prev=prev, txs=len(txs), date=date) accounts = set() comments = [] json_ops = [] deleted = [] dirty = set() for tx in txs: for operation in tx['operations']: op_type, op = operation if op_type == 'pow': accounts.add(op['worker_account']) elif op_type == 'pow2': accounts.add(op['work'][1]['input']['worker_account']) elif op_type in [ 'account_create', 'account_create_with_delegation' ]: accounts.add(op['new_account_name']) elif op_type == 'comment': comments.append(op) dirty.add(op['author'] + '/' + op['permlink']) Accounts.dirty(op['author']) if op['parent_author']: Accounts.dirty(op['parent_author']) elif op_type == 'delete_comment': deleted.append(op) elif op_type == 'custom_json': json_ops.append(op) elif op_type == 'vote': dirty.add(op['author'] + '/' + op['permlink']) Accounts.dirty(op['author']) Accounts.dirty(op['voter']) Accounts.register( accounts, date) # if an account does not exist, mark it as created in this block Posts.register( comments, date ) # if this is a new post, add the entry and validate community param Posts.delete(deleted) # mark hive_posts.is_deleted = 1 for op in json_ops: if op['id'] not in ['follow', 'com.steemit.community']: continue # we are assuming `required_posting_auths` is always used and length 1. # it may be that some ops will require `required_active_auths` instead # (e.g. if we use that route for admin action of acct creation) # if op['required_active_auths']: # log.warning("unexpected active auths: %s" % op) if len(op['required_posting_auths']) != 1: log.warning("unexpected auths: %s" % op) continue account = op['required_posting_auths'][0] op_json = load_json_key(op, 'json') if op['id'] == 'follow': if block_num < 6000000 and type(op_json) != list: op_json = ['follow', op_json] # legacy compat process_json_follow_op(account, op_json, date) elif op['id'] == 'com.steemit.community': if block_num > 13e6: process_json_community_op(account, op_json, date) # return all posts modified this block return dirty
def _process(cls, block): """Process a single block. Assumes a trx is open.""" #pylint: disable=too-many-branches assert issubclass(type(block), Block) num = cls._push(block) cls._current_block_date = block.get_date() # head block date shall point to last imported block (not yet current one) to conform hived behavior. # that's why operations processed by node are included in the block being currently produced, so its processing time is equal to last produced block. # unfortunately it is not true to all operations, most likely in case of dates that used to come from # FatNode where it supplemented it with its-current head block, since it was already past block processing, # it saw later block (equal to _current_block_date here) if cls._head_block_date is None: cls._head_block_date = cls._current_block_date ineffective_deleted_ops = Blocks.prepare_vops( Posts.comment_payout_ops, block, cls._current_block_date, num, num <= cls._last_safe_cashout_block) json_ops = [] for transaction in block.get_next_transaction(): assert issubclass(type(transaction), Transaction) for operation in transaction.get_next_operation(): assert issubclass(type(operation), Operation) start = OPSM.start() op_type = operation.get_type() assert op_type, "Only supported types are expected" op = operation.get_body() assert 'block_num' not in op op['block_num'] = num account_name = None op_details = None potentially_new_account = False # account ops if op_type == OperationType.Pow: account_name = op['worker_account'] potentially_new_account = True elif op_type == OperationType.Pow2: account_name = op['work']['value']['input'][ 'worker_account'] potentially_new_account = True elif op_type == OperationType.AccountCreate: account_name = op['new_account_name'] op_details = op potentially_new_account = True elif op_type == OperationType.AccountCreateWithDelegation: account_name = op['new_account_name'] op_details = op potentially_new_account = True elif op_type == OperationType.CreateClaimedAccount: account_name = op['new_account_name'] op_details = op potentially_new_account = True if potentially_new_account and not Accounts.register( account_name, op_details, cls._head_block_date, num): log.error( "Failed to register account {} from operation: {}". format(account_name, op)) # account metadata updates if op_type == OperationType.AccountUpdate: Accounts.update_op(op, False) elif op_type == OperationType.AccountUpdate2: Accounts.update_op(op, True) # post ops elif op_type == OperationType.Comment: Posts.comment_op(op, cls._head_block_date) elif op_type == OperationType.DeleteComment: key = "{}/{}".format(op['author'], op['permlink']) if key not in ineffective_deleted_ops: Posts.delete_op(op, cls._head_block_date) elif op_type == OperationType.CommentOption: Posts.comment_options_op(op) elif op_type == OperationType.Vote: Votes.vote_op(op, cls._head_block_date) # misc ops elif op_type == OperationType.Transfer: Payments.op_transfer(op, transaction.get_id(), num, cls._head_block_date) elif op_type == OperationType.CustomJson: # follow/reblog/community ops CustomOp.process_op(op, num, cls._head_block_date) OPSM.op_stats(str(op_type), OPSM.stop(start)) cls._head_block_date = cls._current_block_date return num
def _check_migrations(cls): """Check current migration version and perform updates as needed.""" #pylint: disable=line-too-long,too-many-branches,too-many-statements cls._ver = cls.db().query_one( "SELECT db_version FROM hive_state LIMIT 1") assert cls._ver is not None, 'could not load state record' if cls._ver == 0: raise Exception("dbv cannot be 0; reindex required") if cls._ver == 1: cls._set_ver(2) if cls._ver == 2: cls._set_ver(3) if cls._ver == 3: cls.db().query( "CREATE INDEX hive_accounts_ix3 ON hive_accounts (vote_weight, name varchar_pattern_ops)" ) cls._set_ver(4) if cls._ver == 4: cls.db().query( "CREATE INDEX hive_follows_ix4 ON hive_follows (follower, following) WHERE state = 2" ) cls._set_ver(5) if cls._ver == 5: # recover acct names lost to issue #151 from hive.steem.client import SteemClient from hive.indexer.accounts import Accounts names = SteemClient().get_all_account_names() Accounts.load_ids() Accounts.register(names, '1970-01-01T00:00:00') Accounts.clear_ids() cls._set_ver(6) if cls._ver == 6: cls.db().query("DROP INDEX hive_posts_cache_ix6") cls.db().query( "CREATE INDEX hive_posts_cache_ix6a ON hive_posts_cache (sc_trend, post_id) WHERE is_paidout = '0'" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix6b ON hive_posts_cache (post_id, sc_trend) WHERE is_paidout = '0'" ) cls.db().query("DROP INDEX hive_posts_cache_ix7") cls.db().query( "CREATE INDEX hive_posts_cache_ix7a ON hive_posts_cache (sc_hot, post_id) WHERE is_paidout = '0'" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix7b ON hive_posts_cache (post_id, sc_hot) WHERE is_paidout = '0'" ) cls._set_ver(7) if cls._ver == 7: cls.db().query( "CREATE INDEX hive_accounts_ix4 ON hive_accounts (id, name)") cls.db().query( "CREATE INDEX hive_accounts_ix5 ON hive_accounts (cached_at, name)" ) cls._set_ver(8) if cls._ver == 8: cls.db().query("DROP INDEX hive_follows_ix2") cls.db().query("DROP INDEX hive_follows_ix3") cls.db().query("DROP INDEX hive_follows_ix4") cls.db().query( "CREATE INDEX hive_follows_5a ON hive_follows (following, state, created_at, follower)" ) cls.db().query( "CREATE INDEX hive_follows_5b ON hive_follows (follower, state, created_at, following)" ) cls._set_ver(9) if cls._ver == 9: from hive.indexer.follow import Follow Follow.force_recount() cls._set_ver(10) if cls._ver == 10: cls.db().query( "CREATE INDEX hive_posts_cache_ix8 ON hive_posts_cache (category, payout, depth) WHERE is_paidout = '0'" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix9a ON hive_posts_cache (depth, payout, post_id) WHERE is_paidout = '0'" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix9b ON hive_posts_cache (category, depth, payout, post_id) WHERE is_paidout = '0'" ) cls._set_ver(11) if cls._ver == 11: cls.db().query("DROP INDEX hive_posts_ix1") cls.db().query("DROP INDEX hive_posts_ix2") cls.db().query( "CREATE INDEX hive_posts_ix3 ON hive_posts (author, depth, id) WHERE is_deleted = '0'" ) cls.db().query( "CREATE INDEX hive_posts_ix4 ON hive_posts (parent_id, id) WHERE is_deleted = '0'" ) cls._set_ver(12) if cls._ver == 12: # community schema assert False, 'not finalized' for table in [ 'hive_members', 'hive_flags', 'hive_modlog', 'hive_communities', 'hive_subscriptions', 'hive_roles', 'hive_notifs' ]: cls.db().query("DROP TABLE IF EXISTS %s" % table) build_metadata_community().create_all(cls.db().engine()) cls.db().query( "ALTER TABLE hive_accounts ADD COLUMN lr_notif_id integer") cls.db().query( "ALTER TABLE hive_posts DROP CONSTRAINT hive_posts_fk2") cls.db().query("ALTER TABLE hive_posts DROP COLUMN community") cls.db().query( "ALTER TABLE hive_posts ADD COLUMN community_id integer") cls.db().query( "ALTER TABLE hive_posts_cache ADD COLUMN community_id integer") cls._set_ver(13) if cls._ver == 13: sqls = ( "CREATE INDEX hive_posts_ix5 ON hive_posts (id) WHERE is_pinned = '1' AND is_deleted = '0'", "CREATE INDEX hive_posts_ix6 ON hive_posts (community_id, id) WHERE community_id IS NOT NULL AND is_pinned = '1' AND is_deleted = '0'", "CREATE INDEX hive_posts_cache_ix10 ON hive_posts_cache (post_id, payout) WHERE is_grayed = '1' AND payout > 0", "CREATE INDEX hive_posts_cache_ix30 ON hive_posts_cache (community_id, sc_trend, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0", "CREATE INDEX hive_posts_cache_ix31 ON hive_posts_cache (community_id, sc_hot, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0", "CREATE INDEX hive_posts_cache_ix32 ON hive_posts_cache (community_id, created_at, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND depth = 0", "CREATE INDEX hive_posts_cache_ix33 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '0' AND is_paidout = '0'", "CREATE INDEX hive_posts_cache_ix34 ON hive_posts_cache (community_id, payout, post_id) WHERE community_id IS NOT NULL AND is_grayed = '1' AND is_paidout = '0'" ) for sql in sqls: cls.db().query(sql) cls._set_ver(14) if cls._ver == 14: cls.db().query( "ALTER TABLE hive_communities ADD COLUMN primary_tag VARCHAR(32) NOT NULL DEFAULT ''" ) cls.db().query( "ALTER TABLE hive_communities ADD COLUMN category VARCHAR(32) NOT NULL DEFAULT ''" ) cls.db().query( "ALTER TABLE hive_communities ADD COLUMN avatar_url VARCHAR(1024) NOT NULL DEFAULT ''" ) cls.db().query( "ALTER TABLE hive_communities ADD COLUMN num_authors INTEGER NOT NULL DEFAULT 0" ) cls.db().query( "CREATE INDEX hive_posts_cache_ix20 ON hive_posts_cache (community_id, author, payout, post_id) WHERE is_paidout = '0'" ) cls._set_ver(15) if cls._ver == 15: cls.db().query("ALTER TABLE hive_accounts DROP COLUMN lr_notif_id") cls.db().query( "ALTER TABLE hive_accounts ADD COLUMN lastread_at TIMESTAMP WITHOUT TIME ZONE DEFAULT '1970-01-01 00:00:00' NOT NULL" ) cls.db().query( "CREATE INDEX hive_notifs_ix6 ON hive_notifs (dst_id, created_at, score, id) WHERE dst_id IS NOT NULL" ) cls._set_ver(16) reset_autovac(cls.db()) log.info("[HIVE] db version: %d", cls._ver) assert cls._ver == DB_VERSION, "migration missing or invalid DB_VERSION"