def _valid_payment(cls, memo): """Checks for valid ad payment memo. Example memo: `hna:hive-133333/interesting-promo`""" if memo[:4] == "hna:": ref = memo[4:].strip( ) # strip() to avoid invalidating legitimate payments assert ref.count('/') == 1, ( "invalid ad payment memo; found (%d) / characters instead of 1" % ref.count) _values = ref.split('/') comm = _values[0].strip() link = _values[1].strip() from hive.indexer.community import Community valid_comm = Community.validated_name(comm) assert valid_comm, 'invalid community name entered (%s)' % comm comm_id = Community.get_id(comm) assert comm_id, 'community not found: %s' % comm return { 'community_id': comm_id, 'community_name': comm, 'permlink': link } return None
def register(cls, names, block_date): """Block processing: register "candidate" names. There are four ops which can result in account creation: *account_create*, *account_create_with_delegation*, *pow*, and *pow2*. *pow* ops result in account creation only when the account they name does not already exist! """ # filter out names which already registered new_names = list(filter(lambda n: not cls.exists(n), set(names))) if not new_names: return for name in new_names: DB.query( "INSERT INTO hive_accounts (name, created_at) " "VALUES (:name, :date)", name=name, date=block_date) # pull newly-inserted ids and merge into our map sql = "SELECT name, id FROM hive_accounts WHERE name IN :names" for name, _id in DB.query_all(sql, names=tuple(new_names)): cls._ids[name] = _id # post-insert: pass to communities to check for new registrations from hive.indexer.community import Community, START_DATE if block_date > START_DATE: Community.register(new_names, block_date)
def register(cls, name, op_details, block_date, block_num): """Block processing: register "candidate" names. There are four ops which can result in account creation: *account_create*, *account_create_with_delegation*, *pow*, and *pow2*. *pow* ops result in account creation only when the account they name does not already exist! """ if name is None: return False # filter out names which already registered if cls.exists(name): return True ( _posting_json_metadata, _json_metadata ) = get_profile_str( op_details ) sql = """ INSERT INTO hive_accounts (name, created_at, posting_json_metadata, json_metadata ) VALUES ( '{}', '{}', {}, {} ) RETURNING id """.format( name, block_date, cls.get_json_data( _posting_json_metadata ), cls.get_json_data( _json_metadata ) ) new_id = DB.query_one( sql ) if new_id is None: return False cls._ids[name] = new_id # post-insert: pass to communities to check for new registrations from hive.indexer.community import Community if block_num > Community.start_block: Community.register(name, block_date, block_num) return True
def _build_post(cls, op, date, pid=None): """Validate and normalize a post operation. Post is muted if: - parent was muted - author unauthorized Post is invalid if: - parent is invalid - author unauthorized """ # TODO: non-nsfw post in nsfw community is `invalid` # if this is a top-level post: if not op['parent_author']: parent_id = None depth = 0 category = op['parent_permlink'] community_id = None if date > START_DATE: community_id = Community.validated_id(category) is_valid = True is_muted = False # this is a comment; inherit parent props. else: parent_id = cls.get_id(op['parent_author'], op['parent_permlink']) sql = """SELECT depth, category, community_id, is_valid, is_muted FROM hive_posts WHERE id = :id""" (parent_depth, category, community_id, is_valid, is_muted) = DB.query_row(sql, id=parent_id) depth = parent_depth + 1 if not is_valid: error = 'replying to invalid post' elif is_muted: error = 'replying to muted post' # check post validity in specified context error = None if community_id and is_valid and not Community.is_post_valid( community_id, op): error = 'not authorized' #is_valid = False # TODO: reserved for future blacklist status? is_muted = True return dict(author=op['author'], permlink=op['permlink'], id=pid, is_valid=is_valid, is_muted=is_muted, parent_id=parent_id, depth=depth, category=category, community_id=community_id, date=date, error=error)
def run(self): """Initialize state; setup/recovery checks; sync and runloop.""" # ensure db schema up to date, check app status DbState.initialize() # prefetch id->name and id->rank memory maps Accounts.load_ids() Accounts.fetch_ranks() Community.recalc_pending_payouts() if DbState.is_initial_sync(): # resume initial sync self.initial() DbState.finish_initial_sync() else: # recover from fork Blocks.verify_head(self._steem) # perform cleanup if process did not exit cleanly CachedPost.recover_missing_posts(self._steem) #audit_cache_missing(self._db, self._steem) #audit_cache_deleted(self._db) self._update_chain_state() if self._conf.get('test_max_block'): # debug mode: partial sync return self.from_steemd() if self._conf.get('test_disable_sync'): # debug mode: no sync, just stream return self.listen() while True: # sync up to irreversible block self.from_steemd() # take care of payout backlog CachedPost.dirty_paidouts(Blocks.head_date()) CachedPost.flush(self._steem, trx=True) try: # listen for new blocks self.listen() except MicroForkException as e: # attempt to recover by restarting stream log.error("NOTIFYALERT microfork: %s", repr(e))
def listen(self): """Live (block following) mode.""" trail_blocks = self._conf.get('trail_blocks') assert trail_blocks >= 0 assert trail_blocks <= 100 # debug: no max gap if disable_sync in effect max_gap = None if self._conf.get('test_disable_sync') else 100 steemd = self._steem hive_head = Blocks.head_num() for block in steemd.stream_blocks(hive_head + 1, trail_blocks, max_gap): start_time = perf() self._db.query("START TRANSACTION") num = Blocks.process(block) follows = Follow.flush(trx=False) accts = Accounts.flush(steemd, trx=False, spread=8) CachedPost.dirty_paidouts(block['timestamp']) cnt = CachedPost.flush(steemd, trx=False) self._db.query("COMMIT") ms = (perf() - start_time) * 1000 log.info( "[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits," "% 3d payouts,% 3d votes,% 3d counts,% 3d accts,% 3d follows" " --% 5dms%s", num, block['timestamp'], len(block['transactions']), cnt['insert'], cnt['update'], cnt['payout'], cnt['upvote'], cnt['recount'], accts, follows, ms, ' SLOW' if ms > 1000 else '') if num % 1200 == 0: #1hr log.warning("head block %d @ %s", num, block['timestamp']) log.info("[LIVE] hourly stats") Accounts.fetch_ranks() #Community.recalc_pending_payouts() if num % 200 == 0: #10min Community.recalc_pending_payouts() if num % 100 == 0: #5min log.info("[LIVE] 5-min stats") Accounts.dirty_oldest(500) if num % 20 == 0: #1min self._update_chain_state()
def _verify_post_against_community(cls, op, community_id, is_valid, is_muted): error = None if community_id and is_valid and not Community.is_post_valid( community_id, op): error = 'not authorized' #is_valid = False # TODO: reserved for future blacklist status? is_muted = True return error