def reblog(cls, account, op_json, block_date): """Handle legacy 'reblog' op""" blogger = op_json['account'] author = op_json['author'] permlink = op_json['permlink'] if blogger != account: return # impersonation if not all(map(Accounts.exists, [author, blogger])): return post_id, depth = Posts.get_id_and_depth(author, permlink) if depth > 0: return # prevent comment reblogs if not post_id: log.debug("reblog: post not found: %s/%s", author, permlink) return if 'delete' in op_json and op_json['delete'] == 'delete': DB.query("DELETE FROM hive_reblogs WHERE account = :a AND " "post_id = :pid LIMIT 1", a=blogger, pid=post_id) if not DbState.is_initial_sync(): FeedCache.delete(post_id, Accounts.get_id(blogger)) else: sql = ("INSERT INTO hive_reblogs (account, post_id, created_at) " "VALUES (:a, :pid, :date) ON CONFLICT (account, post_id) DO NOTHING") DB.query(sql, a=blogger, pid=post_id, date=block_date) if not DbState.is_initial_sync(): FeedCache.insert(post_id, Accounts.get_id(blogger), block_date)
def _validated(cls, op, tx_idx, num, date): if op['to'] != 'null': return # only care about payments to null amount, token = parse_amount(op['amount']) if token != 'SBD': return # only care about SBD payments url = op['memo'] if not cls._validate_url(url): print("invalid url: {}".format(url)) return # invalid url author, permlink = cls._split_url(url) if not Accounts.exists(author): return post_id = Posts.get_id(author, permlink) if not post_id: print("post does not exist: %s" % url) return return { 'id': None, 'block_num': num, 'tx_idx': tx_idx, 'post_id': post_id, 'from_account': Accounts.get_id(op['from']), 'to_account': Accounts.get_id(op['to']), 'amount': amount, 'token': token }
def _validated(cls, op, tx_idx, num, date): """Validate and normalize the transfer op.""" # pylint: disable=unused-argument if op['to'] != 'null': return # only care about payments to null amount, token = parse_amount(op['amount']) if token != 'HBD': return # only care about HBD payments url = op['memo'] if not cls._validate_url(url): log.debug("invalid url: %s", url) return # invalid url author, permlink = cls._split_url(url) author_id = Accounts.get_id_noexept(author) if not author_id: return return [{ 'id': None, 'block_num': num, 'tx_idx': tx_idx, 'from_account': Accounts.get_id(op['from']), 'to_account': Accounts.get_id(op['to']), 'amount': amount, 'token': token }, author_id, permlink]
def _notifs(cls, post, pid, level, payout): # pylint: disable=too-many-locals,too-many-branches author = post['author'] author_id = Accounts.get_id(author) parent_author = post['parent_author'] date = post['last_update'] # reply notif if level == 'insert' and parent_author and parent_author != author: irredeemable = parent_author in Mutes.all() parent_author_id = Accounts.get_id(parent_author) if not irredeemable and not cls._muted(parent_author_id, author_id): ntype = 'reply' if post['depth'] == 1 else 'reply_comment' Notify(ntype, src_id=author_id, dst_id=parent_author_id, score=Accounts.default_score(author), post_id=pid, when=date).write() # mentions notif if level in ('insert', 'update'): accounts = set(filter(Accounts.exists, mentions(post['body']))) accounts -= {author, parent_author} score = Accounts.default_score(author) if score < 30: max_mentions = 5 elif score < 60: max_mentions = 10 else: max_mentions = 25 if len(accounts) <= max_mentions: penalty = min([score, 2 * (len(accounts) - 1)]) for mention in accounts: mention_id = Accounts.get_id(mention) if (not cls._mentioned(pid, mention_id) and not cls._muted(mention_id, author_id)): Notify('mention', src_id=author_id, dst_id=mention_id, post_id=pid, when=date, score=(score - penalty)).write() else: url = '@%s/%s' % (author, post['permlink']) log.info("skip %d mentions in %s", len(accounts), url) # votes notif url = post['author'] + '/' + post['permlink'] if url in cls._votes: voters = cls._votes[url] del cls._votes[url] net = float(post['net_rshares']) ratio = float(payout) / net if net else 0 for vote in post['active_votes']: rshares = int(vote['rshares']) if vote['voter'] not in voters or rshares < 10e9: continue contrib = int(1000 * ratio * rshares) if contrib < 20: continue # < $0.020 voter_id = Accounts.get_id(vote['voter']) if not cls._voted(pid, author_id, voter_id): score = min(100, (len(str(contrib)) - 1) * 25) # $1 = 75 payload = "$%.3f" % (contrib / 1000) Notify('vote', src_id=voter_id, dst_id=author_id, when=vote['time'], post_id=pid, score=score, payload=payload).write()
def check_ad_payment(cls, op, date, num): """Triggers an adFund operation for validated Native Ads transfers.""" memo = op['memo'] try: payment = cls._valid_payment(memo) if payment: amount, token = parse_amount(op['amount'], bypass_nai_lookup=True) params = { 'amount': amount, 'token': token, 'to_account': op['to'], 'community_name': payment['community_name'] } from hive.indexer.accounts import Accounts from hive.indexer.posts import Posts _post_id = Posts.get_id(op['from'], payment['permlink']) assert _post_id, 'post not found: @%s/%s' % ( op['from'], payment['permlink']) _account_id = Accounts.get_id(op['from']) _community_id = payment['community_id'] ad_op = NativeAdOp(_community_id, _post_id, _account_id, { 'action': 'adFund', 'params': params }, num) ad_op.validate_op() ad_op.process() except AssertionError as e: payload = str(e) Notify('error', dst_id=_account_id, when=date, payload=payload).write()
def insert(cls, op, date): """Inserts new post records.""" sql = """INSERT INTO hive_posts (is_valid, is_muted, parent_id, author, permlink, category, community_id, depth, created_at) VALUES (:is_valid, :is_muted, :parent_id, :author, :permlink, :category, :community_id, :depth, :date)""" sql += ";SELECT currval(pg_get_serial_sequence('hive_posts','id'))" post = cls._build_post(op, date) result = DB.query(sql, **post) post['id'] = int(list(result)[0][0]) cls._set_id(op['author'] + '/' + op['permlink'], post['id']) if not DbState.is_initial_sync(): if post['error']: author_id = Accounts.get_id(post['author']) Notify('error', dst_id=author_id, when=date, post_id=post['id'], payload=post['error']).write() CachedPost.insert(op['author'], op['permlink'], post['id']) if op['parent_author']: # update parent's child count CachedPost.recount(op['parent_author'], op['parent_permlink'], post['parent_id']) cls._insert_feed_cache(post)
def is_post_valid(cls, community_id, comment_op: dict): """ Given a new post/comment, check if valid as per community rules For a comment to be valid, these conditions apply: - Author is not muted in this community - For council post/comment, author must be a member - For journal post, author must be a member - Community must exist """ assert community_id, 'no community_id' community = cls._get_name(community_id) account_id = Accounts.get_id(comment_op['author']) role = cls.get_user_role(community_id, account_id) type_id = int(community[5]) # TODO: check `nsfw` tag requirement #267 # TODO: (1.5) check that beneficiaries are valid if type_id == TYPE_JOURNAL: if not comment_op['parent_author']: return role >= Role.member elif type_id == TYPE_COUNCIL: return role >= Role.member return role >= Role.guest # or at least not muted
def validate(self, raw_op): """Pre-processing and validation of custom_json payload.""" log.info("validating @%s op %s", self.actor, raw_op) try: # validate basic structure self._validate_raw_op(raw_op) self.action = raw_op[0] self.op = raw_op[1] self.actor_id = Accounts.get_id(self.actor) # validate and read schema self._read_schema() # validate permissions self._validate_permissions() self.valid = True except AssertionError as e: payload = str(e) Notify('error', dst_id=self.actor_id, when=self.date, payload=payload).write() return self.valid
def register(cls, names, block_date): """Block processing: hooks into new account registration. `Accounts` calls this method with any newly registered names. This method checks for any valid community names and inserts them. """ for name in names: #if not re.match(r'^hive-[123]\d{4,6}$', name): if not re.match(r'^hive-[1]\d{4,6}$', name): continue type_id = int(name[5]) _id = Accounts.get_id(name) # insert community sql = """INSERT INTO hive_communities (id, name, type_id, created_at) VALUES (:id, :name, :type_id, :date)""" DB.query(sql, id=_id, name=name, type_id=type_id, date=block_date) # insert owner sql = """INSERT INTO hive_roles (community_id, account_id, role_id, created_at) VALUES (:community_id, :account_id, :role_id, :date)""" DB.query(sql, community_id=_id, account_id=_id, role_id=Role.owner.value, date=block_date) Notify('new_community', src_id=None, dst_id=_id, when=block_date, community_id=_id).write()
def _validated_op(cls, account, op, date): """Validate and normalize the operation.""" if (not 'what' in op or not isinstance(op['what'], list) or not 'follower' in op or not 'following' in op): return what = first(op['what']) or '' defs = {'': 0, 'blog': 1, 'ignore': 2} if what not in defs: return if (op['follower'] == op['following'] # can't follow self or op['follower'] != account # impersonation or not Accounts.exists(op['following']) # invalid account or not Accounts.exists(op['follower'])): # invalid account return return dict(flr=Accounts.get_id(op['follower']), flg=Accounts.get_id(op['following']), state=defs[what], at=date)
def undelete(cls, op, date, pid): """Re-allocates an existing record flagged as deleted.""" sql = """UPDATE hive_posts SET is_valid = :is_valid, is_muted = :is_muted, is_deleted = '0', is_pinned = '0', parent_id = :parent_id, category = :category, community_id = :community_id, depth = :depth WHERE id = :id""" post = cls._build_post(op, date, pid) DB.query(sql, **post) if not DbState.is_initial_sync(): if post['error']: author_id = Accounts.get_id(post['author']) Notify('error', dst_id=author_id, when=date, post_id=post['id'], payload=post['error']).write() CachedPost.undelete(pid, post['author'], post['permlink'], post['category']) cls._insert_feed_cache(post)
def validate(self, raw_op): """Pre-processing and validation of custom_json payload.""" log.info("validating @%s op %s", self.actor, raw_op) try: # validate basic structure self._validate_raw_op(raw_op) self.action = raw_op[0] self.op = raw_op[1] self.actor_id = Accounts.get_id(self.actor) # validate and read schema self._read_schema() # validate permissions self._validate_permissions() # init native ad context and validate op if self.action in NATIVE_AD_ACTIONS: if self.block_num < NA_START_BLOCK: return False self.native_ad = NativeAdOp(self.community_id, self.post_id, self.account_id, { 'action': self.action, 'params': self.na_params }, self.block_num) self.native_ad.validate_op() self.valid = True except AssertionError as e: payload = str(e) Notify('error', dst_id=self.actor_id, when=self.date, payload=payload).write() return self.valid
def _sql(cls, pid, post, level=None): """Given a post and "update level", generate SQL edit statement. Valid levels are: - `insert`: post does not yet exist in cache - `payout`: post was paidout - `update`: post was modified - `upvote`: post payout/votes changed - `recount`: post child count changed """ #pylint: disable=bad-whitespace assert post['author'], "post {} is blank".format(pid) # last-minute sanity check to ensure `pid` is correct #78 pid2 = cls._get_id(post['author'] + '/' + post['permlink']) assert pid == pid2, "hpc id %d maps to %d" % (pid, pid2) # inserts always sequential. if pid > last_id, this operation # *must* be an insert; so `level` must not be any form of update. if pid > cls.last_id() and level != 'insert': raise Exception("WARNING: new pid, but level=%s. #%d vs %d, %s" % (level, pid, cls.last_id(), repr(post))) # start building the queries acc_id = Accounts.get_id(post['author']) values = [('post_id', pid)] # immutable; write only once (*edge case: undeleted posts) if level == 'insert': values.extend([('author', post['author']), ('permlink', post['permlink']), ('category', post['category']), ('depth', post['depth'])]) # always write, unless simple vote update if level in ['insert', 'payout', 'update']: basic = post_basic(post) values.extend([ ('community_id', post['community_id']), # immutable* ('created_at', post['created']), # immutable* ('updated_at', post['last_update']), ('title', post['title']), ('payout_at', basic['payout_at']), # immutable* ('preview', basic['preview']), ('body', basic['body']), ('img_url', basic['image']), ('is_nsfw', basic['is_nsfw']), ('is_declined', basic['is_payout_declined']), ('is_full_power', basic['is_full_power']), ('is_paidout', basic['is_paidout']), ('json', json.dumps(basic['json_metadata'])), ('raw_json', json.dumps(post_legacy(post))), ]) # if there's a pending promoted value to write, pull it out if pid in cls._pending_promoted: bal = cls._pending_promoted.pop(pid) values.append(('promoted', bal)) # update unconditionally payout = post_payout(post) stats = post_stats(post) # //-- # if community - override fields. # TODO: make conditional (date-based?) assert 'community_id' in post, 'comm_id not loaded' if post['community_id']: stats['hide'] = post['hide'] stats['gray'] = post['gray'] # //-- values.extend([ ('payout', payout['payout']), ('rshares', payout['rshares']), ('votes', payout['csvotes']), ('sc_trend', payout['sc_trend']), ('sc_hot', payout['sc_hot']), ('flag_weight', stats['flag_weight']), ('total_votes', stats['total_votes']), ('up_votes', stats['up_votes']), ('is_hidden', stats['hide']), ('is_grayed', stats['gray']), ('author_rep', stats['author_rep']), ('children', min(post['children'], 32767)), ]) # update tags if action is insert/update and is root post tag_sqls = [] if level in ['insert', 'update'] and not post['depth']: diff = level != 'insert' # do not attempt tag diff on insert tag_sqls.extend(cls._tag_sqls(pid, basic['tags'], diff=diff)) # if recounting, update the parent next pass. if level == 'recount' and post['depth']: cls.recount(post['parent_author'], post['parent_permlink']) # trigger any notifications cls._notifs(post, pid, level, payout['payout']) # build the post insert/update SQL, add tag SQLs if level == 'insert': sql = cls._insert(values) # process new native ad, if valid ad_sql = NativeAd.process_ad(values, acc_id) else: sql = cls._update(values) # update ad content, if draft(0) in all communities ad_sql = NativeAd.process_ad(values, acc_id, new=False) # return ad SQL only if it is present if ad_sql is not None: _final = [sql] + tag_sqls + [ad_sql] else: _final = [sql] + tag_sqls return _final
def _read_account(self): _name = read_key_str(self.op, 'account', 16) assert _name, 'must name an account' assert Accounts.exists(_name), 'account `%s` not found' % _name self.account = _name self.account_id = Accounts.get_id(_name)
def _read_account(self): _name = read_key_str(self.op, 'account', 16) assert _name, 'must name an account' self.account_id = Accounts.get_id(_name) self.account = _name
def process_json_community_op(account, op_json, date): """Validates community op and apply state changes to db.""" #pylint: disable=line-too-long cmd_name, cmd_op = op_json # ['flagPost', {community: '', author: '', ...}] commands = list(flatten(PERMISSIONS.values())) if cmd_name not in commands: return print("community op from {} @ {} -- {}".format(account, date, op_json)) community = cmd_op['community'] community_exists = is_community(community) # special case: community creation. TODO: does this require ACTIVE auth? or POSTING will suffice? if cmd_name == 'create' and not community_exists: if account != community: # only the OWNER may create return ctype = cmd_op['type'] # restricted, open-comment, public # INSERT INTO hive_communities (account, name, about, description, lang, is_nsfw, is_private, created_at) # VALUES ('%s', '%s', '%s', '%s', '%s', %d, %d, '%s')" % [account, name, about, description, lang, is_nsfw ? 1 : 0, is_private ? 1 : 0, block_date] # INSERT ADMINS--- # validate permissions if not community_exists or not is_permitted(account, community, cmd_name): return # If command references a post, ensure it's valid post_id, depth = Posts.get_id_and_depth(cmd_op.get('author'), cmd_op.get('permlink')) if not post_id: return # If command references an account, ensure it's valid account_id = Accounts.get_id(cmd_op.get('account')) # If command references a list of accounts, ensure they are valid account_ids = list(map(Accounts.get_id, cmd_op.get('accounts'))) # ADMIN Actions # ------------- if cmd_name == 'add_admins': assert account_ids # UPDATE hive_members SET is_admin = 1 WHERE account IN (%s) AND community = '%s' if cmd_name == 'remove_admins': assert account_ids # todo: validate at least one admin remains!!! # UPDATE hive_members SET is_admin = 0 WHERE account IN (%s) AND community = '%s' if cmd_name == 'add_mods': assert account_ids # UPDATE hive_members SET is_mod = 1 WHERE account IN (%s) AND community = '%s' if cmd_name == 'remove_mods': assert account_ids # UPDATE hive_members SET is_mod = 0 WHERE account IN (%s) AND community = '%s' # MOD USER Actions # ---------------- if cmd_name == 'update_settings': # name, about, description, lang, is_nsfw # settings {bg_color, bg_color2, text_color} # UPDATE hive_communities SET .... WHERE community = '%s' assert account_id if cmd_name == 'add_posters': assert account_ids # UPDATE hive_members SET is_approved = 1 WHERE account IN (%s) AND community = '%s' if cmd_name == 'remove_posters': assert account_ids # UPDATE hive_members SET is_approved = 0 WHERE account IN (%s) AND community = '%s' if cmd_name == 'mute_user': assert account_id # UPDATE hive_members SET is_muted = 1 WHERE account = '%s' AND community = '%s' if cmd_name == 'unmute_user': assert account_id # UPDATE hive_members SET is_muted = 0 WHERE account = '%s' AND community = '%s' if cmd_name == 'set_user_title': assert account_id # UPDATE hive_members SET title = '%s' WHERE account = '%s' AND community = '%s' # MOD POST Actions # ---------------- if cmd_name == 'mute_post': assert post_id # assert all([account_id, post_id]) # UPDATE hive_posts SET is_muted = 1 WHERE community = '%s' AND author = '%s' AND permlink = '%s' if cmd_name == 'unmute_post': assert post_id # UPDATE hive_posts SET is_muted = 0 WHERE community = '%s' AND author = '%s' AND permlink = '%s' if cmd_name == 'pin_post': assert post_id # UPDATE hive_posts SET is_pinned = 1 WHERE community = '%s' AND author = '%s' AND permlink = '%s' if cmd_name == 'unpin_post': assert post_id # UPDATE hive_posts SET is_pinned = 0 WHERE community = '%s' AND author = '%s' AND permlink = '%s' # GUEST POST Actions # ------------------ if cmd_name == 'flag_post': assert post_id # INSERT INTO hive_flags (account, community, author, permlink, comment, created_at) VALUES () # track success (TODO: failures as well?) # INSERT INTO hive_modlog (account, community, action, created_at) VALUES (account, community, json.inspect, block_date) return True
def _insert_feed_cache(cls, post): if not post['depth']: account_id = Accounts.get_id(post['author']) FeedCache.insert(post['id'], account_id, post['date'])
def _insert_feed_cache(cls, post): """Insert the new post into feed cache if it's not a comment.""" if not post['depth']: account_id = Accounts.get_id(post['author']) FeedCache.insert(post['id'], account_id, post['date'])
def register(cls, ops, block_date): from hive.indexer.community import is_community_post_valid for op in ops: sql = ("SELECT id, is_deleted FROM hive_posts " "WHERE author = :a AND permlink = :p") ret = query_row(sql, a=op['author'], p=op['permlink']) pid = None if not ret: # post does not exist, go ahead and process it pass elif not ret[1]: # post exists and is not deleted, thus it's an edit. ignore. continue else: # post exists but was deleted. time to reinstate. pid = ret[0] # set parent & inherited attributes if op['parent_author'] == '': parent_id = None depth = 0 category = op['parent_permlink'] community = cls._get_op_community(op) or op['author'] else: parent_data = query_row( "SELECT id, depth, category, community FROM hive_posts WHERE author = :a " "AND permlink = :p", a=op['parent_author'], p=op['parent_permlink']) parent_id, parent_depth, category, community = parent_data depth = parent_depth + 1 # community must be an existing account if not Accounts.exists(community): community = op['author'] # validated community; will return None if invalid & defaults to author. is_valid = is_community_post_valid(community, op) if not is_valid: print("Invalid post @{}/{} in @{}".format( op['author'], op['permlink'], community)) # if we're reusing a previously-deleted post (rare!), update it if pid: query( "UPDATE hive_posts SET is_valid = :is_valid, is_deleted = '0', parent_id = :parent_id, category = :category, community = :community, depth = :depth WHERE id = :id", is_valid=is_valid, parent_id=parent_id, category=category, community=community, depth=depth, id=pid) else: sql = """ INSERT INTO hive_posts (is_valid, parent_id, author, permlink, category, community, depth, created_at) VALUES (:is_valid, :parent_id, :author, :permlink, :category, :community, :depth, :date) """ query(sql, is_valid=is_valid, parent_id=parent_id, author=op['author'], permlink=op['permlink'], category=category, community=community, depth=depth, date=block_date) pid = query_one( "SELECT id FROM hive_posts WHERE author = :a AND " "permlink = :p", a=op['author'], p=op['permlink']) # add top-level posts to feed cache if not op['parent_permlink']: sql = "INSERT INTO hive_feed_cache (account_id, post_id, created_at) VALUES (:account_id, :id, :created_at)" query(sql, account_id=Accounts.get_id(op['author']), id=pid, created_at=block_date)