def reblog(cls, account, op_json, block_date):
"""Handle legacy 'reblog' op"""
blogger = op_json['account']
author = op_json['author']
permlink = op_json['permlink']
if blogger != account:
return # impersonation
if not all(map(Accounts.exists, [author, blogger])):
return
post_id, depth = Posts.get_id_and_depth(author, permlink)
if depth > 0:
return # prevent comment reblogs
if not post_id:
log.debug("reblog: post not found: %s/%s", author, permlink)
return
if 'delete' in op_json and op_json['delete'] == 'delete':
DB.query("DELETE FROM hive_reblogs WHERE account = :a AND "
"post_id = :pid LIMIT 1", a=blogger, pid=post_id)
if not DbState.is_initial_sync():
FeedCache.delete(post_id, Accounts.get_id(blogger))
else:
sql = ("INSERT INTO hive_reblogs (account, post_id, created_at) "
"VALUES (:a, :pid, :date) ON CONFLICT (account, post_id) DO NOTHING")
DB.query(sql, a=blogger, pid=post_id, date=block_date)
if not DbState.is_initial_sync():
FeedCache.insert(post_id, Accounts.get_id(blogger), block_date)
def _validated(cls, op, tx_idx, num, date):
if op['to'] != 'null':
return # only care about payments to null
amount, token = parse_amount(op['amount'])
if token != 'SBD':
return # only care about SBD payments
url = op['memo']
if not cls._validate_url(url):
print("invalid url: {}".format(url))
return # invalid url
author, permlink = cls._split_url(url)
if not Accounts.exists(author):
return
post_id = Posts.get_id(author, permlink)
if not post_id:
print("post does not exist: %s" % url)
return
return {
'id': None,
'block_num': num,
'tx_idx': tx_idx,
'post_id': post_id,
'from_account': Accounts.get_id(op['from']),
'to_account': Accounts.get_id(op['to']),
'amount': amount,
'token': token
}
def _validated(cls, op, tx_idx, num, date):
"""Validate and normalize the transfer op."""
# pylint: disable=unused-argument
if op['to'] != 'null':
return # only care about payments to null
amount, token = parse_amount(op['amount'])
if token != 'HBD':
return # only care about HBD payments
url = op['memo']
if not cls._validate_url(url):
log.debug("invalid url: %s", url)
return # invalid url
author, permlink = cls._split_url(url)
author_id = Accounts.get_id_noexept(author)
if not author_id:
return
return [{
'id': None,
'block_num': num,
'tx_idx': tx_idx,
'from_account': Accounts.get_id(op['from']),
'to_account': Accounts.get_id(op['to']),
'amount': amount,
'token': token
}, author_id, permlink]
def _notifs(cls, post, pid, level, payout):
# pylint: disable=too-many-locals,too-many-branches
author = post['author']
author_id = Accounts.get_id(author)
parent_author = post['parent_author']
date = post['last_update']
# reply notif
if level == 'insert' and parent_author and parent_author != author:
irredeemable = parent_author in Mutes.all()
parent_author_id = Accounts.get_id(parent_author)
if not irredeemable and not cls._muted(parent_author_id, author_id):
ntype = 'reply' if post['depth'] == 1 else 'reply_comment'
Notify(ntype, src_id=author_id, dst_id=parent_author_id,
score=Accounts.default_score(author), post_id=pid,
when=date).write()
# mentions notif
if level in ('insert', 'update'):
accounts = set(filter(Accounts.exists, mentions(post['body'])))
accounts -= {author, parent_author}
score = Accounts.default_score(author)
if score < 30: max_mentions = 5
elif score < 60: max_mentions = 10
else: max_mentions = 25
if len(accounts) <= max_mentions:
penalty = min([score, 2 * (len(accounts) - 1)])
for mention in accounts:
mention_id = Accounts.get_id(mention)
if (not cls._mentioned(pid, mention_id)
and not cls._muted(mention_id, author_id)):
Notify('mention', src_id=author_id,
dst_id=mention_id, post_id=pid, when=date,
score=(score - penalty)).write()
else:
url = '@%s/%s' % (author, post['permlink'])
log.info("skip %d mentions in %s", len(accounts), url)
# votes notif
url = post['author'] + '/' + post['permlink']
if url in cls._votes:
voters = cls._votes[url]
del cls._votes[url]
net = float(post['net_rshares'])
ratio = float(payout) / net if net else 0
for vote in post['active_votes']:
rshares = int(vote['rshares'])
if vote['voter'] not in voters or rshares < 10e9: continue
contrib = int(1000 * ratio * rshares)
if contrib < 20: continue # < $0.020
voter_id = Accounts.get_id(vote['voter'])
if not cls._voted(pid, author_id, voter_id):
score = min(100, (len(str(contrib)) - 1) * 25) # $1 = 75
payload = "$%.3f" % (contrib / 1000)
Notify('vote', src_id=voter_id, dst_id=author_id,
when=vote['time'], post_id=pid, score=score,
payload=payload).write()
def check_ad_payment(cls, op, date, num):
"""Triggers an adFund operation for validated Native Ads transfers."""
memo = op['memo']
try:
payment = cls._valid_payment(memo)
if payment:
amount, token = parse_amount(op['amount'],
bypass_nai_lookup=True)
params = {
'amount': amount,
'token': token,
'to_account': op['to'],
'community_name': payment['community_name']
}
from hive.indexer.accounts import Accounts
from hive.indexer.posts import Posts
_post_id = Posts.get_id(op['from'], payment['permlink'])
assert _post_id, 'post not found: @%s/%s' % (
op['from'], payment['permlink'])
_account_id = Accounts.get_id(op['from'])
_community_id = payment['community_id']
ad_op = NativeAdOp(_community_id, _post_id, _account_id, {
'action': 'adFund',
'params': params
}, num)
ad_op.validate_op()
ad_op.process()
except AssertionError as e:
payload = str(e)
Notify('error', dst_id=_account_id, when=date,
payload=payload).write()
def insert(cls, op, date):
"""Inserts new post records."""
sql = """INSERT INTO hive_posts (is_valid, is_muted, parent_id, author,
permlink, category, community_id, depth, created_at)
VALUES (:is_valid, :is_muted, :parent_id, :author,
:permlink, :category, :community_id, :depth, :date)"""
sql += ";SELECT currval(pg_get_serial_sequence('hive_posts','id'))"
post = cls._build_post(op, date)
result = DB.query(sql, **post)
post['id'] = int(list(result)[0][0])
cls._set_id(op['author'] + '/' + op['permlink'], post['id'])
if not DbState.is_initial_sync():
if post['error']:
author_id = Accounts.get_id(post['author'])
Notify('error',
dst_id=author_id,
when=date,
post_id=post['id'],
payload=post['error']).write()
CachedPost.insert(op['author'], op['permlink'], post['id'])
if op['parent_author']: # update parent's child count
CachedPost.recount(op['parent_author'], op['parent_permlink'],
post['parent_id'])
cls._insert_feed_cache(post)
def is_post_valid(cls, community_id, comment_op: dict):
""" Given a new post/comment, check if valid as per community rules
For a comment to be valid, these conditions apply:
- Author is not muted in this community
- For council post/comment, author must be a member
- For journal post, author must be a member
- Community must exist
"""
assert community_id, 'no community_id'
community = cls._get_name(community_id)
account_id = Accounts.get_id(comment_op['author'])
role = cls.get_user_role(community_id, account_id)
type_id = int(community[5])
# TODO: check `nsfw` tag requirement #267
# TODO: (1.5) check that beneficiaries are valid
if type_id == TYPE_JOURNAL:
if not comment_op['parent_author']:
return role >= Role.member
elif type_id == TYPE_COUNCIL:
return role >= Role.member
return role >= Role.guest # or at least not muted
def validate(self, raw_op):
"""Pre-processing and validation of custom_json payload."""
log.info("validating @%s op %s", self.actor, raw_op)
try:
# validate basic structure
self._validate_raw_op(raw_op)
self.action = raw_op[0]
self.op = raw_op[1]
self.actor_id = Accounts.get_id(self.actor)
# validate and read schema
self._read_schema()
# validate permissions
self._validate_permissions()
self.valid = True
except AssertionError as e:
payload = str(e)
Notify('error',
dst_id=self.actor_id,
when=self.date,
payload=payload).write()
return self.valid
def register(cls, names, block_date):
"""Block processing: hooks into new account registration.
`Accounts` calls this method with any newly registered names.
This method checks for any valid community names and inserts them.
"""
for name in names:
#if not re.match(r'^hive-[123]\d{4,6}$', name):
if not re.match(r'^hive-[1]\d{4,6}$', name):
continue
type_id = int(name[5])
_id = Accounts.get_id(name)
# insert community
sql = """INSERT INTO hive_communities (id, name, type_id, created_at)
VALUES (:id, :name, :type_id, :date)"""
DB.query(sql, id=_id, name=name, type_id=type_id, date=block_date)
# insert owner
sql = """INSERT INTO hive_roles (community_id, account_id, role_id, created_at)
VALUES (:community_id, :account_id, :role_id, :date)"""
DB.query(sql,
community_id=_id,
account_id=_id,
role_id=Role.owner.value,
date=block_date)
Notify('new_community',
src_id=None,
dst_id=_id,
when=block_date,
community_id=_id).write()
def _validated_op(cls, account, op, date):
"""Validate and normalize the operation."""
if (not 'what' in op or not isinstance(op['what'], list)
or not 'follower' in op or not 'following' in op):
return
what = first(op['what']) or ''
defs = {'': 0, 'blog': 1, 'ignore': 2}
if what not in defs:
return
if (op['follower'] == op['following'] # can't follow self
or op['follower'] != account # impersonation
or not Accounts.exists(op['following']) # invalid account
or not Accounts.exists(op['follower'])): # invalid account
return
return dict(flr=Accounts.get_id(op['follower']),
flg=Accounts.get_id(op['following']),
state=defs[what],
at=date)
def undelete(cls, op, date, pid):
"""Re-allocates an existing record flagged as deleted."""
sql = """UPDATE hive_posts SET is_valid = :is_valid,
is_muted = :is_muted, is_deleted = '0', is_pinned = '0',
parent_id = :parent_id, category = :category,
community_id = :community_id, depth = :depth
WHERE id = :id"""
post = cls._build_post(op, date, pid)
DB.query(sql, **post)
if not DbState.is_initial_sync():
if post['error']:
author_id = Accounts.get_id(post['author'])
Notify('error', dst_id=author_id, when=date,
post_id=post['id'], payload=post['error']).write()
CachedPost.undelete(pid, post['author'], post['permlink'],
post['category'])
cls._insert_feed_cache(post)
def validate(self, raw_op):
"""Pre-processing and validation of custom_json payload."""
log.info("validating @%s op %s", self.actor, raw_op)
try:
# validate basic structure
self._validate_raw_op(raw_op)
self.action = raw_op[0]
self.op = raw_op[1]
self.actor_id = Accounts.get_id(self.actor)
# validate and read schema
self._read_schema()
# validate permissions
self._validate_permissions()
# init native ad context and validate op
if self.action in NATIVE_AD_ACTIONS:
if self.block_num < NA_START_BLOCK: return False
self.native_ad = NativeAdOp(self.community_id, self.post_id,
self.account_id, {
'action': self.action,
'params': self.na_params
}, self.block_num)
self.native_ad.validate_op()
self.valid = True
except AssertionError as e:
payload = str(e)
Notify('error',
dst_id=self.actor_id,
when=self.date,
payload=payload).write()
return self.valid
def _sql(cls, pid, post, level=None):
"""Given a post and "update level", generate SQL edit statement.
Valid levels are:
- `insert`: post does not yet exist in cache
- `payout`: post was paidout
- `update`: post was modified
- `upvote`: post payout/votes changed
- `recount`: post child count changed
"""
#pylint: disable=bad-whitespace
assert post['author'], "post {} is blank".format(pid)
# last-minute sanity check to ensure `pid` is correct #78
pid2 = cls._get_id(post['author'] + '/' + post['permlink'])
assert pid == pid2, "hpc id %d maps to %d" % (pid, pid2)
# inserts always sequential. if pid > last_id, this operation
# *must* be an insert; so `level` must not be any form of update.
if pid > cls.last_id() and level != 'insert':
raise Exception("WARNING: new pid, but level=%s. #%d vs %d, %s" %
(level, pid, cls.last_id(), repr(post)))
# start building the queries
acc_id = Accounts.get_id(post['author'])
values = [('post_id', pid)]
# immutable; write only once (*edge case: undeleted posts)
if level == 'insert':
values.extend([('author', post['author']),
('permlink', post['permlink']),
('category', post['category']),
('depth', post['depth'])])
# always write, unless simple vote update
if level in ['insert', 'payout', 'update']:
basic = post_basic(post)
values.extend([
('community_id', post['community_id']), # immutable*
('created_at', post['created']), # immutable*
('updated_at', post['last_update']),
('title', post['title']),
('payout_at', basic['payout_at']), # immutable*
('preview', basic['preview']),
('body', basic['body']),
('img_url', basic['image']),
('is_nsfw', basic['is_nsfw']),
('is_declined', basic['is_payout_declined']),
('is_full_power', basic['is_full_power']),
('is_paidout', basic['is_paidout']),
('json', json.dumps(basic['json_metadata'])),
('raw_json', json.dumps(post_legacy(post))),
])
# if there's a pending promoted value to write, pull it out
if pid in cls._pending_promoted:
bal = cls._pending_promoted.pop(pid)
values.append(('promoted', bal))
# update unconditionally
payout = post_payout(post)
stats = post_stats(post)
# //--
# if community - override fields.
# TODO: make conditional (date-based?)
assert 'community_id' in post, 'comm_id not loaded'
if post['community_id']:
stats['hide'] = post['hide']
stats['gray'] = post['gray']
# //--
values.extend([
('payout', payout['payout']),
('rshares', payout['rshares']),
('votes', payout['csvotes']),
('sc_trend', payout['sc_trend']),
('sc_hot', payout['sc_hot']),
('flag_weight', stats['flag_weight']),
('total_votes', stats['total_votes']),
('up_votes', stats['up_votes']),
('is_hidden', stats['hide']),
('is_grayed', stats['gray']),
('author_rep', stats['author_rep']),
('children', min(post['children'], 32767)),
])
# update tags if action is insert/update and is root post
tag_sqls = []
if level in ['insert', 'update'] and not post['depth']:
diff = level != 'insert' # do not attempt tag diff on insert
tag_sqls.extend(cls._tag_sqls(pid, basic['tags'], diff=diff))
# if recounting, update the parent next pass.
if level == 'recount' and post['depth']:
cls.recount(post['parent_author'], post['parent_permlink'])
# trigger any notifications
cls._notifs(post, pid, level, payout['payout'])
# build the post insert/update SQL, add tag SQLs
if level == 'insert':
sql = cls._insert(values)
# process new native ad, if valid
ad_sql = NativeAd.process_ad(values, acc_id)
else:
sql = cls._update(values)
# update ad content, if draft(0) in all communities
ad_sql = NativeAd.process_ad(values, acc_id, new=False)
# return ad SQL only if it is present
if ad_sql is not None:
_final = [sql] + tag_sqls + [ad_sql]
else:
_final = [sql] + tag_sqls
return _final
def _read_account(self):
_name = read_key_str(self.op, 'account', 16)
assert _name, 'must name an account'
assert Accounts.exists(_name), 'account `%s` not found' % _name
self.account = _name
self.account_id = Accounts.get_id(_name)
def _read_account(self):
_name = read_key_str(self.op, 'account', 16)
assert _name, 'must name an account'
self.account_id = Accounts.get_id(_name)
self.account = _name
def process_json_community_op(account, op_json, date):
"""Validates community op and apply state changes to db."""
#pylint: disable=line-too-long
cmd_name, cmd_op = op_json # ['flagPost', {community: '', author: '', ...}]
commands = list(flatten(PERMISSIONS.values()))
if cmd_name not in commands:
return
print("community op from {} @ {} -- {}".format(account, date, op_json))
community = cmd_op['community']
community_exists = is_community(community)
# special case: community creation. TODO: does this require ACTIVE auth? or POSTING will suffice?
if cmd_name == 'create' and not community_exists:
if account != community: # only the OWNER may create
return
ctype = cmd_op['type'] # restricted, open-comment, public
# INSERT INTO hive_communities (account, name, about, description, lang, is_nsfw, is_private, created_at)
# VALUES ('%s', '%s', '%s', '%s', '%s', %d, %d, '%s')" % [account, name, about, description, lang, is_nsfw ? 1 : 0, is_private ? 1 : 0, block_date]
# INSERT ADMINS---
# validate permissions
if not community_exists or not is_permitted(account, community, cmd_name):
return
# If command references a post, ensure it's valid
post_id, depth = Posts.get_id_and_depth(cmd_op.get('author'),
cmd_op.get('permlink'))
if not post_id:
return
# If command references an account, ensure it's valid
account_id = Accounts.get_id(cmd_op.get('account'))
# If command references a list of accounts, ensure they are valid
account_ids = list(map(Accounts.get_id, cmd_op.get('accounts')))
# ADMIN Actions
# -------------
if cmd_name == 'add_admins':
assert account_ids
# UPDATE hive_members SET is_admin = 1 WHERE account IN (%s) AND community = '%s'
if cmd_name == 'remove_admins':
assert account_ids
# todo: validate at least one admin remains!!!
# UPDATE hive_members SET is_admin = 0 WHERE account IN (%s) AND community = '%s'
if cmd_name == 'add_mods':
assert account_ids
# UPDATE hive_members SET is_mod = 1 WHERE account IN (%s) AND community = '%s'
if cmd_name == 'remove_mods':
assert account_ids
# UPDATE hive_members SET is_mod = 0 WHERE account IN (%s) AND community = '%s'
# MOD USER Actions
# ----------------
if cmd_name == 'update_settings':
# name, about, description, lang, is_nsfw
# settings {bg_color, bg_color2, text_color}
# UPDATE hive_communities SET .... WHERE community = '%s'
assert account_id
if cmd_name == 'add_posters':
assert account_ids
# UPDATE hive_members SET is_approved = 1 WHERE account IN (%s) AND community = '%s'
if cmd_name == 'remove_posters':
assert account_ids
# UPDATE hive_members SET is_approved = 0 WHERE account IN (%s) AND community = '%s'
if cmd_name == 'mute_user':
assert account_id
# UPDATE hive_members SET is_muted = 1 WHERE account = '%s' AND community = '%s'
if cmd_name == 'unmute_user':
assert account_id
# UPDATE hive_members SET is_muted = 0 WHERE account = '%s' AND community = '%s'
if cmd_name == 'set_user_title':
assert account_id
# UPDATE hive_members SET title = '%s' WHERE account = '%s' AND community = '%s'
# MOD POST Actions
# ----------------
if cmd_name == 'mute_post':
assert post_id
# assert all([account_id, post_id])
# UPDATE hive_posts SET is_muted = 1 WHERE community = '%s' AND author = '%s' AND permlink = '%s'
if cmd_name == 'unmute_post':
assert post_id
# UPDATE hive_posts SET is_muted = 0 WHERE community = '%s' AND author = '%s' AND permlink = '%s'
if cmd_name == 'pin_post':
assert post_id
# UPDATE hive_posts SET is_pinned = 1 WHERE community = '%s' AND author = '%s' AND permlink = '%s'
if cmd_name == 'unpin_post':
assert post_id
# UPDATE hive_posts SET is_pinned = 0 WHERE community = '%s' AND author = '%s' AND permlink = '%s'
# GUEST POST Actions
# ------------------
if cmd_name == 'flag_post':
assert post_id
# INSERT INTO hive_flags (account, community, author, permlink, comment, created_at) VALUES ()
# track success (TODO: failures as well?)
# INSERT INTO hive_modlog (account, community, action, created_at) VALUES (account, community, json.inspect, block_date)
return True
def _insert_feed_cache(cls, post):
if not post['depth']:
account_id = Accounts.get_id(post['author'])
FeedCache.insert(post['id'], account_id, post['date'])
def _insert_feed_cache(cls, post):
"""Insert the new post into feed cache if it's not a comment."""
if not post['depth']:
account_id = Accounts.get_id(post['author'])
FeedCache.insert(post['id'], account_id, post['date'])
def register(cls, ops, block_date):
from hive.indexer.community import is_community_post_valid
for op in ops:
sql = ("SELECT id, is_deleted FROM hive_posts "
"WHERE author = :a AND permlink = :p")
ret = query_row(sql, a=op['author'], p=op['permlink'])
pid = None
if not ret:
# post does not exist, go ahead and process it
pass
elif not ret[1]:
# post exists and is not deleted, thus it's an edit. ignore.
continue
else:
# post exists but was deleted. time to reinstate.
pid = ret[0]
# set parent & inherited attributes
if op['parent_author'] == '':
parent_id = None
depth = 0
category = op['parent_permlink']
community = cls._get_op_community(op) or op['author']
else:
parent_data = query_row(
"SELECT id, depth, category, community FROM hive_posts WHERE author = :a "
"AND permlink = :p",
a=op['parent_author'],
p=op['parent_permlink'])
parent_id, parent_depth, category, community = parent_data
depth = parent_depth + 1
# community must be an existing account
if not Accounts.exists(community):
community = op['author']
# validated community; will return None if invalid & defaults to author.
is_valid = is_community_post_valid(community, op)
if not is_valid:
print("Invalid post @{}/{} in @{}".format(
op['author'], op['permlink'], community))
# if we're reusing a previously-deleted post (rare!), update it
if pid:
query(
"UPDATE hive_posts SET is_valid = :is_valid, is_deleted = '0', parent_id = :parent_id, category = :category, community = :community, depth = :depth WHERE id = :id",
is_valid=is_valid,
parent_id=parent_id,
category=category,
community=community,
depth=depth,
id=pid)
else:
sql = """
INSERT INTO hive_posts (is_valid, parent_id, author, permlink,
category, community, depth, created_at)
VALUES (:is_valid, :parent_id, :author, :permlink,
:category, :community, :depth, :date)
"""
query(sql,
is_valid=is_valid,
parent_id=parent_id,
author=op['author'],
permlink=op['permlink'],
category=category,
community=community,
depth=depth,
date=block_date)
pid = query_one(
"SELECT id FROM hive_posts WHERE author = :a AND "
"permlink = :p",
a=op['author'],
p=op['permlink'])
# add top-level posts to feed cache
if not op['parent_permlink']:
sql = "INSERT INTO hive_feed_cache (account_id, post_id, created_at) VALUES (:account_id, :id, :created_at)"
query(sql,
account_id=Accounts.get_id(op['author']),
id=pid,
created_at=block_date)