def _validated_op(cls, account, op, date):
"""Validate and normalize the operation."""
if (not 'what' in op
or not isinstance(op['what'], list)
or not 'follower' in op
or not 'following' in op):
log.info("follow_op %s ignored due to basic errors", op)
return None
what = first(op['what']) or ''
# ABW: the empty 'what' is used to clear existing 'blog' or 'ignore' state, however it can also be used to
# introduce new empty relation record in hive_follows adding unnecessary data (it might become a problem
# only if we wanted to immediately remove empty records)
# we could add aliases for '' - 'unfollow' and 'unignore'/'unmute'
# we could add alias for 'ignore' - 'mute'
defs = {'': Action.Nothing, 'blog': Action.Blog, 'follow': Action.Blog, 'ignore': Action.Ignore,
'blacklist': Action.Blacklist, 'follow_blacklist': Action.Follow_blacklist,
'unblacklist': Action.Unblacklist, 'unfollow_blacklist': Action.Unfollow_blacklist,
'follow_muted': Action.Follow_muted, 'unfollow_muted': Action.Unfollow_muted,
'reset_blacklist' : Action.Reset_blacklist, 'reset_following_list': Action.Reset_following_list,
'reset_muted_list': Action.Reset_muted_list, 'reset_follow_blacklist': Action.Reset_follow_blacklist,
'reset_follow_muted_list': Action.Reset_follow_muted_list, 'reset_all_lists': Action.Reset_all_lists}
if not isinstance(what, str) or what not in defs:
log.info("follow_op %s ignored due to unknown type of follow", op)
return None
# follower is empty or follower account does not exist, or it wasn't that account that authorized operation
if not op['follower'] or not Accounts.exists(op['follower']) or op['follower'] != account:
log.info("follow_op %s ignored due to invalid follower", op)
return None
# normalize following to list
op['following'] = op['following'] if isinstance(op['following'], list) else [op['following']]
# if following name does not exist do not process it: basically equal to drop op for single following entry
op['following'] = [following for following in op['following'] if following and Accounts.exists(following) and following != op['follower']]
# ABW: note that since you could make 'following' list empty anyway by supplying nonexisting account
# there was no point in excluding follow_op with provided empty list/empty string - such call actually
# makes sense for state > 8 when 'following' is ignored
state = defs[what]
if not op['following'] and state < Action.Reset_blacklist:
log.info("follow_op %s is void due to effectively empty list of following", op)
return None
return dict(follower=escape_characters(op['follower']),
following=[escape_characters(following) for following in op['following']],
state=state,
at=date)
def effective_comment_vote_op(cls, vop):
""" Process effective_comment_vote_operation """
post_key = "{}/{}".format(vop['author'], vop['permlink'])
key = "{}/{}".format(vop['voter'], post_key)
if key in cls._votes_data:
vote_data = cls._votes_data[key]
vote_data["weight"] = vop["weight"]
vote_data["rshares"] = vop["rshares"]
vote_data["is_effective"] = True
vote_data["num_changes"] += 1
vote_data["block_num"] = vop["block_num"]
else:
if not post_key in cls._votes_per_post:
cls._votes_per_post[post_key] = []
cls._votes_per_post[post_key].append(vop['voter'])
cls._votes_data[key] = dict(voter=vop["voter"],
author=vop["author"],
permlink=escape_characters(
vop["permlink"]),
vote_percent=0,
weight=vop["weight"],
rshares=vop["rshares"],
last_update="1970-01-01 00:00:00",
is_effective=True,
num_changes=0,
block_num=vop["block_num"])
def vote_op(cls, vote_operation, date):
""" Process vote_operation """
voter = vote_operation['voter']
author = vote_operation['author']
permlink = vote_operation['permlink']
weight = vote_operation['weight']
block_num = vote_operation['block_num']
if cls.inside_flush:
log.exception("Adding new vote-info into '_votes_data' dict")
raise RuntimeError("Fatal error")
post_key = "{}/{}".format(author, permlink)
key = "{}/{}".format(voter, post_key)
if key in cls._votes_data:
vote_data = cls._votes_data[key]
vote_data["vote_percent"] = weight
vote_data["last_update"] = date
# only effective vote edits increase num_changes counter
else:
if not post_key in cls._votes_per_post:
cls._votes_per_post[post_key] = []
cls._votes_per_post[post_key].append(voter)
cls._votes_data[key] = dict(voter=voter,
author=author,
permlink=escape_characters(permlink),
vote_percent=weight,
weight=0,
rshares=0,
last_update=date,
is_effective=False,
num_changes=0,
block_num=block_num)
def flush(cls):
""" Flush collected data to database """
sql_prefix = """
INSERT INTO hive_reblogs (blogger_id, post_id, created_at, block_num)
SELECT
data_source.blogger_id, data_source.post_id, data_source.created_at, data_source.block_num
FROM
(
SELECT
ha_b.id as blogger_id, hp.id as post_id, t.block_date as created_at, t.block_num
FROM
(VALUES
{}
) AS T(blogger, author, permlink, block_date, block_num)
INNER JOIN hive_accounts ha ON ha.name = t.author
INNER JOIN hive_accounts ha_b ON ha_b.name = t.blogger
INNER JOIN hive_permlink_data hpd ON hpd.permlink = t.permlink
INNER JOIN hive_posts hp ON hp.author_id = ha.id AND hp.permlink_id = hpd.id AND hp.counter_deleted = 0
) AS data_source (blogger_id, post_id, created_at, block_num)
ON CONFLICT ON CONSTRAINT hive_reblogs_ux1 DO NOTHING
"""
item_count = len(cls.reblog_items_to_flush)
if item_count > 0:
values = []
limit = 1000
count = 0
cls.beginTx()
for k, v in cls.reblog_items_to_flush.items():
reblog_item = v['op']
if count < limit:
values.append("({}, {}, {}, '{}'::timestamp, {})".format(escape_characters(reblog_item['account']),
escape_characters(reblog_item['author']),
escape_characters(reblog_item['permlink']),
reblog_item['block_date'],
reblog_item['block_num']))
count = count + 1
else:
values_str = ",".join(values)
query = sql_prefix.format(values_str, values_str)
cls.db.query_prepared(query)
values.clear()
values.append("({}, {}, {}, '{}'::timestamp, {})".format(escape_characters(reblog_item['account']),
escape_characters(reblog_item['author']),
escape_characters(reblog_item['permlink']),
reblog_item['block_date'],
reblog_item['block_num']))
count = 1
if len(values) > 0:
values_str = ",".join(values)
query = sql_prefix.format(values_str, values_str)
cls.db.query_prepared(query)
values.clear()
cls.commitTx()
cls.reblog_items_to_flush.clear()
return item_count
def process_vote(self, block_num, effective_vote_op):
tuple = "('{}', '{}', {}, {}, {})".format(
effective_vote_op['author'], effective_vote_op['voter'],
escape_characters(effective_vote_op['permlink']),
effective_vote_op['rshares'], block_num)
self._values.append(tuple)
def flush(cls, print_query = False):
""" Flush data from cache to db """
if cls._data:
values_insert = []
values_update = []
cls.beginTx()
sql = """
INSERT INTO
hive_post_data (id, title, preview, img_url, body, json)
VALUES
"""
for k, data in cls._data.items():
title = 'NULL' if data['title'] is None else "{}".format(escape_characters(data['title']))
body = 'NULL' if data['body'] is None else "{}".format(escape_characters(data['body']))
preview = 'NULL' if data['body'] is None else "{}".format(escape_characters(data['body'][0:1024]))
json = 'NULL' if data['json'] is None else "{}".format(escape_characters(data['json']))
img_url = 'NULL' if data['img_url'] is None else "{}".format(escape_characters(data['img_url']))
value = "({},{},{},{},{},{})".format(k, title, preview, img_url, body, json)
if data['is_new_post']:
values_insert.append(value)
else:
values_update.append(value)
if len(values_insert) > 0:
sql = """
INSERT INTO
hive_post_data (id, title, preview, img_url, body, json)
VALUES
"""
sql += ','.join(values_insert)
if print_query:
log.info("Executing query:\n{}".format(sql))
cls.db.query_prepared(sql)
values_insert.clear();
if len(values_update) > 0:
sql = """
UPDATE hive_post_data AS hpd SET
title = COALESCE( data_source.title, hpd.title ),
preview = COALESCE( data_source.preview, hpd.preview ),
img_url = COALESCE( data_source.img_url, hpd.img_url ),
body = COALESCE( data_source.body, hpd.body ),
json = COALESCE( data_source.json, hpd.json )
FROM
( SELECT * FROM
( VALUES
"""
sql += ','.join(values_update)
sql += """
) AS T(id, title, preview, img_url, body, json)
) AS data_source
WHERE hpd.id = data_source.id
"""
if print_query:
log.info("Executing query:\n{}".format(sql))
cls.db.query_prepared(sql)
values_update.clear()
cls.commitTx()
n = len(cls._data.keys())
cls._data.clear()
return n
def follow_op(cls, account, op_json, date, block_num):
"""Process an incoming follow op."""
def true_false_none(state, to_true, to_false):
if state == to_true:
return True
if state == to_false:
return False
return None
op = cls._validated_op(account, op_json, date)
if not op:
return
op['block_num'] = block_num
state = int(op['state'])
follower = op['follower']
# log.info("follow_op accepted as %s", op)
if state >= Action.Reset_blacklist:
# choose action specific to requested list resetting
add_null_blacklist = False
add_null_muted = False
if state == Action.Reset_blacklist:
reset_list = Follow._reset_blacklist
cls.list_resets_to_flush.append(dict(follower=follower, reset_call='follow_reset_blacklist', block_num=block_num))
elif state == Action.Reset_following_list:
reset_list = Follow._reset_following_list
cls.list_resets_to_flush.append(dict(follower=follower, reset_call='follow_reset_following_list', block_num=block_num))
elif state == Action.Reset_muted_list:
reset_list = Follow._reset_muted_list
cls.list_resets_to_flush.append(dict(follower=follower, reset_call='follow_reset_muted_list', block_num=block_num))
elif state == Action.Reset_follow_blacklist:
reset_list = Follow._reset_follow_blacklist
cls.list_resets_to_flush.append(dict(follower=follower, reset_call='follow_reset_follow_blacklist', block_num=block_num))
add_null_blacklist = True
elif state == Action.Reset_follow_muted_list:
reset_list = Follow._reset_follow_muted_list
cls.list_resets_to_flush.append(dict(follower=follower, reset_call='follow_reset_follow_muted_list', block_num=block_num))
add_null_muted = True
elif state == Action.Reset_all_lists:
reset_list = Follow._reset_all_lists
cls.list_resets_to_flush.append(dict(follower=follower, reset_call='follow_reset_all_lists', block_num=block_num))
add_null_blacklist = True
add_null_muted = True
else:
assert False, 'Unhandled follow state'
# apply action to existing cached data as well as to database (ABW: with expected frequency of list resetting
# there is no point in grouping such operations from group of blocks - we can just execute them one by one
# in order of appearance)
for k, data in cls.follow_items_to_flush.items():
if data['follower'] == follower:
reset_list(data, op)
if add_null_blacklist or add_null_muted:
# since 'null' account can't have its blacklist/mute list, following such list is only used
# as an indicator for frontend to no longer bother user with proposition of following predefined
# lists (since that user is already choosing his own lists)
cls._follow_single(follower, escape_characters('null'), op['at'], op['block_num'], None, None, add_null_blacklist, add_null_muted)
else:
# set new state/flags to be applied to each pair with changing 'following'
new_state = state if state in (Action.Nothing, Action.Blog, Action.Ignore) else None
new_blacklisted = true_false_none(state, Action.Blacklist, Action.Unblacklist)
new_follow_blacklists = true_false_none(state, Action.Follow_blacklist, Action.Unfollow_blacklist)
new_follow_muted = true_false_none(state, Action.Follow_muted, Action.Unfollow_muted)
for following in op['following']:
cls._follow_single(follower, following, op['at'], block_num,
new_state, new_blacklisted, new_follow_blacklists, new_follow_muted)
async def find_comments(context, comments: list):
""" Search for comments: limit and order is ignored in hive code """
result = []
assert isinstance(comments,
list), "Expected array of author+permlink pairs"
assert len(
comments
) <= 1000, "Parameters count is greather than max allowed (1000)"
db = context['db']
SQL_TEMPLATE = """
SELECT
pv.id,
pv.community_id,
pv.author,
pv.permlink,
pv.title,
pv.body,
pv.category,
pv.depth,
pv.promoted,
pv.payout,
pv.last_payout_at,
pv.cashout_time,
pv.is_paidout,
pv.children,
pv.votes,
pv.created_at,
pv.updated_at,
pv.rshares,
pv.json,
pv.is_hidden,
pv.is_grayed,
pv.total_votes,
pv.net_votes,
pv.total_vote_weight,
pv.parent_permlink_or_category,
pv.curator_payout_value,
pv.root_author,
pv.root_permlink,
pv.max_accepted_payout,
pv.percent_hbd,
pv.allow_replies,
pv.allow_votes,
pv.allow_curation_rewards,
pv.beneficiaries,
pv.url,
pv.root_title,
pv.abs_rshares,
pv.active,
pv.author_rewards,
pv.parent_author
FROM (
SELECT
hp.id
FROM
live_posts_comments_view hp
JOIN hive_accounts_view ha_a ON ha_a.id = hp.author_id
JOIN hive_permlink_data hpd_p ON hpd_p.id = hp.permlink_id
JOIN (VALUES {}) AS t (author, permlink, number) ON ha_a.name = t.author AND hpd_p.permlink = t.permlink
WHERE
NOT hp.is_muted
ORDER BY t.number
) ds,
LATERAL get_post_view_by_id (ds.id) pv
"""
idx = 0
values = ""
for arg in comments:
if not isinstance(arg, list) or len(arg) < 2:
continue
author = arg[0]
permlink = arg[1]
if not isinstance(author, str) or not isinstance(permlink, str):
continue
if idx > 0:
values += ","
values += "({},{},{})".format(escape_characters(author),
escape_characters(permlink), idx)
idx += 1
sql = SQL_TEMPLATE.format(values)
if idx > 0:
rows = await db.query_all(sql)
for row in rows:
cpo = database_post_object(dict(row))
result.append(cpo)
return {"comments": result}
def comment_payout_op(cls):
values_limit = 1000
""" Process comment payment operations """
for k, v in cls.comment_payout_ops.items():
author = None
permlink = None
# author payouts
author_rewards = 0
author_rewards_hive = None
author_rewards_hbd = None
author_rewards_vests = None
# total payout for comment
#comment_author_reward = None
#curators_vesting_payout = None
total_payout_value = None
curator_payout_value = None
#beneficiary_payout_value = None;
payout = None
pending_payout = None
payout_at = None
last_payout_at = None
cashout_time = None
is_paidout = None
total_vote_weight = None
# [final] payout indicator - by default all rewards are zero, but might be overwritten by other operations
# ABW: prior to some early HF that was not necessarily final payout since those were discussion driven so new comment/vote could trigger new cashout window, see f.e.
# soulsistashakti/re-emily-cook-let-me-introduce-myself-my-name-is-emily-cook-and-i-m-the-producer-and-presenter-of-a-monthly-film-show-film-focus-20160701t012330329z
# it emits that "final" operation at blocks: 2889020, 3053237, 3172559 and 4028469
if v[VirtualOperationType.CommentPayoutUpdate] is not None:
value, date = v[VirtualOperationType.CommentPayoutUpdate]
if author is None:
author = value['author']
permlink = value['permlink']
is_paidout = True
payout_at = date
last_payout_at = date
cashout_time = "infinity"
pending_payout = 0
total_vote_weight = 0
# author rewards in current (final or nonfinal) payout (always comes with comment_reward_operation)
if v[VirtualOperationType.AuthorReward] is not None:
value, date = v[VirtualOperationType.AuthorReward]
if author is None:
author = value['author']
permlink = value['permlink']
author_rewards_hive = value['hive_payout']['amount']
author_rewards_hbd = value['hbd_payout']['amount']
author_rewards_vests = value['vesting_payout']['amount']
#curators_vesting_payout = value['curators_vesting_payout']['amount']
# summary of comment rewards in current (final or nonfinal) payout (always comes with author_reward_operation)
if v[VirtualOperationType.CommentReward] is not None:
value, date = v[VirtualOperationType.CommentReward]
if author is None:
author = value['author']
permlink = value['permlink']
#comment_author_reward = value['payout']
author_rewards = value['author_rewards']
total_payout_value = value['total_payout_value']
curator_payout_value = value['curator_payout_value']
#beneficiary_payout_value = value['beneficiary_payout_value']
payout = sum([
sbd_amount(total_payout_value),
sbd_amount(curator_payout_value)
])
pending_payout = 0
last_payout_at = date
# estimated pending_payout from vote (if exists with actual payout the value comes from vote cast after payout)
if v[VirtualOperationType.EffectiveCommentVote] is not None:
value, date = v[VirtualOperationType.EffectiveCommentVote]
if author is None:
author = value['author']
permlink = value['permlink']
pending_payout = sbd_amount(value['pending_payout'])
total_vote_weight = value['total_vote_weight']
cls._comment_payout_ops.append(
"('{}', {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {})"
.format(
author, escape_characters(permlink), "NULL" if
(total_payout_value is None) else
("'{}'".format(legacy_amount(total_payout_value))),
"NULL" if (curator_payout_value is None) else
("'{}'".format(legacy_amount(curator_payout_value))),
author_rewards, "NULL" if
(author_rewards_hive is None) else author_rewards_hive,
"NULL" if
(author_rewards_hbd is None) else author_rewards_hbd,
"NULL" if
(author_rewards_vests is None) else author_rewards_vests,
"NULL" if (payout is None) else payout, "NULL" if
(pending_payout is None) else pending_payout, "NULL" if
(payout_at is None) else
("'{}'::timestamp".format(payout_at)), "NULL" if
(last_payout_at is None) else
("'{}'::timestamp".format(last_payout_at)), "NULL" if
(cashout_time is None) else
("'{}'::timestamp".format(cashout_time)), "NULL" if
(is_paidout is None) else is_paidout, "NULL" if
(total_vote_weight is None) else total_vote_weight))
n = len(cls.comment_payout_ops)
cls.comment_payout_ops.clear()
return n
def get_json_data(cls, source ):
"""json-data preprocessing."""
return escape_characters( source )