def _valid_payment(cls, memo):
"""Checks for valid ad payment memo. Example memo:
`hna:hive-133333/interesting-promo`"""
if memo[:4] == "hna:":
ref = memo[4:].strip(
) # strip() to avoid invalidating legitimate payments
assert ref.count('/') == 1, (
"invalid ad payment memo; found (%d) / characters instead of 1"
% ref.count)
_values = ref.split('/')
comm = _values[0].strip()
link = _values[1].strip()
from hive.indexer.community import Community
valid_comm = Community.validated_name(comm)
assert valid_comm, 'invalid community name entered (%s)' % comm
comm_id = Community.get_id(comm)
assert comm_id, 'community not found: %s' % comm
return {
'community_id': comm_id,
'community_name': comm,
'permlink': link
}
return None
def register(cls, names, block_date):
"""Block processing: register "candidate" names.
There are four ops which can result in account creation:
*account_create*, *account_create_with_delegation*, *pow*,
and *pow2*. *pow* ops result in account creation only when
the account they name does not already exist!
"""
# filter out names which already registered
new_names = list(filter(lambda n: not cls.exists(n), set(names)))
if not new_names:
return
for name in new_names:
DB.query(
"INSERT INTO hive_accounts (name, created_at) "
"VALUES (:name, :date)",
name=name,
date=block_date)
# pull newly-inserted ids and merge into our map
sql = "SELECT name, id FROM hive_accounts WHERE name IN :names"
for name, _id in DB.query_all(sql, names=tuple(new_names)):
cls._ids[name] = _id
# post-insert: pass to communities to check for new registrations
from hive.indexer.community import Community, START_DATE
if block_date > START_DATE:
Community.register(new_names, block_date)
def register(cls, name, op_details, block_date, block_num):
"""Block processing: register "candidate" names.
There are four ops which can result in account creation:
*account_create*, *account_create_with_delegation*, *pow*,
and *pow2*. *pow* ops result in account creation only when
the account they name does not already exist!
"""
if name is None:
return False
# filter out names which already registered
if cls.exists(name):
return True
( _posting_json_metadata, _json_metadata ) = get_profile_str( op_details )
sql = """
INSERT INTO hive_accounts (name, created_at, posting_json_metadata, json_metadata )
VALUES ( '{}', '{}', {}, {} )
RETURNING id
""".format( name, block_date, cls.get_json_data( _posting_json_metadata ), cls.get_json_data( _json_metadata ) )
new_id = DB.query_one( sql )
if new_id is None:
return False
cls._ids[name] = new_id
# post-insert: pass to communities to check for new registrations
from hive.indexer.community import Community
if block_num > Community.start_block:
Community.register(name, block_date, block_num)
return True
def _build_post(cls, op, date, pid=None):
"""Validate and normalize a post operation.
Post is muted if:
- parent was muted
- author unauthorized
Post is invalid if:
- parent is invalid
- author unauthorized
"""
# TODO: non-nsfw post in nsfw community is `invalid`
# if this is a top-level post:
if not op['parent_author']:
parent_id = None
depth = 0
category = op['parent_permlink']
community_id = None
if date > START_DATE:
community_id = Community.validated_id(category)
is_valid = True
is_muted = False
# this is a comment; inherit parent props.
else:
parent_id = cls.get_id(op['parent_author'], op['parent_permlink'])
sql = """SELECT depth, category, community_id, is_valid, is_muted
FROM hive_posts WHERE id = :id"""
(parent_depth, category, community_id, is_valid,
is_muted) = DB.query_row(sql, id=parent_id)
depth = parent_depth + 1
if not is_valid: error = 'replying to invalid post'
elif is_muted: error = 'replying to muted post'
# check post validity in specified context
error = None
if community_id and is_valid and not Community.is_post_valid(
community_id, op):
error = 'not authorized'
#is_valid = False # TODO: reserved for future blacklist status?
is_muted = True
return dict(author=op['author'],
permlink=op['permlink'],
id=pid,
is_valid=is_valid,
is_muted=is_muted,
parent_id=parent_id,
depth=depth,
category=category,
community_id=community_id,
date=date,
error=error)
def run(self):
"""Initialize state; setup/recovery checks; sync and runloop."""
# ensure db schema up to date, check app status
DbState.initialize()
# prefetch id->name and id->rank memory maps
Accounts.load_ids()
Accounts.fetch_ranks()
Community.recalc_pending_payouts()
if DbState.is_initial_sync():
# resume initial sync
self.initial()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head(self._steem)
# perform cleanup if process did not exit cleanly
CachedPost.recover_missing_posts(self._steem)
#audit_cache_missing(self._db, self._steem)
#audit_cache_deleted(self._db)
self._update_chain_state()
if self._conf.get('test_max_block'):
# debug mode: partial sync
return self.from_steemd()
if self._conf.get('test_disable_sync'):
# debug mode: no sync, just stream
return self.listen()
while True:
# sync up to irreversible block
self.from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(self._steem, trx=True)
try:
# listen for new blocks
self.listen()
except MicroForkException as e:
# attempt to recover by restarting stream
log.error("NOTIFYALERT microfork: %s", repr(e))
def listen(self):
"""Live (block following) mode."""
trail_blocks = self._conf.get('trail_blocks')
assert trail_blocks >= 0
assert trail_blocks <= 100
# debug: no max gap if disable_sync in effect
max_gap = None if self._conf.get('test_disable_sync') else 100
steemd = self._steem
hive_head = Blocks.head_num()
for block in steemd.stream_blocks(hive_head + 1, trail_blocks,
max_gap):
start_time = perf()
self._db.query("START TRANSACTION")
num = Blocks.process(block)
follows = Follow.flush(trx=False)
accts = Accounts.flush(steemd, trx=False, spread=8)
CachedPost.dirty_paidouts(block['timestamp'])
cnt = CachedPost.flush(steemd, trx=False)
self._db.query("COMMIT")
ms = (perf() - start_time) * 1000
log.info(
"[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits,"
"% 3d payouts,% 3d votes,% 3d counts,% 3d accts,% 3d follows"
" --% 5dms%s", num, block['timestamp'],
len(block['transactions']), cnt['insert'], cnt['update'],
cnt['payout'], cnt['upvote'], cnt['recount'], accts, follows,
ms, ' SLOW' if ms > 1000 else '')
if num % 1200 == 0: #1hr
log.warning("head block %d @ %s", num, block['timestamp'])
log.info("[LIVE] hourly stats")
Accounts.fetch_ranks()
#Community.recalc_pending_payouts()
if num % 200 == 0: #10min
Community.recalc_pending_payouts()
if num % 100 == 0: #5min
log.info("[LIVE] 5-min stats")
Accounts.dirty_oldest(500)
if num % 20 == 0: #1min
self._update_chain_state()
def _verify_post_against_community(cls, op, community_id, is_valid,
is_muted):
error = None
if community_id and is_valid and not Community.is_post_valid(
community_id, op):
error = 'not authorized'
#is_valid = False # TODO: reserved for future blacklist status?
is_muted = True
return error