def _condenser_profile_object(row):
"""Convert an internal account record into legacy-steemd style."""
blacklists = Mutes.lists(row['name'], row['reputation'])
return {
'id': row['id'],
'name': row['name'],
'created': json_date(row['created_at']),
'active': json_date(row['active_at']),
'post_count': row['post_count'],
'reputation': row['reputation'],
'blacklists': blacklists,
'stats': {
'sp': int(row['vote_weight'] * 0.0005037),
'rank': row['rank'],
'following': row['following'],
'followers': row['followers'],
},
'metadata': {
'profile': {
'name': row['display_name'],
'about': row['about'],
'website': row['website'],
'location': row['location'],
'cover_image': row['cover_image'],
'profile_image': row['profile_image'],
}
}
}
def _notifs(cls, post, pid, level, payout):
# pylint: disable=too-many-locals,too-many-branches
author = post['author']
author_id = Accounts.get_id(author)
parent_author = post['parent_author']
date = post['last_update']
# reply notif
if level == 'insert' and parent_author and parent_author != author:
irredeemable = parent_author in Mutes.all()
parent_author_id = Accounts.get_id(parent_author)
if not irredeemable and not cls._muted(parent_author_id, author_id):
ntype = 'reply' if post['depth'] == 1 else 'reply_comment'
Notify(ntype, src_id=author_id, dst_id=parent_author_id,
score=Accounts.default_score(author), post_id=pid,
when=date).write()
# mentions notif
if level in ('insert', 'update'):
accounts = set(filter(Accounts.exists, mentions(post['body'])))
accounts -= {author, parent_author}
score = Accounts.default_score(author)
if score < 30: max_mentions = 5
elif score < 60: max_mentions = 10
else: max_mentions = 25
if len(accounts) <= max_mentions:
penalty = min([score, 2 * (len(accounts) - 1)])
for mention in accounts:
mention_id = Accounts.get_id(mention)
if (not cls._mentioned(pid, mention_id)
and not cls._muted(mention_id, author_id)):
Notify('mention', src_id=author_id,
dst_id=mention_id, post_id=pid, when=date,
score=(score - penalty)).write()
else:
url = '@%s/%s' % (author, post['permlink'])
log.info("skip %d mentions in %s", len(accounts), url)
# votes notif
url = post['author'] + '/' + post['permlink']
if url in cls._votes:
voters = cls._votes[url]
del cls._votes[url]
net = float(post['net_rshares'])
ratio = float(payout) / net if net else 0
for vote in post['active_votes']:
rshares = int(vote['rshares'])
if vote['voter'] not in voters or rshares < 10e9: continue
contrib = int(1000 * ratio * rshares)
if contrib < 20: continue # < $0.020
voter_id = Accounts.get_id(vote['voter'])
if not cls._voted(pid, author_id, voter_id):
score = min(100, (len(str(contrib)) - 1) * 25) # $1 = 75
payload = "$%.3f" % (contrib / 1000)
Notify('vote', src_id=voter_id, dst_id=author_id,
when=vote['time'], post_id=pid, score=score,
payload=payload).write()
async def _load_discussion(db, author, permlink):
"""Load a full discussion thread."""
root_id = await get_post_id(db, author, permlink)
if not root_id:
return {}
# build `ids` list and `tree` map
ids = []
tree = {}
todo = [root_id]
while todo:
ids.extend(todo)
rows = await _child_ids(db, todo)
todo = []
for pid, cids in rows:
tree[pid] = cids
todo.extend(cids)
# load all post objects, build ref-map
posts = await load_posts_keyed(db, ids)
# remove posts/comments from muted accounts
muted_accounts = Mutes.all()
rem_pids = []
for pid, post in posts.items():
if post['author'] in muted_accounts:
rem_pids.append(pid)
for pid in rem_pids:
if pid in posts:
del posts[pid]
if pid in tree:
rem_pids.extend(tree[pid])
refs = {pid: _ref(post) for pid, post in posts.items()}
# add child refs to parent posts
for pid, post in posts.items():
if pid in tree:
post['replies'] = [refs[cid] for cid in tree[pid]
if cid in refs]
# return all nodes keyed by ref
return {refs[pid]: post for pid, post in posts.items()}
async def load_posts_keyed(db, ids, truncate_body=0):
"""Given an array of post ids, returns full posts objects keyed by id."""
assert ids, 'no ids passed to load_posts_keyed'
# fetch posts and associated author reps
sql = """SELECT post_id, author, permlink, title, body, category, depth,
promoted, payout, payout_at, is_paidout, children, votes,
created_at, updated_at, rshares, raw_json, json
FROM hive_posts_cache WHERE post_id IN :ids"""
result = await db.query_all(sql, ids=tuple(ids))
author_reps = await _query_author_rep_map(db, result)
muted_accounts = Mutes.all()
posts_by_id = {}
for row in result:
row = dict(row)
row['author_rep'] = author_reps[row['author']]
post = _condenser_post_object(row, truncate_body=truncate_body)
post['active_votes'] = _mute_votes(post['active_votes'],
muted_accounts)
posts_by_id[row['post_id']] = post
return posts_by_id
def run_server(conf):
"""Configure and launch the API server."""
#pylint: disable=too-many-statements
# configure jsonrpcserver logging
log_level = conf.log_level()
logging.getLogger('aiohttp.access').setLevel(logging.WARNING)
logging.getLogger('jsonrpcserver.dispatcher.response').setLevel(log_level)
truncate_response_log(
logging.getLogger('jsonrpcserver.dispatcher.request'))
truncate_response_log(
logging.getLogger('jsonrpcserver.dispatcher.response'))
# init
log = logging.getLogger(__name__)
methods = build_methods()
mutes = Mutes(conf.get('muted_accounts_url'))
Mutes.set_shared_instance(mutes)
app = web.Application()
app['config'] = dict()
app['config']['args'] = conf.args()
app['config']['hive.MAX_DB_ROW_RESULTS'] = 100000
#app['config']['hive.logger'] = logger
async def init_db(app):
"""Initialize db adapter."""
args = app['config']['args']
app['db'] = await Db.create(args['database_url'])
stats = PayoutStats(app['db'])
stats.set_shared_instance(stats)
async def close_db(app):
"""Teardown db adapter."""
app['db'].close()
await app['db'].wait_closed()
app.on_startup.append(init_db)
app.on_cleanup.append(close_db)
async def head_age(request):
"""Get hive head block age in seconds. 500 status if age > 15s."""
#pylint: disable=unused-argument
healthy_age = 15 # hive is synced if head block within 15s
try:
state = await db_head_state(app)
curr_age = state['db_head_age']
except Exception as e:
log.info("could not get head state (%s)", e)
curr_age = 31e6
status = 500 if curr_age > healthy_age else 200
return web.Response(status=status, text=str(curr_age))
async def health(request):
"""Get hive health state. 500 if db unavailable or too far behind."""
#pylint: disable=unused-argument
is_syncer = conf.get('sync_to_s3')
# while 1 hr is a bit stale, such a condition is a symptom of a
# writer issue, *not* a reader node issue. Discussion in #174.
max_head_age = 3600 # 1hr
try:
state = await db_head_state(app)
except OperationalError as e:
state = None
log.warning("could not get head state (%s)", e)
if not state:
status = 500
result = 'db not available'
elif not is_syncer and state['db_head_age'] > max_head_age:
status = 500
result = 'head block age (%s) > max (%s); head block num: %s' % (
state['db_head_age'], max_head_age, state['db_head_block'])
else:
status = 200
result = 'head block age is %d, head block num is %d' % (
state['db_head_age'], state['db_head_block'])
return web.json_response(
status=status,
data=dict(state=state,
result=result,
status='OK' if status == 200 else 'WARN',
sync_service=is_syncer,
source_commit=os.environ.get('SOURCE_COMMIT'),
schema_hash=os.environ.get('SCHEMA_HASH'),
docker_tag=os.environ.get('DOCKER_TAG'),
timestamp=datetime.utcnow().isoformat()))
async def jsonrpc_handler(request):
"""Handles all hive jsonrpc API requests."""
request = await request.text()
# debug=True refs https://github.com/bcb/jsonrpcserver/issues/71
response = await dispatch(request,
methods=methods,
debug=True,
context=app)
if response.wanted:
headers = {'Access-Control-Allow-Origin': '*'}
return web.json_response(response.deserialized(),
status=200,
headers=headers)
return web.Response()
if conf.get('sync_to_s3'):
app.router.add_get('/head_age', head_age)
app.router.add_get('/.well-known/healthcheck.json', health)
app.router.add_get('/health', health)
app.router.add_post('/', jsonrpc_handler)
web.run_app(app, port=app['config']['args']['http_server_port'])
async def load_posts_keyed(db, ids, truncate_body=0):
"""Given an array of post ids, returns full posts objects keyed by id."""
# pylint: disable=too-many-locals
assert ids, 'no ids passed to load_posts_keyed'
# fetch posts and associated author reps
sql = """SELECT post_id, community_id, author, permlink, title, body, category, depth,
promoted, payout, payout_at, is_paidout, children, votes,
created_at, updated_at, rshares, raw_json, json,
is_hidden, is_grayed, total_votes, flag_weight
FROM hive_posts_cache WHERE post_id IN :ids"""
result = await db.query_all(sql, ids=tuple(ids))
author_map = await _query_author_map(db, result)
# TODO: author affiliation?
ctx = {}
posts_by_id = {}
author_ids = {}
post_cids = {}
for row in result:
row = dict(row)
author = author_map[row['author']]
author_ids[author['id']] = author['name']
row['author_rep'] = author['reputation']
post = _condenser_post_object(row, truncate_body=truncate_body)
post['blacklists'] = Mutes.lists(post['author'], author['reputation'])
posts_by_id[row['post_id']] = post
post_cids[row['post_id']] = row['community_id']
cid = row['community_id']
if cid:
if cid not in ctx:
ctx[cid] = []
ctx[cid].append(author['id'])
# TODO: optimize
titles = {}
roles = {}
for cid, account_ids in ctx.items():
sql = "SELECT title FROM hive_communities WHERE id = :id"
titles[cid] = await db.query_one(sql, id=cid)
sql = """SELECT account_id, role_id, title
FROM hive_roles
WHERE community_id = :cid
AND account_id IN :ids"""
roles[cid] = {}
ret = await db.query_all(sql, cid=cid, ids=tuple(account_ids))
for row in ret:
name = author_ids[row['account_id']]
roles[cid][name] = (row['role_id'], row['title'])
for pid, post in posts_by_id.items():
author = post['author']
cid = post_cids[pid]
if cid:
post['community'] = post['category'] # TODO: True?
post['community_title'] = titles[cid] or post['category']
role = roles[cid][author] if author in roles[cid] else (0, '')
post['author_role'] = ROLES[role[0]]
post['author_title'] = role[1]
else:
post['stats']['gray'] = ('irredeemables' in post['blacklists']
or len(post['blacklists']) >= 2)
post['stats']['hide'] = 'irredeemables' in post['blacklists']
sql = """SELECT id FROM hive_posts
WHERE id IN :ids AND is_pinned = '1' AND is_deleted = '0'"""
for pid in await db.query_col(sql, ids=tuple(ids)):
if pid in posts_by_id:
posts_by_id[pid]['stats']['is_pinned'] = True
return posts_by_id