async def health(request):
"""Get hive health data. 500 if behind by more than 3 blocks."""
#pylint: disable=unused-argument
is_syncer = Conf.get('sync_to_s3')
max_head_age = (Conf.get('trail_blocks') + 3) * 3
state = await _head_state()
if not state:
status = 500
result = 'db not available'
elif not is_syncer and state['db_head_age'] > max_head_age:
status = 500
result = 'head block age (%s) > max (%s); head block num: %s' % (
state['db_head_age'], max_head_age, state['db_head_block'])
else:
status = 200
result = 'head block age is %d, head block num is %d' % (
state['db_head_age'], state['db_head_block'])
return web.json_response(
status=status,
data=dict(state=state,
result=result,
status='OK' if status == 200 else 'WARN',
sync_service=is_syncer,
source_commit=os.environ.get('SOURCE_COMMIT'),
schema_hash=os.environ.get('SCHEMA_HASH'),
docker_tag=os.environ.get('DOCKER_TAG'),
timestamp=datetime.utcnow().isoformat()))
def instance(cls):
"""Get a singleton, lazily initialized"""
if not cls._instance:
cls._instance = SteemClient(url=Conf.get('steemd_url'),
max_batch=Conf.get('max_batch'),
max_workers=Conf.get('max_workers'))
return cls._instance
def run():
Conf.init_argparse()
mode = '/'.join(Conf.get('mode'))
if mode == 'server':
run_server()
elif mode == 'sync':
run_sync()
elif mode == 'status':
print(DbState.status())
else:
raise Exception("unknown run mode %s" % mode)
def run(cls):
"""Initialize state; setup/recovery checks; sync and runloop."""
# ensure db schema up to date, check app status
DbState.initialize()
# prefetch id->name memory map
Accounts.load_ids()
if DbState.is_initial_sync():
# resume initial sync
cls.initial()
DbState.finish_initial_sync()
else:
# recover from fork
Blocks.verify_head()
# perform cleanup if process did not exit cleanly
CachedPost.recover_missing_posts()
# debug mode: no sync, just stream
if Conf.get('disable_sync'):
return cls.listen()
while True:
# sync up to irreversible block
cls.from_steemd()
# take care of payout backlog
CachedPost.dirty_paidouts(Blocks.head_date())
CachedPost.flush(trx=True)
# listen for new blocks
cls.listen()
def create_engine(echo=False):
engine = sqlalchemy.create_engine(
Conf.get('database_url'),
isolation_level="READ UNCOMMITTED", # only works in mysql
pool_recycle=3600,
echo=echo)
return engine
def listen(cls):
trail_blocks = Conf.get('trail_blocks')
assert trail_blocks >= 0
assert trail_blocks < 25
steemd = SteemClient.instance()
hive_head = Blocks.head_num()
for block in steemd.stream_blocks(hive_head + 1, trail_blocks, max_gap=40):
start_time = time.perf_counter()
query("START TRANSACTION")
num = Blocks.process(block)
follows = Follow.flush(trx=False)
accts = Accounts.flush(trx=False, period=8)
CachedPost.dirty_paidouts(block['timestamp'])
cnt = CachedPost.flush(trx=False)
query("COMMIT")
ms = (time.perf_counter() - start_time) * 1000
print("[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits,"
"% 3d payouts,% 3d votes,% 3d accounts,% 3d follows --% 5dms%s"
% (num, block['timestamp'], len(block['transactions']),
cnt['insert'], cnt['update'], cnt['payout'], cnt['upvote'],
accts, follows, int(ms), ' SLOW' if ms > 1000 else ''))
# once per hour, update accounts
if num % 1200 == 0:
Accounts.dirty_oldest(10000)
Accounts.flush(trx=True)
#Accounts.update_ranks()
# once a minute, update chain props
if num % 20 == 0:
cls._update_chain_state(steemd)
def run():
"""Run the proper routine as indicated by hive --mode argument."""
conf = Conf.init_argparse()
Db.set_shared_instance(conf.db())
mode = conf.mode()
if mode == 'server':
from hive.server.serve import run_server
run_server(conf=conf)
elif mode == 'sync':
from hive.indexer.sync import Sync
Sync(conf=conf).run()
elif mode == 'status':
from hive.db.db_state import DbState
print(DbState.status())
#elif mode == 'sync-profile':
# from hive.indexer.sync import Sync
# from hive.utils.profiler import Profiler
# with Profiler():
# Sync(conf=conf).run()
else:
raise Exception("unknown run mode %s" % mode)
def create_engine(echo=False):
"""Create a new SA db engine. Use echo=True for ultra verbose."""
engine = sqlalchemy.create_engine(
Conf.get('database_url'),
isolation_level="READ UNCOMMITTED", # only works in mysql
pool_recycle=3600,
echo=echo)
return engine
def run():
"""Run the proper routine as indicated by hive --mode argument."""
Conf.init_argparse()
mode = Conf.run_mode()
if mode == 'server':
run_server()
elif mode == 'sync':
Sync.run()
elif mode == 'status':
print(DbState.status())
else:
raise Exception("unknown run mode %s" % mode)
def listen(cls):
"""Live (block following) mode."""
trail_blocks = Conf.get('trail_blocks')
assert trail_blocks >= 0
assert trail_blocks <= 100
# debug: no max gap if disable_sync in effect
max_gap = None if Conf.get('disable_sync') else 100
steemd = SteemClient.instance()
hive_head = Blocks.head_num()
for block in steemd.stream_blocks(hive_head + 1, trail_blocks,
max_gap):
start_time = perf()
query("START TRANSACTION")
num = Blocks.process(block)
follows = Follow.flush(trx=False)
accts = Accounts.flush(trx=False, spread=8)
CachedPost.dirty_paidouts(block['timestamp'])
cnt = CachedPost.flush(trx=False)
query("COMMIT")
ms = (perf() - start_time) * 1000
log.info(
"[LIVE] Got block %d at %s --% 4d txs,% 3d posts,% 3d edits,"
"% 3d payouts,% 3d votes,% 3d accts,% 3d follows --% 5dms%s",
num, block['timestamp'], len(block['transactions']),
cnt['insert'], cnt['update'], cnt['payout'], cnt['upvote'],
accts, follows, int(ms), ' SLOW' if ms > 1000 else '')
# once per hour, update accounts
if num % 1200 == 0:
Accounts.dirty_oldest(10000)
Accounts.flush(trx=True)
#Accounts.update_ranks()
# once a minute, update chain props
if num % 20 == 0:
cls._update_chain_state(steemd)
def run():
"""Run the service specified in the `--mode` argument."""
conf = Conf.init_argparse()
Db.set_shared_instance(conf.db())
mode = conf.mode()
if conf.get('test_profile'):
from hive.utils.profiler import Profiler
with Profiler():
launch_mode(mode, conf)
else:
launch_mode(mode, conf)
async def health(request):
"""Get hive health data. 500 if behind by more than a few secs."""
#pylint: disable=unused-argument
is_syncer = Conf.get('sync_to_s3')
max_head_age = (Conf.get('trail_blocks') + 1) * 3
try:
state = await hive_api.db_head_state()
except OperationalError as e:
if 'could not connect to server: Connection refused' in str(e):
logging.error("hive /health could not connect to db")
state = None
else:
raise e
if not state:
status = 500
result = 'db not available'
elif not is_syncer and state['db_head_age'] > max_head_age:
status = 500
result = 'head block age (%s) > max (%s); head block num: %s' % (
state['db_head_age'], max_head_age, state['db_head_block'])
else:
status = 200
result = 'head block age is %d, head block num is %d' % (
state['db_head_age'], state['db_head_block'])
return web.json_response(
status=status,
data=dict(state=state,
result=result,
status='OK' if status == 200 else 'WARN',
sync_service=is_syncer,
source_commit=os.environ.get('SOURCE_COMMIT'),
schema_hash=os.environ.get('SCHEMA_HASH'),
docker_tag=os.environ.get('DOCKER_TAG'),
timestamp=datetime.utcnow().isoformat()))
def run():
"""Run the proper routine as indicated by hive --mode argument."""
conf = Conf.init_argparse()
Db.set_shared_instance(conf.db())
mode = '/'.join(conf.get('mode'))
if mode == 'server':
from hive.server.serve import run_server
run_server(conf=conf)
elif mode == 'sync':
from hive.indexer.sync import Sync
Sync(conf=conf).run()
elif mode == 'status':
from hive.db.db_state import DbState
print(DbState.status())
else:
raise Exception("unknown run mode %s" % mode)
async def health(request):
#pylint: disable=unused-argument
state = await hive_api.db_head_state()
max_head_age = (Conf.get('trail_blocks') + 1) * 3
if state['db_head_age'] > max_head_age:
status = 500
result = 'head block age (%s) > max (%s); head block num: %s' % (
state['db_head_age'], max_head_age, state['db_head_block'])
else:
status = 200
result = 'head block age is %d, head block num is %d' % (
state['db_head_age'], state['db_head_block'])
return web.json_response(
status=status,
data=dict(state=state,
result=result,
status='OK' if status == 200 else 'WARN',
source_commit=os.environ.get('SOURCE_COMMIT'),
schema_hash=os.environ.get('SCHEMA_HASH'),
docker_tag=os.environ.get('DOCKER_TAG'),
timestamp=datetime.utcnow().isoformat()))
def run_server():
"""Configure and launch the API server."""
log_level = Conf.log_level()
config.debug = (log_level == logging.DEBUG)
logging.basicConfig(level=log_level)
logger = logging.getLogger(__name__)
logging.getLogger('jsonrpcserver.dispatcher.response').setLevel(log_level)
methods = build_methods()
app = web.Application()
app['config'] = dict()
app['config']['args'] = Conf.args()
app['config']['hive.MAX_DB_ROW_RESULTS'] = 100000
app['config'][
'hive.DB_QUERY_LIMIT'] = app['config']['hive.MAX_DB_ROW_RESULTS'] + 1
app['config']['hive.logger'] = logger
#async def init_db(app):
# args = app['config']['args']
# db = make_url(args.database_url)
# engine = await create_engine(user=db.username,
# database=db.database,
# password=db.password,
# host=db.host,
# port=db.port,
# **db.query)
# app['db'] = engine
#
#async def close_db(app):
# app['db'].close()
# await app['db'].wait_closed()
#
#app.on_startup.append(init_db)
#app.on_cleanup.append(close_db)
async def head_age(request):
"""Get hive head block age in seconds. 500 if greater than 15s."""
#pylint: disable=unused-argument
healthy_age = 15 # hive is synced if head block within 15s
curr_age = (await hive_api.db_head_state())['db_head_age']
status = 500 if curr_age > healthy_age else 200
return web.Response(status=status, text=str(curr_age))
async def health(request):
"""Get hive health data. 500 if behind by more than a few secs."""
#pylint: disable=unused-argument
is_syncer = Conf.get('sync_to_s3')
max_head_age = (Conf.get('trail_blocks') + 1) * 3
try:
state = await hive_api.db_head_state()
except OperationalError as e:
if 'could not connect to server: Connection refused' in str(e):
logging.error("hive /health could not connect to db")
state = None
else:
raise e
if not state:
status = 500
result = 'db not available'
elif not is_syncer and state['db_head_age'] > max_head_age:
status = 500
result = 'head block age (%s) > max (%s); head block num: %s' % (
state['db_head_age'], max_head_age, state['db_head_block'])
else:
status = 200
result = 'head block age is %d, head block num is %d' % (
state['db_head_age'], state['db_head_block'])
return web.json_response(
status=status,
data=dict(state=state,
result=result,
status='OK' if status == 200 else 'WARN',
sync_service=is_syncer,
source_commit=os.environ.get('SOURCE_COMMIT'),
schema_hash=os.environ.get('SCHEMA_HASH'),
docker_tag=os.environ.get('DOCKER_TAG'),
timestamp=datetime.utcnow().isoformat()))
async def jsonrpc_handler(request):
"""Handles all hive jsonrpc API requests."""
request = await request.text()
response = await methods.dispatch(request)
return web.json_response(response,
status=200,
headers={'Access-Control-Allow-Origin': '*'})
app.router.add_get('/.well-known/healthcheck.json', health)
app.router.add_get('/head_age', head_age)
app.router.add_get('/health', health)
app.router.add_post('/', jsonrpc_handler)
web.run_app(app, port=app['config']['args'].http_server_port)
return web.json_response(
status=status,
data=dict(state=state,
result=result,
status='OK' if status == 200 else 'WARN',
sync_service=is_syncer,
source_commit=os.environ.get('SOURCE_COMMIT'),
schema_hash=os.environ.get('SCHEMA_HASH'),
docker_tag=os.environ.get('DOCKER_TAG'),
timestamp=datetime.utcnow().isoformat()))
async def jsonrpc_handler(request):
"""Handles all hive jsonrpc API requests."""
request = await request.text()
response = await methods.dispatch(request)
return web.json_response(response,
status=200,
headers={'Access-Control-Allow-Origin': '*'})
app.router.add_get('/.well-known/healthcheck.json', health)
app.router.add_get('/head_age', head_age)
app.router.add_get('/health', health)
app.router.add_post('/', jsonrpc_handler)
web.run_app(app, port=app['config']['args'].http_server_port)
if __name__ == '__main__':
Conf.init_argparse()
run_server()
"""Hive server and API tests.""" from hive.conf import Conf from hive.db.adapter import Db Db.set_shared_instance(Conf.init_test().db())
import datetime
import pytest
from hive.steem.client import SteemClient
from hive.utils.normalize import parse_time
from hive.conf import Conf
Conf.init_argparse(ignore_unknown=True)
def test_instance():
assert isinstance(SteemClient.instance(), SteemClient)
def test_get_accounts():
client = SteemClient.instance()
accounts = client.get_accounts(['steemit', 'test-safari'])
assert len(accounts) == 2
assert accounts[0]['name'] == 'steemit'
def test_get_content_batch():
client = SteemClient.instance()
tuples = [('test-safari', 'may-spam'), ('test-safari', 'june-spam')]
posts = client.get_content_batch(tuples)
assert len(posts) == 2
assert posts[0]['author'] == 'test-safari'
assert posts[1]['author'] == 'test-safari'
def test_get_block():
def instance(cls):
if not cls._instance:
cls._instance = SteemClient(url=Conf.get('steemd_url'),
max_batch=Conf.get('max_batch'),
max_workers=Conf.get('max_workers'))
return cls._instance
"""Hive server and API tests.""" from hive.conf import Conf Conf.init_test()
def run_server():
log_level = Conf.log_level()
config.debug = (log_level == logging.DEBUG)
logging.basicConfig(level=log_level)
logger = logging.getLogger(__name__)
logging.getLogger('jsonrpcserver.dispatcher.response').setLevel(log_level)
methods = build_methods()
app = web.Application()
app['config'] = dict()
app['config']['args'] = Conf.args()
app['config']['hive.MAX_DB_ROW_RESULTS'] = 100000
app['config'][
'hive.DB_QUERY_LIMIT'] = app['config']['hive.MAX_DB_ROW_RESULTS'] + 1
app['config']['hive.logger'] = logger
async def init_db(app):
args = app['config']['args']
db = make_url(args.database_url)
engine = await create_engine(user=db.username,
database=db.database,
password=db.password,
host=db.host,
port=db.port,
**db.query)
app['db'] = engine
async def close_db(app):
app['db'].close()
await app['db'].wait_closed()
app.on_startup.append(init_db)
app.on_cleanup.append(close_db)
async def health(request):
#pylint: disable=unused-argument
state = await hive_api.db_head_state()
max_head_age = (Conf.get('trail_blocks') + 1) * 3
if state['db_head_age'] > max_head_age:
status = 500
result = 'head block age (%s) > max (%s); head block num: %s' % (
state['db_head_age'], max_head_age, state['db_head_block'])
else:
status = 200
result = 'head block age is %d, head block num is %d' % (
state['db_head_age'], state['db_head_block'])
return web.json_response(
status=status,
data=dict(state=state,
result=result,
status='OK' if status == 200 else 'WARN',
source_commit=os.environ.get('SOURCE_COMMIT'),
schema_hash=os.environ.get('SCHEMA_HASH'),
docker_tag=os.environ.get('DOCKER_TAG'),
timestamp=datetime.utcnow().isoformat()))
async def jsonrpc_handler(request):
request = await request.text()
response = await methods.dispatch(request)
return web.json_response(response,
status=200,
headers={'Access-Control-Allow-Origin': '*'})
app.router.add_get('/health', health)
app.router.add_post('/', jsonrpc_handler)
web.run_app(app, port=app['config']['args'].http_server_port)