def __init__(self, db_path, rpc_server, mint_addr):
Thread.__init__(self)
self.db_path = db_path
started = False
logger.info('transactions db worker starting')
while not started:
try:
start_rpc_client_instance(rpc_server, mint_addr)
started = True
except:
sleep(10)
def __init__(self, config):
Thread.__init__(self)
self.url = "{DB_DIALECT}+{DB_DRIVER}://{DB_USERNAME}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}".format(**config)
logger.info('sqlalchemy.url: {}'.format(self.url))
self.db_backup_path = config['DB_BACKUP_PATH']
running = False
while not running:
try:
start_rpc_client_instance(config['RPC_SERVER'], config['MINT_ACCOUNT'])
running = True
except:
sleep(10)
########
# Main #
########
if __name__ == '__main__':
with open('config.json', 'r') as f:
config = json.load(f)
try:
config = config[os.getenv("BROWSER")]
except:
config = config["PRODUCTION"]
app.logger.info("system configuration: {}".format(
json.dumps(config, indent=4)))
TxDBWorker(config['DB_PATH'], config['RPC_SERVER'],
config['MINT_ACCOUNT']).start()
start_rpc_client_instance(config['RPC_SERVER'], config['MINT_ACCOUNT'])
start_client_instance(config['CLIENT_PATH'], config['ACCOUNT_FILE'])
sleep(1)
app.run(port=config['FLASK_PORT'],
threaded=config['FLASK_THREADED'],
host=config['FLASK_HOST'],
debug=config['FLASK_DEBUG'])
def tx_db_worker(db_path, rpc_server, mint_addr):
while True:
try:
logger.info('transactions db worker starting')
# create rpc connection
try:
start_rpc_client_instance(rpc_server, mint_addr)
except:
sleep(10)
start_rpc_client_instance(rpc_server, mint_addr)
# connect to DB
c, conn = connect_to_db(db_path) # returns cursor object
init_db(c)
# get latest version in the db
cur_ver = get_latest_version(c)
cur_ver += 1 # TODO: later handle genesis
logger.info('starting update at version {}'.format(cur_ver))
# start the main loop
while True:
try:
bver = get_latest_version_from_ledger()
except:
sleep(1)
continue
if cur_ver > bver:
sleep(1)
continue
# batch update
num = min(1000, bver - cur_ver) # at most 5000 records at once
tx_data = get_raw_tx_lst(cur_ver, num)
# read records
res = parse_raw_tx_lst(*tx_data)
if len(res) == 0:
sleep(5)
continue
# do the insertion
db_data = [tuple(x.values()) for x in res]
c.executemany(
"INSERT INTO transactions VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);",
db_data)
# update counter to the latest version we inserted
cur_ver = res[-1]['version']
logger.debug('update to version: {} - success'.format(cur_ver))
# Save (commit) the changes
conn.commit()
# update latest version to next
cur_ver += 1
# sleep relative to amount of rows fetched so we don't get a 429 error
sleep(0.001 * num)
except:
logger.exception('Major error in tx_db_worker')
sleep(2)
logger.info('restarting tx_db_worker')