def get(self, id=None, height=0, range=None, fields=None):
database = lib.get_db()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn(
"WorkerAPI_stats get id:{} height:{} range:{} fields:{}".format(
id, height, range, fields))
fields = lib.fields_to_list(fields)
if height == 0:
height = Blocks.get_latest().height
stats = []
if id is None:
for stat in Worker_stats.get_by_height(height, range):
#print("YYY: {}".format(stats))
stats.append(stat.to_json(fields))
return stats
else:
if range is None:
res = Worker_stats.get_by_height_and_id(id, height)
if res is None:
return "[]".to_json()
return res.to_json(fields)
else:
for stat in Worker_stats.get_by_height_and_id(
id, height, range):
stats.append(stat.to_json(fields))
return stats
def get(self, id=None, height=0, range=0, fields=None):
database = lib.get_db()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn(
"WorkerAPI_shares get id:{} height:{} range:{} fields:{}".format(
id, height, range, fields))
fields = lib.fields_to_list(fields)
if height == 0:
height = Blocks.get_latest().height
shares_records = []
if id is None:
for shares in Worker_shares.get_by_height(height, range):
shares_records.append(shares.to_json(fields))
return shares_records
else:
if range is None:
worker_sh_recs = Worker_shares.get_by_height_and_id(height, id)
#print("worker_sh_recs = {}".format(worker_sh_recs))
if res is None:
return "[]".to_json()
return res.to_json(fields)
else:
for share in Worker_shares.get_by_height_and_id(
height, id, range):
shares_records.append(share.to_json(fields))
return shares_records
def get(self, height=None, range=None):
database = lib.get_db()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn(
"PoolAPI_shareCount get height:{} range:{} fields:{}".format(
height, range))
# Totals across all workers are stored in the Pool_stats record
if range is None:
if height is None:
height = 0
pool_st_rec = Pool_stats.get_by_height(height)
if pool_st_rec is None:
count = 0
else:
count = pool_st_rec.total_shares_processed
return {
"height": height,
"count": count,
}
else:
counts = []
pool_st_recs = Pool_stats.get_by_height(height, range)
for st_rec in pool_st_recs:
rec = {
"height": st_rec.height,
"count": st_rec.total_shares_processed,
}
counts.append(rec)
return counts
def initialize(window_size, logger):
database = lib.get_db()
# Special case for new pool startup
block_zero = None
while block_zero is None:
logger.warn("Waiting for the first block record in the database")
time.sleep(1)
block_zero = Blocks.get_earliest()
print("block_zero={}".format(block_zero))
stat_height = max(0, block_zero.height + window_size)
seed_stat = Pool_stats(
height=stat_height,
timestamp=datetime.utcnow(),
active_miners=0,
shares_processed=0,
share_counts=None,
total_blocks_found=0,
total_shares_processed=0,
dirty=False,
)
database.db.createDataObj(seed_stat)
seed_share = Worker_shares(
height=stat_height,
user_id=1,
timestamp=datetime.utcnow(),
)
database.db.createDataObj(seed_share)
def main():
global CONFIG
global LOGGER
CONFIG = lib.get_config()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
# Number of blocks of share data used to calculate rewards
PPLNG_WINDOW_SIZE = 60
try:
PPLNG_WINDOW_SIZE = int(os.environ["PPLNG_WINDOW_SIZE"])
except Exception as e:
LOGGER.error(
"Failed to get PPLNG_WINDOW_SIZE from the environment: {}. Using default size of {}"
.format(e, PPLNG_WINDOW_SIZE))
# Connect to DB
database = lib.get_db()
esitmated = [
] # Blocks we know have already been estimated - XXX TODO: Clean paid blocks out of this list
# Get Config settings
pool_fee = float(CONFIG[PROCESS]["pool_fee"])
while True:
# Generate pool block reward estimates for all new and unlocked blocks
try:
database.db.initializeSession()
unlocked_blocks = Pool_blocks.get_all_unlocked()
new_blocks = Pool_blocks.get_all_new()
unlocked_blocks_h = [blk.height for blk in unlocked_blocks]
new_blocks_h = [blk.height for blk in new_blocks]
need_estimates = []
for height in unlocked_blocks_h + new_blocks_h:
if height not in esitmated:
need_estimates.append(height)
if need_estimates:
LOGGER.warn("Will ensure estimate for blocks: {}".format(
need_estimates))
# Generate Estimate
for height in need_estimates:
LOGGER.warn("Ensure estimate for block: {}".format(height))
payout_map = pool.calculate_block_payout_map(
height, PPLNG_WINDOW_SIZE, pool_fee, LOGGER, True)
# Double check the total paid is correct
esitmated.append(height)
LOGGER.warn(
"Completed estimate for block: {}".format(height))
LOGGER.warn("Completed estimates")
database.db.destroySession()
sleep(check_interval)
except Exception as e: # AssertionError as e:
LOGGER.error("Something went wrong: {} - {}".format(
e, traceback.print_stack()))
LOGGER.warn("=== Completed {}".format(PROCESS))
sleep(check_interval)
def main():
CONFIG = lib.get_config()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
# Connect to DB
database = lib.get_db()
database.db.initializeSession()
while True:
# Generate pool block reward estimates for all new and unlocked blocks
try:
unlocked_blocks = Pool_blocks.get_all_unlocked()
unlocked_blocks_h = [blk.height for blk in unlocked_blocks]
new_blocks = Pool_blocks.get_all_new()
new_blocks_h = [blk.height for blk in new_blocks]
LOGGER.warn("Will ensure estimate for blocks: {}".format(
unlocked_blocks_h + new_blocks_h))
# Generate Estimate
for height in unlocked_blocks_h + new_blocks_h:
LOGGER.warn("Ensure estimate for block: {}".format(height))
payout_map = pool.calculate_block_payout_map(
height, 60, LOGGER, True)
LOGGER.warn("Completed estimate for block: {}".format(height))
#database.db.getSession().commit()
LOGGER.warn("Completed estimates")
sleep(check_interval)
except Exception as e: # AssertionError as e:
LOGGER.error("Something went wrong: {} - {}".format(
e, traceback.print_stack()))
LOGGER.warn("=== Completed {}".format(PROCESS))
sleep(check_interval)
def main():
config = lib.get_config()
PROCESS = "libNetworkTest"
LOGGER = lib.get_logger(PROCESS)
database = lib.get_db()
# Get stats
stats = get_stats()
LOGGER.warn("stats = {}".format(stats))
LOGGER.warn("")
# Get blocks found
bf = get_blocks_found_data(5)
LOGGER.warn("blocks found = {}".format(bf))
LOGGER.warn("")
# Get graph rate
gr = get_graph_rate_data(20)
LOGGER.warn("graph rate = {}".format(gr))
LOGGER.warn("")
# Get difficulty data
diff = get_difficulty_data(200)
LOGGER.warn("difficulty = {}".format(diff))
sys.exit(1)
def main():
CONFIG = lib.get_config()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
# Connect to DB
database = lib.get_db()
database.db.initializeSession()
pp = pprint.PrettyPrinter(indent=4)
# Fetch and print pool block reward estimates for latest N pool blocks
try:
pool_blocks = Pool_blocks.get_latest(NUM_BLOCKS)
pool_blocks_h = [blk.height for blk in pool_blocks]
LOGGER.warn(
"Will report estimates for pool blocks: {}".format(pool_blocks_h))
# Print Estimate
for height in pool_blocks_h:
pp.pprint("Eestimate for block: {}".format(height))
payout_map = pool.get_block_payout_map_estimate(height, LOGGER)
pp.pprint(payout_map)
except Exception as e: # AssertionError as e:
LOGGER.error("Something went wrong: {} - {}".format(
e, traceback.print_stack()))
LOGGER.warn("=== Completed {}".format(PROCESS))
def initialize():
database = lib.get_db()
# Special case for new pool startup - Need 3 stats records to bootstrap
block_zero = Blocks.get_by_height(0)
seed_stat0 = Grin_stats(height=0,
timestamp=block_zero.timestamp,
gps=0,
difficulty=block_zero.total_difficulty,
total_utxoset_size=block_zero.num_inputs)
database.db.createDataObj(seed_stat0)
block_one = Blocks.get_by_height(1)
seed_stat1 = Grin_stats(height=1,
timestamp=block_one.timestamp,
gps=0,
difficulty=block_one.total_difficulty -
block_zero.total_difficulty,
total_utxoset_size=seed_stat0.total_utxoset_size +
block_one.num_outputs - block_one.num_inputs)
database.db.createDataObj(seed_stat1)
block_two = Blocks.get_by_height(2)
seed_stat2 = Grin_stats(height=2,
timestamp=block_two.timestamp,
gps=0,
difficulty=block_two.total_difficulty -
block_one.total_difficulty,
total_utxoset_size=seed_stat1.total_utxoset_size +
block_two.num_outputs - block_two.num_inputs)
database.db.createDataObj(seed_stat2)
def process_pool_log():
global LOGGER
global CONFIG
# Connect to DB
database = lib.get_db()
POOL_LOG = CONFIG["stratum"]["log_dir"] + "/" + CONFIG["stratum"]["log_filename"]
# (re)Process all logs
logfiles = glob.glob(POOL_LOG + '*')
LOGGER.warn("Processing existing logs: {}".format(logfiles))
sys.stdout.flush()
for logfile in logfiles:
with open(logfile) as f:
for line in f:
try:
process_pool_logmessage(line, database)
except Exception as e:
LOGGER.error("Failed to process pool log message: {} {}".format(line, e))
f.close()
# Read future log messages
LOGGER.warn("Processing new logs: {}".format(POOL_LOG))
sys.stdout.flush()
poollog = subprocess.Popen(
['tail', '-F', POOL_LOG],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
while True:
line = poollog.stdout.readline().decode('utf-8')
try:
process_pool_logmessage(line, database)
except Exception as e:
LOGGER.error("Failed to process pool log message: {} {}".format(line, e))
def main():
CONFIG = lib.get_config()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
# Connect to DB
database = lib.get_db()
# Get the list of pool_blocks that have been paid
paid_poolblocks = Pool_blocks.get_all_paid()
print("Number of paid_poolblocks: {}".format(len(paid_poolblocks)))
sys.stdout.flush()
for pb in paid_poolblocks:
# Get the blockchain data for this block
response = grin.get_block_by_height(pb.height)
if response == None:
LOGGER.error("Failed to get block {}".format(pb.height))
continue
if int(response["header"]["nonce"]) != int(pb.nonce):
print("")
sys.stdout.flush()
LOGGER.warn("Processed orphan pool block at height: {}".format(
pb.height))
else:
sys.stdout.write(".")
sys.stdout.flush()
def ShareCommitScheduler(interval, database):
global LOGGER
global SHARES
global HEIGHT
global GRINSHARE_HEIGHT
global POOLSHARE_HEIGHT
database = lib.get_db()
try:
# XXX TODO: enhance
while True:
bc_height = Blocks.get_latest(
).height # grin.blocking_get_current_height()
LOGGER.warn(
"HEIGHT={}, POOLSHARE_HEIGHT={}, GRINSHARE_HEIGHT={}".format(
HEIGHT, POOLSHARE_HEIGHT, GRINSHARE_HEIGHT))
while (HEIGHT < POOLSHARE_HEIGHT
and HEIGHT < GRINSHARE_HEIGHT) or (bc_height > HEIGHT):
# Commit and purge current block share data if we are starting a new block
LOGGER.warn("Commit shares for height: {}".format(HEIGHT))
# time.sleep(5) # Give straggler shares a chance to come in
SHARES.commit(HEIGHT)
HEIGHT = HEIGHT + 1
# Commit and purge all old share data (except current block) every 'interval' seconds
try:
SHARES.commit() # All except current block
except Exception as e:
LOGGER.error("Failed to commit: {}".format(e))
time.sleep(interval)
except Exception as e:
LOGGER.error("Something went wrong: {}\n{}".format(
e,
traceback.format_exc().splitlines()))
time.sleep(interval)
lib.teardown_db()
def main():
CONFIG = lib.get_config()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
# Connect to DB
database = lib.get_db()
atexit.register(lib.teardown_db)
# Get config
check_interval = float(CONFIG[PROCESS]["check_interval"])
avg_over_range = int(CONFIG[PROCESS]["avg_over_range"])
# Initialize poolStats records if this is the first run
latest_stat = Pool_stats.get_latest()
if latest_stat is None:
# Special case for new pool startup
poolstats.initialize(avg_over_range, LOGGER)
latest_stat = Pool_stats.get_latest()
LOGGER.warn("Starting at height: {}".format(latest_stat.height))
# Generate pool stats records - one per grin block
while True:
# Find the height of the latest stats record
latest_stat = Pool_stats.get_latest()
height = latest_stat.height + 1
LOGGER.warn("Starting at height: {}".format(height))
try:
while True:
share_height = Worker_shares.get_latest_height()
while share_height is None:
LOGGER.warn("Waiting for shares")
share_height = Worker_shares.get_latest_height()
sleep(10)
latest = Blocks.get_latest().height
stats_height = height - 1
LOGGER.warn(
"Running: Chain height: {}, share height: {}, stats height: {}"
.format(latest, share_height, stats_height))
while share_height - 1 > height:
new_stats = poolstats.calculate(height, avg_over_range)
# Batch new stats when possible, but commit at reasonable intervals
database.db.getSession().add(new_stats)
if ((height % BATCHSZ == 0) or (height >= (latest - 10))):
database.db.getSession().commit()
LOGGER.warn(
"Added Pool_stats for block: {} - {} {} {}".format(
new_stats.height, new_stats.gps,
new_stats.active_miners,
new_stats.shares_processed))
height = height + 1
sys.stdout.flush()
sleep(check_interval)
except Exception as e:
LOGGER.error("Something went wrong: {} - {}".format(
e, traceback.print_stack()))
database.db.getSession().rollback()
sleep(check_interval)
LOGGER.warn("=== Completed {}".format(PROCESS))
def main():
global LOGGER
global CONFIG
global SHARES
global HEIGHT
global GRINSHARE_HEIGHT
global POOLSHARE_HEIGHT
CONFIG = lib.get_config()
# XXX TODO: Put in config
HOST = "0.0.0.0"
PORT = 32080
GRINSHARE_HEIGHT = 0
POOLSHARE_HEIGHT = 0
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
database = lib.get_db()
HEIGHT = Worker_shares.get_latest_height()
if HEIGHT is None:
HEIGHT = grin.blocking_get_current_height()
SHARES = WorkerShares(LOGGER)
#server = ThreadedHTTPServer((HOST, PORT), ShareHandler)
#server = HTTPServer((HOST, PORT), ShareHandler)
# server = socketserver.TCPServer((HOST, PORT), ShareHandler)
# server.handle_request()
# server.server_close()
commit_thread = threading.Thread(target=ShareCommitScheduler, args=(15, ))
commit_thread.start()
server = ThreadedTCPServer((HOST, PORT), ShareHandler)
server.serve_forever()
def get(self, id=None, height=0, range=None):
database = lib.get_db()
print("id={} , height={}, range = {}".format(id, height, range))
if range == 0:
range = grin.get_current_height()
if height == 0:
height = grin.get_current_height()
return Pool_shares.count(height, range, id)
def get_reward_by_block(height):
# Get the block and determine how much its worth to the winner
database = lib.get_db()
theblock = Blocks.get_by_height(height)
#print("The block {}".format(theblock.to_json()))
if theblock is None:
return 0
return 60 * 1000000000 + theblock.fee
def get(self, id=None, fields=None):
database = lib.get_db()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("WorkerAPI_payments get id:{} fields:{}".format(
id, fields))
fields = lib.fields_to_list(fields)
utxo = Pool_utxo.get_by_address(id)
return utxo.to_json(fields)
def main():
CONFIG = lib.get_config()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
##
# Update user records in MySQL and REDIS
database = lib.get_db()
database.db.initializeSession()
redisdb = lib.get_redis_db()
redis_userid_key = "userid."
id = 1
try:
while True:
thisuser = Users.get_by_id(id)
if thisuser is None:
if id > 2358:
LOGGER.warn("last id = {}".format(id))
break
id = id + 1
continue
if thisuser.username == "bjg62hj8byyksphuw95vqc3f74.lionm1":
orig_username = thisuser.username
thisuser.username = "******"
LOGGER.warn("Updated: {} to {}".format(orig_username,
thisuser.username))
if thisuser.username == "[email protected]_d1":
orig_username = thisuser.username
thisuser.username = "******"
LOGGER.warn("Updated: {} to {}".format(orig_username,
thisuser.username))
if thisuser.username == "*****@*****.**":
orig_username = thisuser.username
thisuser.username = "******"
LOGGER.warn("Updated: {} to {}".format(orig_username,
thisuser.username))
if "." in thisuser.username:
orig_username = thisuser.username
# Update mysql
thisuser.username = thisuser.username.replace(".", "_")
# Update redis
redis_key = redis_userid_key + orig_username
COMMIT and redisdb.delete(redis_key)
redis_key = redis_userid_key + thisuser.username
COMMIT and redisdb.set(redis_key, id)
LOGGER.warn("Updated: {} to {}".format(orig_username,
thisuser.username))
id = id + 1
except Exception as e: # AssertionError as e:
LOGGER.error("Something went wrong: {} - {}".format(
e, traceback.print_stack()))
COMMIT or LOGGER.warn("XXX No Commit - Edit for final run")
COMMIT and database.db.getSession().commit()
LOGGER.warn("=== Completed {}".format(PROCESS))
def main():
global LOGGER
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
# Connect to DB
database = lib.get_db()
latest_block = 0
# XXX All in one db transaction....
# Get unlocked blocks from the db
unlocked_blocks = Pool_blocks.get_all_unlocked()
database.db.getSession().commit()
for pb in unlocked_blocks:
try:
LOGGER.warn("Processing unlocked block: {}".format(pb))
if pb.height > latest_block:
latest_block = pb.height
# Get valid pool_shares for that block from the db
pool_shares = Pool_shares.get_valid_by_height(pb.height)
# Calculate Payment info:
worker_shares = {}
for ps in pool_shares:
LOGGER.warn("Processing pool_shares: {}".format(ps))
# Need to get actual_difficulty
gs = Grin_shares.get_by_nonce(ps.nonce)
if gs == None:
# XXX NOTE: no payout for shares not accepted by grin node
continue
if ps.found_by in worker_shares:
worker_shares[ps.found_by] += gs.actual_difficulty
else:
worker_shares[ps.found_by] = gs.actual_difficulty
if len(worker_shares) > 0:
# Calcualte reward/difficulty: XXX TODO: Enhance
# What algorithm to use? Maybe: https://slushpool.com/help/manual/rewards
r_per_d = REWARD / sum(worker_shares.values())
for worker in worker_shares.keys():
# Calculate reward per share
worker_rewards = worker_shares[worker] * r_per_d
# Add or create worker rewards
worker_utxo = Pool_utxo.credit_worker(
worker, worker_rewards)
LOGGER.warn("Credit to user: {} = {}".format(
worker, worker_rewards))
# Mark the pool_block state="paid" (maybe "processed" would be more accurate?)
pb.state = "paid"
database.db.getSession().commit()
except Exception as e:
database.db.getSession().rollback()
LOGGER.error("Something went wrong: {}".format(e))
#database.db.getSession().commit()
# db.set_last_run(PROCESS, str(time.time()))
LOGGER.warn("=== Completed {}".format(PROCESS))
sys.stdout.flush()
def main():
CONFIG = lib.get_config()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
# Connect to DB
database = lib.get_db()
# Get config
check_interval = float(CONFIG[PROCESS]["check_interval"])
avg_over_range = int(CONFIG[PROCESS]["avg_over_range"])
# Find the height of the latest stats record
last_height = 0
latest_stat = Worker_stats.get_latest()
if latest_stat != None:
last_height = latest_stat.height
height = last_height + 1
LOGGER.warn("Starting at block height: {}".format(height))
# Generate worker stats records - one per grin block for each active worker
while True:
# latest = grin.blocking_get_current_height()
latest = Blocks.get_latest().height
#LOGGER.warn("Latest Network Block Height = {}".format(latest))
while latest > height:
try:
new_stats = workerstats.calculate(height, avg_over_range)
LOGGER.warn("{} new stats for height {}".format(
len(new_stats), height))
# mark any existing pool_stats dirty
pool_stats = Pool_stats.get_by_height(height)
if pool_stats is not None:
LOGGER.warn(
"Marked existing pool_stats dirty for height: {}".
format(height))
pool_stats.dirty = True
database.db.getSession().bulk_save_objects(new_stats)
if ((height % BATCHSZ == 0) or (height >= (latest - 10))):
database.db.getSession().commit()
for stats in new_stats:
LOGGER.warn(
"Added Worker_stats for block: {}, Worker: {} - {} {} {} {} {} {}"
.format(stats.height, stats.worker, stats.gps,
stats.shares_processed,
stats.total_shares_processed, stats.grin_paid,
stats.total_grin_paid, stats.balance))
height = height + 1
except Exception as e:
LOGGER.error("Something went wrong: {}".format(e))
LOGGER.error("Traceback: {}".format(
traceback.format_exc().splitlines()))
database.db.getSession().rollback()
sleep(check_interval)
sys.stdout.flush()
sleep(check_interval)
LOGGER.warn("=== Completed {}".format(PROCESS))
def main():
global LOGGER
global CONFIG
CONFIG = lib.get_config()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
# Get Config settings
pool_fee = float(CONFIG[PROCESS]["pool_fee"])
# Number of blocks of share data used to calculate rewards
PPLNG_WINDOW_SIZE = 60
try:
PPLNG_WINDOW_SIZE = int(os.environ["PPLNG_WINDOW_SIZE"])
except Exception as e:
LOGGER.error(
"Failed to get PPLNG_WINDOW_SIZE from the environment: {} Using default size of {}"
.format(e, PPLNG_WINDOW_SIZE))
# Connect to DB
database = lib.get_db()
# Get current blockchain height
chain_height = grin.blocking_get_current_height()
# Get unlocked blocks from the db
unlocked_blocks = Pool_blocks.get_all_unlocked()
unlocked_blocks = [blk.height for blk in unlocked_blocks]
LOGGER.warn("Paying for {} pool blocks: {}".format(len(unlocked_blocks),
unlocked_blocks))
for height in unlocked_blocks:
try:
LOGGER.warn("Processing unlocked block: {}".format(height))
# Call the library routine to get this blocks payout map
payout_map = pool.calculate_block_payout_map(
height, PPLNG_WINDOW_SIZE, pool_fee, LOGGER, False)
#print("payout_map = {}".format(payout_map))
# Store the payment map for this block
credits_record = Pool_credits(chain_height, height, payout_map)
database.db.getSession().add(credits_record)
# Make payments based on the workers total share_value
Pool_blocks.setState(height, "paid")
for user_id, payment_amount in payout_map.items():
# Add worker rewards to pool account balance
LOGGER.warn("Credit to user: {} = {}".format(
user_id, payment_amount))
worker_utxo = Pool_utxo.credit_worker(user_id, payment_amount)
# Worker_stats accounting and running totals
#latest_worker_stats = Worker_stats.get_latest_by_id(user_id)
#latest_worker_stats.dirty = True
database.db.getSession().commit()
except Exception as e:
database.db.getSession().rollback()
LOGGER.exception("Something went wrong: {}".format(repr(e)))
LOGGER.warn("=== Completed {}".format(PROCESS))
sys.stdout.flush()
def RigDataCommitScheduler(max_lag, commit_interval, logger):
global RIGDATA_MUTEX
global RIGDATA_GROUPSIZE
global REDIS_RIGDATA_KEY
global REDIS_RIGDATA_EXPIRETIME
while True:
try:
database = lib.get_db()
latest_block = Blocks.get_latest()
while latest_block is None:
logger.warn("Cant get latest block from database")
time.sleep(10)
latest_block = Blocks.get_latest()
chain_height = latest_block.height
lib.teardown_db()
RIGDATA_MUTEX.acquire()
try:
logger.warn("= Begin RigDataCommitScheduler")
# Itterate over each rigdata cache key in redis
redisdb = lib.get_redis_db()
redis_key = "{}-*".format(REDIS_RIGDATA_KEY)
keys = []
for key in redisdb.scan_iter(match=redis_key, count=100):
keys.append(key.decode())
for key in sorted(keys):
share_height = int(key.split("-")[1])
if share_height < chain_height - RIGDATA_GROUPSIZE - max_lag:
# Commit this set of rigdata records
logger.warn(
"-- RigDataCommitScheduler processing record at height: {}"
.format(share_height))
redis_cached_rigdata = redisdb.get(key)
redis_cached_rigdata = json.loads(
redis_cached_rigdata.decode())
for user, rigdata in redis_cached_rigdata.items():
redis_key = "{}.{}.{}".format(
REDIS_RIGDATA_KEY, share_height, user)
if redisdb.exists(redis_key):
# XXX TODO
logger.warn(
"XXX TODO: DUPLICATE RIGDATA WORKER KEY - MERGE ???"
)
else:
redisdb.set(redis_key,
json.dumps(rigdata),
ex=REDIS_RIGDATA_EXPIRETIME)
# Wrote this rigdata to REDIS, so remove the cache record now
redisdb.delete(key)
finally:
RIGDATA_MUTEX.release()
logger.warn("= End RigDataCommitScheduler")
time.sleep(commit_interval)
except Exception as e:
logger.exception("Something went wrong: {}".format(
traceback.format_exc()))
time.sleep(10)
def main():
global LOGGER
global CONFIG
CONFIG = lib.get_config()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
# Connect to DB
database = lib.get_db()
grin_api_url = "http://" + CONFIG["grin_node"]["address"] + ":" + CONFIG["grin_node"]["api_port"]
status_url = grin_api_url + "/v1/status"
blocks_url = grin_api_url + "/v1/blocks/"
validation_depth = int(CONFIG[PROCESS]["validation_depth"])
response = requests.get(status_url)
latest = int(response.json()["tip"]["height"])
last = latest - validation_depth # start a reasonable distance back
if last < 0:
last = 1
LOGGER.warn("Starting from block #{}".format(last))
# last = 0
for i in range(last, latest):
url = blocks_url + str(i)
response = requests.get(url).json()
# print("{}: {}".format(response["header"]["height"], response["header"]["hash"]))
try:
rec = Blocks.get_by_height([i])
if rec is not None:
if rec.hash != response["header"]["hash"] and rec.state != "orphan":
LOGGER.warn("Found an orphan - height: {}, hash: {} vs {}".format(rec.height, rec.hash, response["header"]["hash"]))
rec.state = "orphan"
database.db.getSession().commit()
else:
LOGGER.warn("Adding missing block - height: {}".format(response["header"]["height"]))
# XXX TODO: Probably want to mark it as "missing" so we know it was filled in after the fact?
missing_block = Blocks(hash=response["header"]["hash"],
version=response["header"]["version"],
height = response["header"]["height"],
previous = response["header"]["previous"],
timestamp = response["header"]["timestamp"][:-1],
output_root = response["header"]["output_root"],
range_proof_root = response["header"]["range_proof_root"],
kernel_root = response["header"]["kernel_root"],
nonce = response["header"]["nonce"],
total_difficulty = response["header"]["total_difficulty"],
total_kernel_offset = response["header"]["total_kernel_offset"],
state = "missing")
database.db.createDataObj(missing_block)
except Exception as e:
# XXX TODO: Something more ?
LOGGER.error("Something went wrong: {}".format(e))
sys.stdout.flush()
# db.set_last_run(PROCESS, str(time.time()))
database.db.getSession().commit()
def initialize():
database = lib.get_db()
# Special case for new pool startup
seed_stat = Pool_stats(height=0,
timestamp=datetime.utcnow(),
gps=0,
active_miners=0,
shares_processed=0,
total_shares_processed=0,
total_grin_paid=0,
total_blocks_found=0)
database.db.createDataObj(seed_stat)
def main():
global LOGGER
global CONFIG
CONFIG = lib.get_config()
LOGGER = lib.get_logger(PROCESS)
database = lib.get_db()
LOGGER.warn("=== Starting {}".format(PROCESS))
check_interval = float(CONFIG[PROCESS]["check_interval"])
max_rebuild_depth = float(CONFIG[PROCESS]["max_rebuild_depth"])
avg_over_range_grin = int(CONFIG["grinStats"]["avg_over_range"])
avg_over_range_pool = int(CONFIG["poolStats"]["avg_over_range"])
avg_over_range_worker = int(CONFIG["workerStats"]["avg_over_range"])
current_height = grin.blocking_get_current_height()
rebuild_height = current_height - max_rebuild_depth
while True:
# Grin blocks and therefore grin stats cant be dirty
# # Check for dirty grin stats
# dirty = Grin_stats.get_first_dirty()
# if dirty is not None:
# LOGGER.warn("Recalculating Grin Stats from {}".format(dirty.height))
# end_height = grinstats.recalculate(dirty.height, avg_over_range_grin)
# LOGGER.warn("Finished Recalculating Grin Stats: {} - {}".format(dirty.height, end_height))
# Check for dirty pool stats
dirty = Pool_stats.get_first_dirty(rebuild_height)
if dirty is not None:
LOGGER.warn("Recalculating Pool Stats from {}".format(
dirty.height))
end_height = poolstats.recalculate(dirty.height,
avg_over_range_pool)
LOGGER.warn("Finished Recalculating Pool Stats: {} - {}".format(
dirty.height, end_height))
# # Check for dirty worker stats
dirty = Worker_stats.get_first_dirty(rebuild_height)
while dirty is not None:
LOGGER.warn("Recalculating Worker Stats for {} from {}".format(
dirty.height, avg_over_range_worker))
end_height = workerstats.recalculate(dirty.height,
avg_over_range_worker)
LOGGER.warn(
"Finished Recalculating Worker Stats for {} - {}".format(
dirty.height, end_height))
dirty = Worker_stats.get_first_dirty()
sys.stdout.flush()
time.sleep(check_interval)
LOGGER.warn("=== Completed {}".format(PROCESS))
def main():
global LOGGER
global CONFIG
global SHARES
global HEIGHT
global GRINSHARE_HEIGHT
global POOLSHARE_HEIGHT
global SHARE_EXPIRETIME
global database
global RABBITMQ_USER
global RABBITMQ_PASSWORD
CONFIG = lib.get_config()
atexit.register(lib.teardown_db)
GRINSHARE_HEIGHT = 0
POOLSHARE_HEIGHT = 0
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
SHARE_EXPIRETIME = int(CONFIG[PROCESS]["share_expire_time"])
commit_interval = int(CONFIG[PROCESS]["commit_interval"])
rmq_endpoints = json.loads(CONFIG[PROCESS]["rmq"])
RABBITMQ_USER = os.environ["RABBITMQ_USER"]
RABBITMQ_PASSWORD = os.environ["RABBITMQ_PASSWORD"]
database = lib.get_db()
HEIGHT = Worker_shares.get_latest_height()
while HEIGHT is None:
LOGGER.warn("Waiting on the first grin block...")
time.sleep(5)
latest_block = Blocks.get_latest()
if latest_block is not None:
HEIGHT = latest_block.height
SHARES = WorkerShares(LOGGER)
##
# Start a thread to commit shares
commit_thread = threading.Thread(target=ShareCommitScheduler,
args=(
commit_interval,
database,
))
commit_thread.start()
##
# Start a pika consumer thread for each rabbit we want to consume from
for rmq in rmq_endpoints:
rmq_thread = threading.Thread(target=RmqConsumer, args=(rmq, ))
rmq_thread.start()
def initialize(avg_over_range, logger):
database = lib.get_db()
# Special case for new pool startup - Need 3 stats records to bootstrap
block_zero = None
while block_zero is None:
logger.warn("Waiting for the first block record in the database")
time.sleep(1)
block_zero = Blocks.get_earliest()
print("block_zero={}".format(block_zero))
height = block_zero.height
# Create avg_over_range dummy block records prior to block_zero
print("Create block filtters: {} - {}".format(height-avg_over_range, height))
for h in range(height-avg_over_range, height):
print("Creating fillter at height {}".format(h))
new_block = Blocks(hash = "x",
version = 0,
height = h,
previous = "x",
timestamp = datetime.utcnow(),
output_root = "x",
range_proof_root = "x",
kernel_root = "x",
nonce = 0,
edge_bits = 29,
total_difficulty = block_zero.total_difficulty,
secondary_scaling = 0,
num_inputs = 0,
num_outputs = 0,
num_kernels = 0,
fee = 0,
lock_height = 0,
total_kernel_offset = "x",
state = "filler")
database.db.getSession().add(new_block)
database.db.getSession().commit()
seed_stat0 = Grin_stats(
height=height-2,
timestamp=block_zero.timestamp,
difficulty=block_zero.total_difficulty)
database.db.createDataObj(seed_stat0)
seed_stat1 = Grin_stats(
height=height-1,
timestamp=block_zero.timestamp,
difficulty=block_zero.total_difficulty)
database.db.createDataObj(seed_stat1)
seed_stat2 = Grin_stats(
height=height,
timestamp=block_zero.timestamp,
difficulty=block_zero.total_difficulty)
database.db.createDataObj(seed_stat2)
return height
def main():
CONFIG = lib.get_config()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
# Connect to DB
database = lib.get_db()
# Get the list of pool_blocks that are
# old enough to unlock and
# are not orphan blocks
# XXX TODO: The node may not be synced, may need to wait?
block_locktime = int(CONFIG[PROCESS]["block_locktime"])
block_expiretime = int(CONFIG[PROCESS]["block_expiretime"])
LOGGER.warn("using locktime: {}, expiretime: {}".format(
block_locktime, block_expiretime))
latest = grin.blocking_get_current_height()
LOGGER.warn("Latest: {}".format(latest))
new_poolblocks = Pool_blocks.get_all_new()
for pb in new_poolblocks:
if pb.height < (latest - block_expiretime):
# Dont re-process very old blocks - protection against duplicate payouts.
LOGGER.error("Processed expired pool block at height: {}".format(
pb.height))
pb.state = "expired"
continue
response = grin.get_block_by_height(pb.height)
if response == None:
# Unknown. Leave as "new" for now and attempt to validate next run
LOGGER.error("Failed to get block {}".format(pb.height))
continue
if int(response["header"]["nonce"]) != int(pb.nonce):
LOGGER.warn("Processed orphan pool block at height: {}".format(
pb.height))
pb.state = "orphan"
continue
if pb.height < (latest - block_locktime):
# This block seems valid, and old enough to unlock
LOGGER.warn("Unlocking pool block at height: {}".format(pb.height))
pb.state = "unlocked"
sys.stdout.flush()
# db.set_last_run(PROCESS, str(time.time()))
database.db.getSession().commit()
LOGGER.warn("=== Completed {}".format(PROCESS))
sys.stdout.flush()
def recalculate(start_height, avg_range):
database = lib.get_db()
height = start_height
while height <= grin.blocking_get_current_height():
old_stats = Worker_stats.get_by_height(height)
new_stats = calculate(height, avg_range)
for old_stat in old_stats:
database.db.deleteDataObj(old_stat)
for stats in new_stats:
print("new/updated stats: {} ".format(stats))
database.db.getSession().add(stats)
if(height % BATCHSZ == 0):
database.db.getSession().commit()
height = height + 1
database.db.getSession().commit()
def get(self, height=0, range=None, fields=None):
database = lib.get_db()
fields = lib.fields_to_list(fields)
if height == 0:
height = grin.get_current_height()
if range == None:
stat = Pool_stats.get_by_height(height)
if stat is None:
return None
return stat.to_json(fields)
else:
stats = []
for stat in Pool_stats.get_by_height(height, range):
stats.append(stat.to_json(fields))
return stats
