def get(self, id=None, height=0, range=None, fields=None):
database = lib.get_db()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn(
"WorkerAPI_stats get id:{} height:{} range:{} fields:{}".format(
id, height, range, fields))
fields = lib.fields_to_list(fields)
if height == 0:
height = Blocks.get_latest().height
stats = []
if id is None:
for stat in Worker_stats.get_by_height(height, range):
#print("YYY: {}".format(stats))
stats.append(stat.to_json(fields))
return stats
else:
if range is None:
res = Worker_stats.get_by_height_and_id(id, height)
if res is None:
return "[]".to_json()
return res.to_json(fields)
else:
for stat in Worker_stats.get_by_height_and_id(
id, height, range):
stats.append(stat.to_json(fields))
return stats
def main():
global LOGGER
global CONFIG
CONFIG = lib.get_config()
LOGGER = lib.get_logger(PROCESS)
database = lib.get_db()
LOGGER.warn("=== Starting {}".format(PROCESS))
check_interval = float(CONFIG[PROCESS]["check_interval"])
max_rebuild_depth = float(CONFIG[PROCESS]["max_rebuild_depth"])
avg_over_range_grin = int(CONFIG["grinStats"]["avg_over_range"])
avg_over_range_pool = int(CONFIG["poolStats"]["avg_over_range"])
avg_over_range_worker = int(CONFIG["workerStats"]["avg_over_range"])
current_height = grin.blocking_get_current_height()
rebuild_height = current_height - max_rebuild_depth
while True:
# Grin blocks and therefore grin stats cant be dirty
# # Check for dirty grin stats
# dirty = Grin_stats.get_first_dirty()
# if dirty is not None:
# LOGGER.warn("Recalculating Grin Stats from {}".format(dirty.height))
# end_height = grinstats.recalculate(dirty.height, avg_over_range_grin)
# LOGGER.warn("Finished Recalculating Grin Stats: {} - {}".format(dirty.height, end_height))
# Check for dirty pool stats
dirty = Pool_stats.get_first_dirty(rebuild_height)
if dirty is not None:
LOGGER.warn("Recalculating Pool Stats from {}".format(
dirty.height))
end_height = poolstats.recalculate(dirty.height,
avg_over_range_pool)
LOGGER.warn("Finished Recalculating Pool Stats: {} - {}".format(
dirty.height, end_height))
# # Check for dirty worker stats
dirty = Worker_stats.get_first_dirty(rebuild_height)
while dirty is not None:
LOGGER.warn("Recalculating Worker Stats for {} from {}".format(
dirty.height, avg_over_range_worker))
end_height = workerstats.recalculate(dirty.height,
avg_over_range_worker)
LOGGER.warn(
"Finished Recalculating Worker Stats for {} - {}".format(
dirty.height, end_height))
dirty = Worker_stats.get_first_dirty()
sys.stdout.flush()
time.sleep(check_interval)
LOGGER.warn("=== Completed {}".format(PROCESS))
def main():
CONFIG = lib.get_config()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
# Connect to DB
database = lib.get_db()
# Get config
check_interval = float(CONFIG[PROCESS]["check_interval"])
avg_over_range = int(CONFIG[PROCESS]["avg_over_range"])
# Find the height of the latest stats record
last_height = 0
latest_stat = Worker_stats.get_latest()
if latest_stat != None:
last_height = latest_stat.height
height = last_height + 1
LOGGER.warn("Starting at block height: {}".format(height))
# Generate worker stats records - one per grin block for each active worker
while True:
# latest = grin.blocking_get_current_height()
latest = Blocks.get_latest().height
#LOGGER.warn("Latest Network Block Height = {}".format(latest))
while latest > height:
try:
new_stats = workerstats.calculate(height, avg_over_range)
LOGGER.warn("{} new stats for height {}".format(
len(new_stats), height))
# mark any existing pool_stats dirty
pool_stats = Pool_stats.get_by_height(height)
if pool_stats is not None:
LOGGER.warn(
"Marked existing pool_stats dirty for height: {}".
format(height))
pool_stats.dirty = True
database.db.getSession().bulk_save_objects(new_stats)
if ((height % BATCHSZ == 0) or (height >= (latest - 10))):
database.db.getSession().commit()
for stats in new_stats:
LOGGER.warn(
"Added Worker_stats for block: {}, Worker: {} - {} {} {} {} {} {}"
.format(stats.height, stats.worker, stats.gps,
stats.shares_processed,
stats.total_shares_processed, stats.grin_paid,
stats.total_grin_paid, stats.balance))
height = height + 1
except Exception as e:
LOGGER.error("Something went wrong: {}".format(e))
LOGGER.error("Traceback: {}".format(
traceback.format_exc().splitlines()))
database.db.getSession().rollback()
sleep(check_interval)
sys.stdout.flush()
sleep(check_interval)
LOGGER.warn("=== Completed {}".format(PROCESS))
def calculate(height, avg_range):
avg_over_first_grin_block = Blocks.get_by_height( max(height-avg_range, 1) )
assert avg_over_first_grin_block is not None, "Missing grin block: {}".format(max(height-avg_range, 1))
grin_block = Blocks.get_by_height(height)
assert grin_block is not None, "Missing grin block: {}".format(height)
# Get all workers share records for the current range of blocks
latest_worker_shares = Worker_shares.get_by_height(height)
# assert len(latest_worker_shares) != 0, "Missing worker shares record for height {}".format(height)
avg_over_worker_shares = Worker_shares.get_by_height(height, avg_range)
# Create a worker_stats for each user who submitted a share in this range
workers = list(set([share.worker for share in latest_worker_shares]))
new_stats = []
for worker in workers:
# Get this workers most recent worker_stats record (for running totals)
last_stat = Worker_stats.get_latest_by_id(worker)
if last_stat is None:
# A new worker
last_stat = Worker_stats(None, datetime.utcnow(), height-1, worker, 0, 0, 0, 0, 0, 0)
new_stats.append(last_stat)
# Calculate this workers stats data
timestamp = grin_block.timestamp
difficulty = POOL_MIN_DIFF # XXX TODO - enchance to support multiple difficulties
num_shares_in_range = sum([shares.valid for shares in avg_over_worker_shares if shares.worker == worker])
gps = grin.calculate_graph_rate(difficulty, avg_over_first_grin_block.timestamp, grin_block.timestamp, num_shares_in_range)
num_valid_this_block = [shares.valid for shares in latest_worker_shares if shares.worker == worker][0]
num_invalid_this_block = [shares.invalid for shares in latest_worker_shares if shares.worker == worker][0]
shares_processed = num_valid_this_block + num_invalid_this_block
# latest_worker_shares = [share for share in latest_pool_shares if share.found_by == worker]
#shares_processed = len(worker_shares_this_block)
total_shares_processed = last_stat.total_shares_processed + shares_processed
stats = Worker_stats(
id = None,
height = height,
timestamp = timestamp,
worker = worker,
gps = gps,
shares_processed = shares_processed,
total_shares_processed = total_shares_processed,
grin_paid = 123, # XXX TODO
total_grin_paid = 456, # XXX TODO
balance = 1) # XXX TODO
new_stats.append(stats)
return new_stats
def get(self, id=None, height=0, range=None, fields=None):
database = lib.get_db()
fields = lib.fields_to_list(fields)
if height == 0:
height = grin.get_current_height()
stats = []
if id == None:
for stat in Worker_stats.get_by_height(height, range):
stats.append(stat.to_json(fields))
return stats
else:
if range == None:
res = Worker_stats.get_by_height_and_id(id, height)
if res is None:
return res
return res.to_json(fields)
else:
for stat in Worker_stats.get_by_height_and_id(
id, height, range):
stats.append(stat.to_json(fields))
return stats
def get(self, id, height=0, range=None, fields=None):
global database
#database = lib.get_db()
LOGGER = lib.get_logger(PROCESS)
# AUTH FILTER
if id != g.user.id:
response = jsonify(
{'message': 'Not authorized to access data for other users'})
response.status_code = 403
return response
debug and LOGGER.warn(
"WorkerAPI_stats get id:{} height:{} range:{} fields:{}".format(
id, height, range, fields))
# Enforce range limit
if range is not None:
range = min(range, worker_stats_range_limit)
fields = lib.fields_to_list(fields)
res = None
if range is None:
# Getting a single record
if height == 0:
# Get the most recent stats for this user
res = Worker_stats.get_latest_by_id(id)
else:
res = Worker_stats.get_by_height_and_id(id, height)
if res is None:
return None
return res.to_json(fields)
else:
# Getting a range of records
if height == 0:
height = Blocks.get_latest().height
res = Worker_stats.get_by_height_and_id(id, height, range)
if res is None:
return None
stats = []
for stat in res:
stats.append(stat.to_json(fields))
return stats
def get(self, height=0, range=None, fields=None):
global database
#database = lib.get_db()
LOGGER = lib.get_logger(PROCESS)
debug and LOGGER.warn("WorkersAPI_stats get height:{} range:{} fields:{}".format(height, range, fields))
fields = lib.fields_to_list(fields)
stats = []
if height == 0:
height = Blocks.get_latest().height
for stat in Worker_stats.get_by_height(height, range):
# AUTH FILTER
if stat.user_id == ADMIN_ID:
stats.append(stat.to_json(fields))
return stats
def recalculate(start_height, avg_range):
database = lib.get_db()
height = start_height
while height <= grin.blocking_get_current_height():
old_stats = Worker_stats.get_by_height(height)
new_stats = calculate(height, avg_range)
for old_stat in old_stats:
database.db.deleteDataObj(old_stat)
for stats in new_stats:
print("new/updated stats: {} ".format(stats))
database.db.getSession().add(stats)
if(height % BATCHSZ == 0):
database.db.getSession().commit()
height = height + 1
database.db.getSession().commit()
def post(self):
global database
LOGGER = lib.get_logger(PROCESS)
username = None
password = None
try:
debug and print("json request = {}".format(request.form))
username = request.form.get('username')
password = request.form.get('password')
debug and LOGGER.warn("PoolAPI_users POST: user:{} password:{}".format(username, password))
except AttributeError as e:
LOGGER.warn("Missing username or password - {}".format(str(e)))
if username is None or password is None:
response = jsonify({ 'message': 'Missing arguments: username and pasword required' })
response.status_code = 400
return response
if username == "" or password == "":
response = jsonify({ 'message': 'Missing arguments: username and pasword required' })
response.status_code = 400
return response
if "." in username:
response = jsonify({ 'message': 'Invalid Username: May not contain "."' })
response.status_code = 400
return response
# Check if the username is taken
exists = Users.check_username_exists(username)
if exists:
debug and print("Failed to add - conflict with existing user = {}".format(username))
response = jsonify({ 'message': 'Conflict with existing account' })
response.status_code = 409
return response
# Create the users record
user_rec = Users.create(username, password)
if user_rec is None:
debug and print("Failed to add - unable to create a new user record")
response = jsonify({ 'message': 'System Error: Failed to create account' })
response.status_code = 500
return response
# initialize a worker_stats record for this user (previous block) so they get instance feedback on the UI
lb = Blocks.get_latest()
if lb is not None:
height = Blocks.get_latest().height
initial_stat = Worker_stats(datetime.utcnow(), height, user_rec.id)
database.db.createDataObj(initial_stat)
debug and print("Added user = {}".format(user_rec))
response = jsonify({ 'username': user_rec.username, 'id': user_rec.id })
response.status_code = 201
return response
def main():
global LOGGER
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
# Connect to DB
database = lib.get_db()
latest_block = 0
# XXX All in one db transaction....
# Get unlocked blocks from the db
unlocked_blocks = Pool_blocks.get_all_unlocked()
database.db.getSession().commit()
for pb in unlocked_blocks:
try:
LOGGER.warn("Processing unlocked block: {}".format(pb))
if pb.height > latest_block:
latest_block = pb.height
# Get Worker_stats of this block to calculate reward for each worker
worker_stats = Worker_stats.get_by_height(pb.height)
# Calculate Payment info:
if len(worker_stats) > 0:
# Calcualte reward/share:
# XXX TODO: Enhance
# What algorithm to use? Maybe: https://slushpool.com/help/manual/rewards
r_per_g = REWARD / sum([st.gps for st in worker_stats])
for stat in worker_stats:
# Calculate reward
worker_rewards = stat.gps * r_per_g
# Add or create worker rewards
worker_utxo = Pool_utxo.credit_worker(
stat.worker, worker_rewards)
LOGGER.warn("Credit to user: {} = {}".format(
stat.worker, worker_rewards))
# Mark the pool_block state="paid" (maybe "processed" would be more accurate?)
pb.state = "paid"
database.db.getSession().commit()
except Exception as e:
database.db.getSession().rollback()
LOGGER.error("Something went wrong: {}".format(e))
#database.db.getSession().commit()
LOGGER.warn("=== Completed {}".format(PROCESS))
sys.stdout.flush()
def recalculate(start_height, avg_range):
database = lib.get_db()
height = start_height
while height <= grin.blocking_get_current_height():
old_stats = Worker_stats.get_by_height(height)
new_stats = calculate(height, avg_range)
for old_stat in old_stats:
database.db.deleteDataObj(old_stat)
for stats in new_stats:
print("new/updated stats: {} ".format(stats))
worker = stats.worker
database.db.getSession().add(stats)
if (height % BATCHSZ == 0):
database.db.getSession().commit()
height = height + 1
# We updated one or more worker stats so we mark the Pool_stats dirty
stats_rec = Pool_stats.get_by_height(height)
if stats_rec is not None:
stats_rec.dirty = True
database.db.getSession().commit()
def main():
CONFIG = lib.get_config()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
# Connect to DB
database = lib.get_db()
# Get config
check_interval = float(CONFIG[PROCESS]["check_interval"])
avg_over_range = int(CONFIG[PROCESS]["avg_over_range"])
# Find the height of the latest stats record
last_height = 0
latest_stat = Worker_stats.get_latest()
if latest_stat != None:
last_height = latest_stat.height
else:
latest = Blocks.get_latest()
while latest is None:
LOGGER.warn("Waiting for the first block...")
sleep(10)
latest = Blocks.get_latest()
last_height = latest.height
height = last_height + 1
LOGGER.warn("Starting at block height: {}".format(height))
# Generate worker stats records - one per grin block for each active worker
while True:
# latest = grin.blocking_get_current_height()
latest = Blocks.get_latest().height
share_height = Worker_shares.get_latest_height()
while share_height is None:
LOGGER.warn("waiting for the first worker shares")
sleep(10)
share_height = Worker_shares.get_latest_height()
stats_height = height - 1
LOGGER.warn(
"Running: chain height: {}, share height: {} vs stats height: {}".
format(latest, share_height, stats_height))
while share_height > height:
try:
new_stats = workerstats.calculate(height, avg_over_range)
LOGGER.warn("{} new stats for height {}".format(
len(new_stats), height))
for stats in new_stats:
LOGGER.warn("Added Worker_stats: {}".format(stats))
# mark any existing pool_stats dirty
pool_stats = Pool_stats.get_by_height(height)
for stat_rec in new_stats:
database.db.getSession().add(stat_rec)
if pool_stats is not None:
LOGGER.warn(
"Marked existing pool_stats dirty for height: {}".
format(height))
pool_stats.dirty = True # Pool_stats need to be recalculated
if ((height % BATCHSZ == 0) or (height >= (latest - 10))):
LOGGER.warn("Commit ---")
database.db.getSession().commit()
height = height + 1
except Exception as e:
LOGGER.exception("Something went wrong: {}".format(e))
database.db.getSession().rollback()
sleep(check_interval)
sys.stdout.flush()
sleep(check_interval)
LOGGER.warn("=== Completed {}".format(PROCESS))
def calculate(height, window_size):
database = lib.get_db()
grin_block = Blocks.get_by_height(height)
assert grin_block is not None, "Missing grin block at height: {}".format(height)
# Get all Worker_share records in the estimation window
window = Worker_shares.get_by_height(height, window_size)
# Get list of all workers who submitted shares OR recvd a payment at this height
pmts = Pool_payment.get_by_height(height)
shares_workers = [share.user_id for share in window]
pmts_workers = [pmt.user_id for pmt in pmts]
workers = list(set(shares_workers + pmts_workers))
# Create a new Worker_stats record for each of these workers
print("Calcualte worker stats for height {}, workers {}".format(height, workers))
new_stats = []
for worker in workers:
# Get this workers most recent worker_stats record (for running totals)
last_stat = Worker_stats.get_latest_by_id(worker)
if last_stat is None:
# A new worker, initialize a last_stat for the previous block
last_stat = Worker_stats(
timestamp=datetime.utcnow(),
height=height-1,
user_id=worker)
new_stats.append(last_stat)
# Calculate this workers stats data
timestamp = grin_block.timestamp
# Caclulate estimated GPS for all sizes with shares submitted
all_gps = estimate_gps_for_all_sizes(worker, window)
# Keep track of share totals - sum counts of all share sizes submitted for this block
print("Looking up shares for height {} user {}".format(height, worker))
this_workers_shares_this_block = Worker_shares.get_by_height_and_id(height, worker)
print("XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX {}".format(this_workers_shares_this_block))
if this_workers_shares_this_block is None or len(this_workers_shares_this_block) == 0:
this_workers_valid_shares = 0
this_workers_invalid_shares = 0
this_workers_stale_shares = 0
else:
this_workers_shares_this_block = this_workers_shares_this_block[-1]
this_workers_valid_shares = this_workers_shares_this_block.num_valid()
this_workers_invalid_shares = this_workers_shares_this_block.num_invalid()
this_workers_stale_shares = this_workers_shares_this_block.num_stale()
this_workers_total_valid_shares = last_stat.total_valid_shares + this_workers_valid_shares
this_workers_total_invalid_shares = last_stat.total_invalid_shares + this_workers_invalid_shares
this_workers_total_stale_shares = last_stat.total_stale_shares + this_workers_stale_shares
# XXX PERFORAMCE = removed bulk insert to debug some other issue, need to put bulk insert back!!!
stats = Worker_stats(
height = height,
timestamp = timestamp,
user_id = worker,
valid_shares = this_workers_valid_shares,
invalid_shares = this_workers_invalid_shares,
stale_shares = this_workers_stale_shares,
total_valid_shares = this_workers_total_valid_shares,
total_invalid_shares = this_workers_total_invalid_shares,
total_stale_shares = this_workers_total_stale_shares,
)
database.db.getSession().add(stats)
database.db.getSession().commit()
#print("AAA: Created Worker_stats with id={}".format(stats.id))
# print("all_gps for worker {}:".format(worker))
# pp.pprint(all_gps)
for gps_est in all_gps:
gps_rec = Gps(
edge_bits = gps_est[0],
gps = gps_est[1],
)
stats.gps.append(gps_rec)
#print("AAA: Appended gps record to Worker_stats: {}".format(gps_rec))
# gps_rec.worker_stats_id = stats.id,
# database.db.getSession().add(gps_rec)
new_stats.append(stats)
database.db.getSession().add(stats)
database.db.getSession().commit()
sys.stdout.flush()
return new_stats
def calculate(height, window_size):
database = lib.get_db()
grin_block = Blocks.get_by_height(height)
assert grin_block is not None, "Missing grin block at height: {}".format(
height)
# Get all Worker_share records in the estimation window
window = Worker_shares.get_by_height(height, window_size)
# Get list of all workers who submitted shares in the window
workers = list(set([share.worker for share in window]))
# Create a new Worker_stats record for each of these workers
print("Calcualte worker stats for height {}, workers {}".format(
height, workers))
new_stats = []
for worker in workers:
# Get this workers most recent worker_stats record (for running totals)
last_stat = Worker_stats.get_latest_by_id(worker)
if last_stat is None:
# A new worker, initialize a last_stat for the previous block
last_stat = Worker_stats(None, datetime.utcnow(), height - 1,
worker, 0, 0, 0, 0, 0, 0)
new_stats.append(last_stat)
# Calculate this workers stats data
timestamp = grin_block.timestamp
# Caclulate estimated GPS for all sizes with shares submitted
all_gps = estimate_gps_for_all_sizes(worker, window)
# Keep track of share totals - sum counts of all share sizes submitted for this block
this_workers_shares = [ws for ws in window if ws.worker == worker]
num_shares_processed = this_workers_shares[-1].num_shares()
print("num_shares_processed={}".format(num_shares_processed))
total_shares_processed = last_stat.total_shares_processed + num_shares_processed
print("total_shares_processed={}".format(total_shares_processed))
# XXX PERFORAMCE = could not get bulk_insert to work...
stats = Worker_stats(
id=None,
height=height,
timestamp=timestamp,
worker=worker,
shares_processed=num_shares_processed,
total_shares_processed=total_shares_processed,
grin_paid=123, # XXX TODO
total_grin_paid=456, # XXX TODO
balance=1) # XXX TODO
database.db.getSession().add(stats)
database.db.getSession().commit()
#print("AAA: Created Worker_stats with id={}".format(stats.id))
# print("all_gps for worker {}:".format(worker))
# pp.pprint(all_gps)
for gps_est in all_gps:
gps_rec = Gps(
edge_bits=gps_est[0],
gps=gps_est[1],
)
stats.gps.append(gps_rec)
#print("AAA: Appended gps record to Worker_stats: {}".format(gps_rec))
# gps_rec.worker_stats_id = stats.id,
# database.db.getSession().add(gps_rec)
# new_stats.append(stats)
database.db.getSession().add(stats)
database.db.getSession().commit()
sys.stdout.flush()
return new_stats