def get(self, height=None, range=None, fields=None): LOGGER = lib.get_logger(PROCESS) debug and LOGGER.warn("PoolAPI_blocks get height:{}, range:{}, fields:{}".format(height, range, fields)) # Enforce range limit if range is not None: range = min(range, pool_blocks_range_limit) fields = lib.fields_to_list(fields) if height is None or height == 0: blocks = Pool_blocks.get_latest(range) else: blocks = Pool_blocks.get_by_height(height, range) if range == None: if blocks is None: return None return blocks.to_json(fields) else: bl = [] for block in blocks: bl = [block.to_json(fields)] + bl return bl
def get_blocks_found_data(num_blocks): ## # Returns data needed to create a *blocks found* chart over the past num_blocks history blocks_found_data = [] latest_blocks = Pool_blocks.get_last_n(num_blocks) for block in iter(latest_blocks): blockdata = {} blockdata["time"] = block.timestamp.strftime('%s') blockdata["height"] = block.height blocks_found_data.append(blockdata) return blocks_found_data
def calculate(height, avg_range): # Get the most recent pool data from which to generate the stats previous_stats_record = Pool_stats.get_by_height(height - 1) assert previous_stats_record is not None, "No previous stats record found" avg_over_first_grin_block = Blocks.get_by_height(max( height - avg_range, 1)) assert avg_over_first_grin_block is not None, "Missing grin block: {}".format( max(height - avg_range, 1)) grin_block = Blocks.get_by_height(height) assert grin_block is not None, "Missing grin block: {}".format(height) latest_worker_shares = Worker_shares.get_by_height(height) # If no shares are found for this height, we have 2 options: # 1) Assume the share data is *delayed* so dont create the stats record now # assert len(latest_worker_shares) > 0, "No worker shares found" # 2) If we want we can create the record without share data and then when shares are added later this record will be recalculated avg_over_worker_shares = Worker_shares.get_by_height(height, avg_range) # Calculate the stats data timestamp = grin_block.timestamp difficulty = POOL_MIN_DIFF # XXX TODO - enchance to support multiple difficulties gps = 0 active_miners = 0 shares_processed = 0 num_shares_in_range = 0 if len(avg_over_worker_shares) > 0: num_shares_in_range = sum( [shares.valid for shares in avg_over_worker_shares]) gps = grin.calculate_graph_rate(difficulty, avg_over_first_grin_block.timestamp, grin_block.timestamp, num_shares_in_range) print("XXX: difficulty={}, {}-{}, len={}".format( difficulty, avg_over_first_grin_block.timestamp, grin_block.timestamp, num_shares_in_range)) if latest_worker_shares is not None: active_miners = len(latest_worker_shares) # XXX NO, FIX THIS num_valid = sum([shares.valid for shares in latest_worker_shares]) num_invalid = sum([shares.invalid for shares in latest_worker_shares]) shares_processed = num_valid + num_invalid total_shares_processed = previous_stats_record.total_shares_processed + shares_processed total_grin_paid = previous_stats_record.total_grin_paid # XXX TODO total_blocks_found = previous_stats_record.total_blocks_found if Pool_blocks.get_by_height(height - 1) is not None: total_blocks_found = total_blocks_found + 1 return Pool_stats(height=height, timestamp=timestamp, gps=gps, active_miners=active_miners, shares_processed=shares_processed, total_shares_processed=total_shares_processed, total_grin_paid=total_grin_paid, total_blocks_found=total_blocks_found)
def calculate(height, window_size): # Get the most recent pool data from which to generate the stats previous_stats_record = Pool_stats.get_by_height(height - 1) assert previous_stats_record is not None, "No previous Pool_stats record found" grin_block = Blocks.get_by_height(height) assert grin_block is not None, "Missing grin block: {}".format(height) window = Worker_shares.get_by_height(height, window_size) # assert window[-1].height - window[0].height >= window_size, "Failed to get proper window size" # print("Sanity: window size: {} vs {}".format(window[-1].height - window[0].height, window_size)) # Calculate the stats data timestamp = grin_block.timestamp active_miners = len(list(set([s.user_id for s in window]))) print("active_miners = {}".format(active_miners)) # Keep track of share totals - sum counts of all share sizes submitted for this block num_shares_processed = 0 share_counts = {} for ws in Worker_shares.get_by_height(height): num_shares_processed += ws.num_shares() for size in ws.sizes(): size_str = "{}{}".format("C", size) if size_str not in share_counts: share_counts[size_str] = {"valid": 0, "invalid": 0, "stale": 0} share_counts[size_str] = { "valid": share_counts[size_str]["valid"] + ws.num_valid(size), "invalid": share_counts[size_str]["invalid"] + ws.num_invalid(size), "stale": share_counts[size_str]["stale"] + ws.num_stale(size) } print("num_shares_processed this block= {}".format(num_shares_processed)) total_shares_processed = previous_stats_record.total_shares_processed + num_shares_processed total_blocks_found = previous_stats_record.total_blocks_found # Caclulate estimated GPS for all sizes with shares submitted all_gps = estimate_gps_for_all_sizes(window) if Pool_blocks.get_by_height(height - 1) is not None: total_blocks_found = total_blocks_found + 1 new_stats = Pool_stats( height=height, timestamp=timestamp, active_miners=active_miners, share_counts=share_counts, shares_processed=num_shares_processed, total_blocks_found=total_blocks_found, total_shares_processed=total_shares_processed, dirty=False, ) print("all_gps for all pool workers") pp.pprint(all_gps) for gps_est in all_gps: gps_rec = Gps(edge_bits=gps_est[0], gps=gps_est[1]) new_stats.gps.append(gps_rec) sys.stdout.flush() return new_stats
def main(): global LOGGER LOGGER = lib.get_logger(PROCESS) LOGGER.warn("=== Starting {}".format(PROCESS)) # Connect to DB database = lib.get_db() # XXX All in one db transaction.... # Get unlocked blocks from the db unlocked_blocks = Pool_blocks.get_all_unlocked() unlocked_blocks = [blk.height for blk in unlocked_blocks] for height in unlocked_blocks: try: LOGGER.warn("Processing unlocked block: {}".format(height)) # Call the library routine to get this blocks payout map payout_map = pool.calculate_block_payout_map( height, PPLNS_WINDOW, LOGGER, False) #print("payout_map = {}".format(payout_map)) # Make payments based on the workers total share_value Pool_blocks.setState(height, "paid") database.db.getSession().commit() for user_id, payment_amount in payout_map.items(): # Add worker rewards to pool account balance LOGGER.warn("Credit to user: {} = {}".format( user_id, payment_amount)) worker_utxo = Pool_utxo.credit_worker(user_id, payment_amount) # Worker_stats accounting and running totals #latest_worker_stats = Worker_stats.get_latest_by_id(user_id) #latest_worker_stats.dirty = True database.db.getSession().commit() except Exception as e: database.db.getSession().rollback() LOGGER.error("Something went wrong: {} - {}".format( e, traceback.print_exc())) LOGGER.warn("=== Completed {}".format(PROCESS)) sys.stdout.flush()
def get(self, id, height=None, range=None, fields=None): global database #database = lib.get_db() fields = lib.fields_to_list(fields) # AUTH FILTER if id != g.user.id: response = jsonify( {'message': 'Not authorized to access data for other users'}) response.status_code = 403 return response if height is None or height == 0: blocks = Pool_blocks.get_latest(range, id) else: blocks = Pool_blocks.get_by_height(height, range, id) if range == None: if blocks is None: return None return blocks.to_json(fields, True) else: bl = [] for block in blocks: bl.append(block.to_json(fields, True)) return bl
def main(): CONFIG = lib.get_config() LOGGER = lib.get_logger(PROCESS) LOGGER.warn("=== Starting {}".format(PROCESS)) # Connect to DB database = lib.get_db() # Get the list of pool_blocks that are # old enough to unlock and # are not orphan blocks # XXX TODO: The node may not be synced, may need to wait? block_locktime = int(CONFIG[PROCESS]["block_locktime"]) block_expiretime = int(CONFIG[PROCESS]["block_expiretime"]) LOGGER.warn("using locktime: {}, expiretime: {}".format( block_locktime, block_expiretime)) latest = grin.blocking_get_current_height() LOGGER.warn("Latest: {}".format(latest)) new_poolblocks = Pool_blocks.get_all_new() for pb in new_poolblocks: if pb.height < (latest - block_expiretime): # Dont re-process very old blocks - protection against duplicate payouts. LOGGER.error("Processed expired pool block at height: {}".format( pb.height)) pb.state = "expired" continue response = grin.get_block_by_height(pb.height) if response == None: # Unknown. Leave as "new" for now and attempt to validate next run LOGGER.error("Failed to get block {}".format(pb.height)) continue if int(response["header"]["nonce"]) != int(pb.nonce): LOGGER.warn("Processed orphan pool block at height: {}".format( pb.height)) pb.state = "orphan" continue if pb.height < (latest - block_locktime): # This block seems valid, and old enough to unlock LOGGER.warn("Unlocking pool block at height: {}".format(pb.height)) pb.state = "unlocked" sys.stdout.flush() # db.set_last_run(PROCESS, str(time.time())) database.db.getSession().commit() LOGGER.warn("=== Completed {}".format(PROCESS)) sys.stdout.flush()
def main(): global LOGGER LOGGER = lib.get_logger(PROCESS) LOGGER.warn("=== Starting {}".format(PROCESS)) # Connect to DB database = lib.get_db() latest_block = 0 # XXX All in one db transaction.... # Get unlocked blocks from the db unlocked_blocks = Pool_blocks.get_all_unlocked() database.db.getSession().commit() for pb in unlocked_blocks: try: LOGGER.warn("Processing unlocked block: {}".format(pb)) if pb.height > latest_block: latest_block = pb.height # Get Worker_stats of this block to calculate reward for each worker worker_stats = Worker_stats.get_by_height(pb.height) # Calculate Payment info: if len(worker_stats) > 0: # Calcualte reward/share: # XXX TODO: Enhance # What algorithm to use? Maybe: https://slushpool.com/help/manual/rewards r_per_g = REWARD / sum([st.gps for st in worker_stats]) for stat in worker_stats: # Calculate reward worker_rewards = stat.gps * r_per_g # Add or create worker rewards worker_utxo = Pool_utxo.credit_worker( stat.worker, worker_rewards) LOGGER.warn("Credit to user: {} = {}".format( stat.worker, worker_rewards)) # Mark the pool_block state="paid" (maybe "processed" would be more accurate?) pb.state = "paid" database.db.getSession().commit() except Exception as e: database.db.getSession().rollback() LOGGER.error("Something went wrong: {}".format(e)) #database.db.getSession().commit() LOGGER.warn("=== Completed {}".format(PROCESS)) sys.stdout.flush()
def get_graph_rate_data(num_blocks): ## # Returns data needed to create a *graph rate* chart over the past num_blocks history graph_rate_data = [] latest_blocks = Pool_blocks.get_last_n(num_blocks) for i in range(0, num_blocks - 5): # rolling 5-block window ratedata = {} ts1 = latest_blocks[i].timestamp ts2 = latest_blocks[i + 5].timestamp difficulty = latest_blocks[i + 5].total_difficulty - latest_blocks[ i + 4].total_difficulty # XXX TODO: This isnt right gps = calculate_graph_rate(difficulty, ts1, ts2, 5) ratedata["height"] = latest_blocks[i + 5].height ratedata["gps"] = gps ratedata["timestamp"] = ts2.strftime('%s') graph_rate_data.append(ratedata) return graph_rate_data
def addPoolBlock(self, share): if share.is_valid == False: return new_pool_block = Pool_blocks(hash=share.hash, height=share.height, nonce=share.nonce, actual_difficulty=share.share_difficulty, net_difficulty=share.network_difficulty, timestamp=share.timestamp, found_by=share.found_by, state="new") duplicate = lib.get_db().db.createDataObj_ignore_duplicates( new_pool_block) if duplicate: self.LOGGER.warn("Failed to add duplicate Pool Block: {}".format( new_pool_block.height)) else: self.LOGGER.warn("Added Pool Block: {}".format( new_pool_block.height))
def get_stats(): ## # Get current Network Info as seen by our grin node stats_json = {} last_n = 10 # Look at shares from the last N blocks to calculate avg time between # Get the poolblocks from the DB latest_blocks = Pool_blocks.get_last_n(last_n) # Latest poolblock height stats_json["height"] = latest_blocks[-1].height # Latest poolblock hash stats_json["latest_hash"] = latest_blocks[-1].hash # Latest poolblock timestamp stats_json["latest_timestamp"] = latest_blocks[-1].timestamp.strftime('%s') # Avg time between poolblocks found_every = (latest_blocks[-1].timestamp - latest_blocks[0].timestamp).strftime('%s') stats_json["found_every"] = found_every # XXX TODO: # # Pool graph rate # stats_json["graph_rate"] = calculate_graph_rate(latest_difficulty, latest_blocks[0].timestamp, latest_blocks[-1].timestamp, last_n) return stats_json
def calculate(height, window_size): # Get the most recent pool data from which to generate the stats previous_stats_record = Pool_stats.get_by_height(height - 1) assert previous_stats_record is not None, "No previous Pool_stats record found" grin_block = Blocks.get_by_height(height) assert grin_block is not None, "Missing grin block: {}".format(height) window = Worker_shares.get_by_height(height, window_size) # Calculate the stats data timestamp = grin_block.timestamp active_miners = len(list(set([s.worker for s in window]))) print("active_miners = {}".format(active_miners)) # Keep track of share totals - sum counts of all share sizes submitted for this block shares_processed = 0 if len(window) > 0: shares_processed = window[-1].num_shares() print("shares_processed this block= {}".format(shares_processed)) total_shares_processed = previous_stats_record.total_shares_processed + shares_processed total_grin_paid = previous_stats_record.total_grin_paid # XXX TODO total_blocks_found = previous_stats_record.total_blocks_found # Caclulate estimated GPS for all sizes with shares submitted all_gps = estimate_gps_for_all_sizes(window) if Pool_blocks.get_by_height(height - 1) is not None: total_blocks_found = total_blocks_found + 1 new_stats = Pool_stats( height=height, timestamp=timestamp, active_miners=active_miners, shares_processed=shares_processed, total_blocks_found=total_blocks_found, total_shares_processed=total_shares_processed, total_grin_paid=total_grin_paid, dirty=False, ) print("all_gps for all pool workers") pp.pprint(all_gps) for gps_est in all_gps: gps_rec = Gps(edge_bits=gps_est[0], gps=gps_est[1]) new_stats.gps.append(gps_rec) sys.stdout.flush() return new_stats
def addPoolBlock(logger, timestamp, height, hash, found_by, serverid): global POOLBLOCK_MUTEX POOLBLOCK_MUTEX.acquire() database = lib.get_db() try: logger.warn( "Adding A PoolBlock: Timestamp: {}, ServerID: {}, Height: {}, Hash: {}" .format(timestamp, serverid, height, hash)) state = "new" this_block = Blocks.get_by_height(height) while this_block is None: this_block = Blocks.get_by_height(height) time.sleep(1) nonce = this_block.nonce actual_difficulty = grin.difficulty(this_block.hash, this_block.edge_bits, this_block.secondary_scaling) net_difficulty = grin.get_network_difficulty(height) # Create the DB record new_pool_block = Pool_blocks(hash=hash, height=height, nonce=nonce, actual_difficulty=actual_difficulty, net_difficulty=net_difficulty, timestamp=timestamp, found_by=found_by, state=state) duplicate = lib.get_db().db.createDataObj_ignore_duplicates( new_pool_block) if duplicate: logger.warn( "Failed to add duplicate Pool Block: {}".format(height)) else: logger.warn("Added Pool Block: {}".format(height)) finally: POOLBLOCK_MUTEX.release()
def main(): global LOGGER LOGGER = lib.get_logger(PROCESS) LOGGER.warn("=== Starting {}".format(PROCESS)) # Connect to DB database = lib.get_db() latest_block = 0 # XXX All in one db transaction.... # Get unlocked blocks from the db unlocked_blocks = Pool_blocks.get_all_unlocked() database.db.getSession().commit() for pb in unlocked_blocks: try: LOGGER.warn("Processing unlocked block: {}".format(pb)) if pb.height > latest_block: latest_block = pb.height # Get Worker_stats of this block + range to calculate reward for each worker worker_shares_window = Worker_shares.get_by_height(pb.height, PPLNS_WINDOW) print("worker_shares_window = {}".format(worker_shares_window)) # Calculate Payment info: if len(worker_shares_window) > 0: # Calcualte reward/share: # XXX TODO: Enhance # What algorithm to use? Maybe: https://slushpool.com/help/manual/rewards # For now, some variation on pplns # Sum up the number of each size share submitted by each user shares_count_map = {} for worker_shares_rec in worker_shares_window: if not worker_shares_rec.worker in shares_count_map: shares_count_map[worker_shares_rec.worker] = {} for pow_size in worker_shares_rec.sizes(): print("pow_size = {}".format(pow_size)) if not pow_size in shares_count_map[worker_shares_rec.worker]: shares_count_map[worker_shares_rec.worker][pow_size] = 0 shares_count_map[worker_shares_rec.worker][pow_size] += worker_shares_rec.num_valid(pow_size) print("Shares Count Map:") pp.pprint(shares_count_map) # Normalize and sum each workers shares to create a "share value" total_value = 0 for worker, worker_shares_count in shares_count_map.items(): print("worker: {}, worker_shares_count: {}".format(worker, worker_shares_count)) sizes = list(worker_shares_count.keys()) print("sizes: {}".format(sizes)) shares_count_map[worker]["value"] = 0 value = 0 for size, count in worker_shares_count.items(): if size == 29: value += float(count) * .33 else: value += float(count) total_value += value shares_count_map[worker]["value"] = value print("Worker {} value: {}".format(worker, value)) # Make payments based on the workers total share_value for worker, worker_shares_count in shares_count_map.items(): worker_rewards = REWARD * worker_shares_count["value"] / total_value # Add or create worker rewards worker_utxo = Pool_utxo.credit_worker(worker, worker_rewards) LOGGER.warn("Credit to user: {} = {}".format(worker, worker_rewards)) # Mark the pool_block state="paid" (maybe "processed" would be more accurate?) pb.state = "paid" database.db.getSession().commit() except Exception as e: database.db.getSession().rollback() LOGGER.error("Something went wrong: {} - {}".format(e, traceback.print_exc())) #database.db.getSession().commit() LOGGER.warn("=== Completed {}".format(PROCESS)) sys.stdout.flush()
def main(): CONFIG = lib.get_config() LOGGER = lib.get_logger(PROCESS) LOGGER.warn("=== Starting {}".format(PROCESS)) # Connect to DB database = lib.get_db() # Get the list of pool_blocks that are # old enough to unlock and # are not orphan blocks # XXX TODO: The node may not be synced, may need to wait? block_locktime = int(CONFIG[PROCESS]["block_locktime"]) block_expiretime = int(CONFIG[PROCESS]["block_expiretime"]) LOGGER.warn("using locktime: {}, expiretime: {}".format(block_locktime, block_expiretime)) latest = grin.blocking_get_current_height() LOGGER.warn("Latest: {}".format(latest)) # Get outputs form the wallet wallet_outputs = wallet.retrieve_outputs(refresh=True) wallet_outputs_map = wallet.outputs_to_map_by_height(wallet_outputs) wallet_output_heights = list(wallet_outputs_map.keys()) wallet_output_heights.sort() #print("wallet_output_heights = {}".format(wallet_output_heights)) new_poolblocks = Pool_blocks.get_all_new() for pb in new_poolblocks: if pb.height < (latest - block_expiretime): # Dont re-process very old blocks - protection against duplicate payouts. LOGGER.error("Processed expired pool block at height: {}".format(pb.height)) pb.state = "expired" continue response = grin.get_block_by_height(pb.height) # Check for unknown block if response == None: # Unknown. Leave as "new" for now and attempt to validate next run LOGGER.error("Failed to get block {}".format(pb.height)) continue # Check for orphans if int(response["header"]["nonce"]) != int(pb.nonce): LOGGER.warn("Processed orphan pool block at height: {}".format(pb.height)) pb.state = "orphan" continue # # Check that we have a coinbase output in the wallet for this block # if pb.height not in wallet_output_heights: # LOGGER.warn("Wallet has no output for pool block at height: {}".format(pb.height)) # pb.state = "no_wallet_output" # continue # Check if its old enough to be mature if pb.height < (latest - block_locktime): # This block seems valid, and old enough to unlock LOGGER.warn("Unlocking pool block at height: {}".format(pb.height)) pb.state = "unlocked" sys.stdout.flush() # db.set_last_run(PROCESS, str(time.time())) database.db.getSession().commit() LOGGER.warn("=== Completed {}".format(PROCESS)) sys.stdout.flush()
def get(self, id, height=None, range=None): LOGGER = lib.get_logger(PROCESS) if id != g.user.id: response = jsonify( {'message': 'Not authorized to access data for other users'}) response.status_code = 403 return response debug and LOGGER.warn("EstimateApi_payment get id:{} height:{}".format( id, height)) id_str = str(id) if height is None: # Immature Balance Estimate LOGGER.warn("Immature Balance Estimate") # Get a list of all new and unlocked blocks unlocked_blocks = Pool_blocks.get_all_unlocked() unlocked_blocks_h = [blk.height for blk in unlocked_blocks] #LOGGER.warn("EstimateApi_payment unlocked blocks: {}".format(unlocked_blocks)) new_blocks = Pool_blocks.get_all_new() new_blocks_h = [blk.height for blk in new_blocks] #LOGGER.warn("EstimateApi_payment new blocks: {}".format(new_blocks)) total = 0 for height in unlocked_blocks_h + new_blocks_h: debug and print("Estimate block at height: {}".format(height)) payout_map = pool.get_block_payout_map_estimate(height, LOGGER) if payout_map is not None and id_str in payout_map: total = total + payout_map[id_str] return {"immature": total} if type(height) == str: if height == "next": # Next block estimate debug and LOGGER.warn("Next block estimate") estimate = 0 payout_map = pool.get_block_payout_map_estimate(height, LOGGER) if payout_map is None: estimate = "TBD" elif id_str in payout_map: estimate = payout_map[id_str] else: estimate = 0 return {"next": estimate} else: response = jsonify({'message': 'Invalid Request'}) response.status_code = 400 return response # Block Reward estimate if range is None: # One specific block estimate debug and LOGGER.warn("One specific block estimate") estimate = 0 payout_map = pool.get_block_payout_map_estimate(height, LOGGER) if payout_map is not None: if id_str in payout_map.keys(): estimate = payout_map[id_str] else: estimate = 0 else: # Maybe this is a pool block but we didnt estimate it yet pb = Pool_blocks.get_by_height(height) if pb is not None: estimate = "TBD" str_height = str(height) return {str_height: estimate} # Range of pool block reward estimates debug and LOGGER.warn("Range of blocks estimate") # Enforce range limit range = min(range, pool_blocks_range_limit) # Get the list of pool block(s) heights if height == 0: blocks = Pool_blocks.get_latest(range) else: blocks = Pool_blocks.get_by_height(height, range) block_heights = [pb.height for pb in blocks] # Get estimates for each of the blocks estimates = {} for height in block_heights: estimate = 0 payout_map = pool.get_block_payout_map_estimate(height, LOGGER) if payout_map is None: estimate = "TBD" elif id_str in payout_map: estimate = payout_map[id_str] else: estimate = 0 str_height = str(height) estimates[str_height] = estimate return estimates
def calculate_block_payout_map(height, window_size, logger, estimate=False): global reward_estimate_mutex reward_estimate_mutex.acquire() try: if estimate == True: cached_map = get_block_payout_map_estimate(height, logger) if cached_map is not None: return cached_map # Get pool_block record and check block state #print("getting the pool bloch: {}".format(height)) #sys.stdout.flush() poolblock = Pool_blocks.get_by_height(height) if poolblock is None: return {} #print("The pool block {}".format(poolblock.to_json())) #sys.stdout.flush() if estimate == True: if poolblock.state != "new" and poolblock.state != "unlocked": return {} else: if poolblock.state != "unlocked": return {} # Get an array with share counts for each block in the window shares = Worker_shares.get_by_height(height, window_size) #print("share data in window = {}".format(shares)) #sys.stdout.flush() # Get total value of this block: reward + fees reward = get_reward_by_block(height) #print("Reward for block {} = {}".format(height, reward)) #sys.stdout.flush() # Get the "secondary_scaling" value for this block scale = get_scale_by_block(height) #print("Secondary Scaling value for block = {}".format(scale)) #sys.stdout.flush() # build a map of total shares of each size for each user shares_count_map = get_share_counts(shares) # DUMMY DATA # scale = 529 # shares_count_map = { # 1: {29: 50}, # 2: {29: 25, 31: 10}, # 3: {32: 5}, # } #print("Shares Count Map:") #sys.stdout.flush() #pp = pprint.PrettyPrinter(indent=4) #pp.pprint(shares_count_map) #sys.stdout.flush() # Calcualte total value of all shares total_value = calculate_total_share_value(shares_count_map, scale) #print("total share value in payment window: {}".format(total_value)) #sys.stdout.flush() block_payout_map = {} # For each user with shares in the window, calculate payout and add to block_payout_map for user_id, worker_shares_count in shares_count_map.items(): #print("xxx: {} {}".format(user_id, worker_shares_count)) #sys.stdout.flush() # Calcualte the total share value from this worker total_worker_value = calculate_total_share_value( {user_id: worker_shares_count}, scale) worker_payment = total_worker_value / total_value * reward #print("worker_payment: {}".format(worker_payment/1000000000)) #sys.stdout.flush() block_payout_map[user_id] = worker_payment #print("block_payout_map = {}".format(block_payout_map)) #sys.stdout.flush() if estimate == True: payout_estimate_map_key = "payout-estimate-for-block-" + str( height) try: # Estimates are cached in redis, save it there if we can redisdb = lib.get_redis_db() #redisdb.hmset(payout_estimate_map_key, json.dumps(block_payout_map)) redisdb.set(payout_estimate_map_key, pickle.dumps(block_payout_map)) except Exception as e: logger.warn( "block_payout_map cache insert failed: {} - {}".format( payout_estimate_map_key, repr(e))) except Exception as e: logger.error("Estimate went wrong: {} - {}".format( e, traceback.print_stack())) finally: reward_estimate_mutex.release() #logger.warn("calculate_map: {}".format(block_payout_map)) return block_payout_map
def calculate_block_payout_map(height, window_size, pool_fee, logger, estimate=False): block_payout_map = {} # Get the grinpool admin user ID for pool fee pool_admin_user_id = 1 # Total the payments for sanity check total_payments_this_block = 0 try: admin_user = os.environ["GRIN_POOL_ADMIN_USER"] pool_admin_user_id = Users.get_id_by_username(admin_user) logger.warn("Pool Fee goes to admin account with id={}".format( pool_admin_user_id)) except Exception as e: logger.warn( "We dont have Admin account info, using default id={}: {}".format( pool_admin_user_id, e)) # Create the payout map # Get pool_block record and check block state #print("getting the pool block: {}".format(height)) #sys.stdout.flush() if estimate == False: poolblock = Pool_blocks.get_by_height(height) if poolblock is None or poolblock.state != "unlocked": return {} #print("The pool block {}".format(poolblock.to_json())) #sys.stdout.flush() # Get total value of this block: reward + tx fees reward = get_reward_by_block(height) print("Reward for block {} = {}".format(height, reward)) sys.stdout.flush() # The Pools Fee the_pools_fee = reward * pool_fee block_payout_map[pool_admin_user_id] = the_pools_fee reward = reward - the_pools_fee logger.warn("Pool Fee = {}".format(block_payout_map)) # Get the "secondary_scaling" value for this block scale = get_scale_by_block(height) #print("Secondary Scaling value for block = {}".format(scale)) #sys.stdout.flush() # build a map of total shares of each size for each user shares_count_map = get_share_counts(height, window_size) # DUMMY DATA # scale = 529 # shares_count_map = { # 1: {29: 50}, # 2: {29: 25, 31: 10}, # 3: {32: 5}, # } #print("Shares Count Map:") #sys.stdout.flush() #pp = pprint.PrettyPrinter(indent=4) #pp.pprint(shares_count_map) #sys.stdout.flush() # Calcualte total value of all shares total_value = calculate_total_share_value(shares_count_map, scale) print("total share value in payment window: {}".format(total_value)) sys.stdout.flush() # For each user with shares in the window, calculate payout and add to block_payout_map for user_id, worker_shares_count in shares_count_map.items(): #print("xxx: {} {}".format(user_id, worker_shares_count)) #sys.stdout.flush() # Calcualte the total share value from this worker total_worker_value = calculate_total_share_value( {user_id: worker_shares_count}, scale) if total_value * reward > 0: worker_payment = int(total_worker_value / total_value * reward) else: # For next block estimate, there may be no shares submitted to the pool worker_payment = 0 total_payments_this_block += worker_payment print("worker_payment: {}".format(worker_payment / 1000000000)) sys.stdout.flush() if user_id in block_payout_map.keys(): block_payout_map[user_id] += worker_payment else: block_payout_map[user_id] = worker_payment logger.warn( "Total Grin Paid Out this block: {} + the_pools_fee: {} ".format( total_payments_this_block, the_pools_fee)) print("block_payout_map = {}".format(block_payout_map)) #sys.stdout.flush() #logger.warn("calculate_map: {}".format(block_payout_map)) return block_payout_map
def main(): global CONFIG global LOGGER CONFIG = lib.get_config() LOGGER = lib.get_logger(PROCESS) LOGGER.warn("=== Starting {}".format(PROCESS)) # Number of blocks of share data used to calculate rewards PPLNG_WINDOW_SIZE = 60 try: PPLNG_WINDOW_SIZE = int(os.environ["PPLNG_WINDOW_SIZE"]) except Exception as e: LOGGER.error( "Failed to get PPLNG_WINDOW_SIZE from the environment: {}. Using default size of {}" .format(e, PPLNG_WINDOW_SIZE)) POOL_FEE = 0.0075 try: POOL_FEE = float(CONFIG[PROCESS]["pool_fee"]) except Exception as e: LOGGER.error( "Failed to get POOL_FEE from the config: {}. Using default fee of {}" .format(e, POOL_FEE)) # Keep track of "next" block estimated next_height_estimated = 0 # Connect to DB database = lib.get_db() while True: # Generate pool block reward estimates for all new and unlocked blocks try: database.db.initializeSession() next_height = Blocks.get_latest( ).height - 5 # A recent height which all worker shares are available unlocked_blocks = Pool_blocks.get_all_unlocked() new_blocks = Pool_blocks.get_all_new() unlocked_blocks_h = [blk.height for blk in unlocked_blocks] new_blocks_h = [blk.height for blk in new_blocks] need_estimates = unlocked_blocks_h + new_blocks_h LOGGER.warn( "Will ensure estimate for blocks: {}".format(need_estimates)) redisdb = lib.get_redis_db() # Generate Estimate for height in need_estimates: if height > next_height: LOGGER.warn( "Delay estimate until we have recent shares availalbe for block: {}" .format(height)) else: LOGGER.warn("Ensure estimate for block: {}".format(height)) # Check if we already have an estimate cached payout_estimate_map_key = key_prefix + str(height) cached_map = redisdb.get(payout_estimate_map_key) if cached_map is None: # We dont have it cached, we need to calcualte it and cache it now payout_map = pool.calculate_block_payout_map( height, PPLNG_WINDOW_SIZE, POOL_FEE, LOGGER, True) payout_map_json = json.dumps(payout_map) redisdb.set(payout_estimate_map_key, payout_map_json, ex=cache_expire) LOGGER.warn( "Created estimate for block {} with key {}".format( height, payout_estimate_map_key)) else: LOGGER.warn( "There is an exiting estimate for block: {}". format(height)) # Generate estimate for "next" block LOGGER.warn( "Ensure estimate for next block: {}".format(next_height)) if next_height_estimated != next_height: payout_map = pool.calculate_block_payout_map( next_height, PPLNG_WINDOW_SIZE, POOL_FEE, LOGGER, True) payout_map_json = json.dumps(payout_map) payout_estimate_map_key = key_prefix + "next" redisdb.set(payout_estimate_map_key, payout_map_json, ex=cache_expire) next_height_estimated = next_height LOGGER.warn("Created estimate for block {} with key {}".format( next_height, payout_estimate_map_key)) else: LOGGER.warn( "There is an exiting next block estimate for : {}".format( next_height)) LOGGER.warn("Completed estimates") database.db.destroySession() # Flush debug print statements sys.stdout.flush() except Exception as e: # AssertionError as e: LOGGER.error("Something went wrong: {} - {}".format( e, traceback.format_exc())) database.db.destroySession() LOGGER.warn("=== Completed {}".format(PROCESS)) sleep(check_interval)
def calculate_block_payout_map(height, window_size, pool_fee, logger, estimate=False): block_payout_map = {} # Get the grinpool admin user ID for pool fee pool_admin_user_id = 1 # Total the payments for sanity check total_payments_this_block = 0 try: admin_user = os.environ["GRIN_POOL_ADMIN_USER"] pool_admin_user_id = Users.get_id_by_username(admin_user) logger.warn("Pool Fee goes to admin account with id={}".format(pool_admin_user_id)) except Exception as e: logger.warn("We dont have Admin account info, using default id={}: {}".format(pool_admin_user_id, e)) # Create the payout map try: if estimate == True: cached_map = get_block_payout_map_estimate(height, logger) if cached_map is not None: return cached_map # Get pool_block record and check block state print("getting the pool block: {}".format(height)) sys.stdout.flush() poolblock = Pool_blocks.get_by_height(height) if poolblock is None: return {} print("The pool block {}".format(poolblock.to_json())) sys.stdout.flush() if estimate == True: if poolblock.state != "new" and poolblock.state != "unlocked": return {} else: if poolblock.state != "unlocked": return {} # Get total value of this block: reward + tx fees reward = get_reward_by_block(height) print("Reward for block {} = {}".format(height, reward)) sys.stdout.flush() # The Pools Fee the_pools_fee = reward * pool_fee block_payout_map[pool_admin_user_id] = the_pools_fee reward = reward - the_pools_fee logger.warn("Pool Fee = {}".format(block_payout_map)) # Get the "secondary_scaling" value for this block scale = get_scale_by_block(height) print("Secondary Scaling value for block = {}".format(scale)) sys.stdout.flush() # build a map of total shares of each size for each user shares_count_map = get_share_counts(height, window_size) # DUMMY DATA # scale = 529 # shares_count_map = { # 1: {29: 50}, # 2: {29: 25, 31: 10}, # 3: {32: 5}, # } #print("Shares Count Map:") #sys.stdout.flush() #pp = pprint.PrettyPrinter(indent=4) #pp.pprint(shares_count_map) #sys.stdout.flush() # Calcualte total value of all shares total_value = calculate_total_share_value(shares_count_map, scale) print("total share value in payment window: {}".format(total_value)) sys.stdout.flush() # For each user with shares in the window, calculate payout and add to block_payout_map for user_id, worker_shares_count in shares_count_map.items(): print("xxx: {} {}".format(user_id, worker_shares_count)) sys.stdout.flush() # Calcualte the total share value from this worker total_worker_value = calculate_total_share_value({user_id:worker_shares_count}, scale) worker_payment = total_worker_value / total_value * reward total_payments_this_block += worker_payment print("worker_payment: {}".format(worker_payment/1000000000)) sys.stdout.flush() if user_id in block_payout_map.keys(): block_payout_map[user_id] += worker_payment else: block_payout_map[user_id] = worker_payment logger.warn("Total Grin Paid Out this block: {} + the_pools_fee: {} ".format(total_payments_this_block, the_pools_fee)) print("block_payout_map = {}".format(block_payout_map)) #sys.stdout.flush() if estimate == True: payout_estimate_map_key = "payout-estimate-for-block-" + str(height) try: # Estimates are cached in redis, save it there if we can redisdb = lib.get_redis_db() #redisdb.hmset(payout_estimate_map_key, json.dumps(block_payout_map)) redisdb.set(payout_estimate_map_key, pickle.dumps(block_payout_map)) except Exception as e: logger.warn("block_payout_map cache insert failed: {} - {}".format(payout_estimate_map_key, repr(e))) except Exception as e: logger.error("Estimate went wrong: {} - {}".format(e, traceback.print_exc(e))) raise e #logger.warn("calculate_map: {}".format(block_payout_map)) return block_payout_map
def get(self, height=None): LOGGER = lib.get_logger(PROCESS) debug and LOGGER.warn("PoolAPI_blocksCount get height:{}".format(height)) count = Pool_blocks.count(height) return count
def main(): global LOGGER global CONFIG CONFIG = lib.get_config() LOGGER = lib.get_logger(PROCESS) while True: try: LOGGER.warn("=== Starting {}".format(PROCESS)) # Connect to DB database = lib.get_db() # Get the prebious audit record to find its height previous_audit_record = Pool_audit.getLatest() if previous_audit_record is None: previous_audit_record = Pool_audit() database.db.createDataObj(previous_audit_record) # Create new pool audit record audit_record = Pool_audit() summary_info = wallet.retrieve_summary_info(refresh=True) # Set the height by wallet audit_record.height = int(summary_info["last_confirmed_height"]) # Set pool bock count audit_record.pool_blocks_count = Pool_blocks.count( audit_record.height) - Pool_blocks.count( previous_audit_record.height) # Audit pools liability vs equity audit_record.equity = int( summary_info["amount_currently_spendable"]) + int( summary_info["amount_awaiting_confirmation"]) audit_record.liability = Pool_utxo.get_liability() audit_record.balance = audit_record.equity - audit_record.liability # Add payouts value payments_made = Pool_payment.get_by_height( audit_record.height, audit_record.height - previous_audit_record.height) audit_record.payouts = sum( [payment.amount for payment in payments_made]) # Add payments value pool_credits = Pool_credits.get_by_height( audit_record.height, audit_record.height - previous_audit_record.height) total_credits = 0 if pool_credits is not None: for credit in pool_credits: credits_this_block = sum(credit.credits.values()) total_credits += credits_this_block print("credits_this_block: {}, total_credits: {}".format( credits_this_block, total_credits)) audit_record.payments = total_credits else: audit_record.payments = 0 # Add and Commit the audit record #LOGGER.warn("Create Audit Record: {}".format(json.dumps(audit_record))) database.db.createDataObj(audit_record) LOGGER.warn("=== Completed {}".format(PROCESS)) except Exception as e: lib.teardown_db() LOGGER.exception("Something went wrong: {} ".format( traceback.format_exc())) time.sleep(999)