def main():
CONFIG = lib.get_config()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
##
# Update user records in MySQL and REDIS
database = lib.get_db()
database.db.initializeSession()
redisdb = lib.get_redis_db()
redis_userid_key = "userid."
id = 1
try:
while True:
thisuser = Users.get_by_id(id)
if thisuser is None:
if id > 2358:
LOGGER.warn("last id = {}".format(id))
break
id = id + 1
continue
if thisuser.username == "bjg62hj8byyksphuw95vqc3f74.lionm1":
orig_username = thisuser.username
thisuser.username = "******"
LOGGER.warn("Updated: {} to {}".format(orig_username,
thisuser.username))
if thisuser.username == "[email protected]_d1":
orig_username = thisuser.username
thisuser.username = "******"
LOGGER.warn("Updated: {} to {}".format(orig_username,
thisuser.username))
if thisuser.username == "*****@*****.**":
orig_username = thisuser.username
thisuser.username = "******"
LOGGER.warn("Updated: {} to {}".format(orig_username,
thisuser.username))
if "." in thisuser.username:
orig_username = thisuser.username
# Update mysql
thisuser.username = thisuser.username.replace(".", "_")
# Update redis
redis_key = redis_userid_key + orig_username
COMMIT and redisdb.delete(redis_key)
redis_key = redis_userid_key + thisuser.username
COMMIT and redisdb.set(redis_key, id)
LOGGER.warn("Updated: {} to {}".format(orig_username,
thisuser.username))
id = id + 1
except Exception as e: # AssertionError as e:
LOGGER.error("Something went wrong: {} - {}".format(
e, traceback.print_stack()))
COMMIT or LOGGER.warn("XXX No Commit - Edit for final run")
COMMIT and database.db.getSession().commit()
LOGGER.warn("=== Completed {}".format(PROCESS))
def RigDataCommitScheduler(max_lag, commit_interval, logger):
global RIGDATA_MUTEX
global RIGDATA_GROUPSIZE
global REDIS_RIGDATA_KEY
global REDIS_RIGDATA_EXPIRETIME
while True:
try:
database = lib.get_db()
latest_block = Blocks.get_latest()
while latest_block is None:
logger.warn("Cant get latest block from database")
time.sleep(10)
latest_block = Blocks.get_latest()
chain_height = latest_block.height
lib.teardown_db()
RIGDATA_MUTEX.acquire()
try:
logger.warn("= Begin RigDataCommitScheduler")
# Itterate over each rigdata cache key in redis
redisdb = lib.get_redis_db()
redis_key = "{}-*".format(REDIS_RIGDATA_KEY)
keys = []
for key in redisdb.scan_iter(match=redis_key, count=100):
keys.append(key.decode())
for key in sorted(keys):
share_height = int(key.split("-")[1])
if share_height < chain_height - RIGDATA_GROUPSIZE - max_lag:
# Commit this set of rigdata records
logger.warn(
"-- RigDataCommitScheduler processing record at height: {}"
.format(share_height))
redis_cached_rigdata = redisdb.get(key)
redis_cached_rigdata = json.loads(
redis_cached_rigdata.decode())
for user, rigdata in redis_cached_rigdata.items():
redis_key = "{}.{}.{}".format(
REDIS_RIGDATA_KEY, share_height, user)
if redisdb.exists(redis_key):
# XXX TODO
logger.warn(
"XXX TODO: DUPLICATE RIGDATA WORKER KEY - MERGE ???"
)
else:
redisdb.set(redis_key,
json.dumps(rigdata),
ex=REDIS_RIGDATA_EXPIRETIME)
# Wrote this rigdata to REDIS, so remove the cache record now
redisdb.delete(key)
finally:
RIGDATA_MUTEX.release()
logger.warn("= End RigDataCommitScheduler")
time.sleep(commit_interval)
except Exception as e:
logger.exception("Something went wrong: {}".format(
traceback.format_exc()))
time.sleep(10)
def get_block_payout_map_estimate(height, logger):
payout_estimate_map_key = "payout-estimate-for-block-" + str(height)
try:
# Estimates are cached in redis, get it from there if we can
redisdb = lib.get_redis_db()
cached_map = redisdb.get(payout_estimate_map_key)
sys.stdout.flush()
if cached_map is None:
return None
#logger.warn("The Map: {}".format(cached_map.decode('utf-8')))
return json.loads(cached_map.decode('utf-8'))
except Exception as e:
logger.warn("block_payout_map Lookup Error: {} - {}".format(
payout_estimate_map_key, repr(e)))
def get_block_payout_map_estimate(height, logger):
payout_estimate_map_key = "payout-estimate-for-block-" + str(height)
try:
# Estimates are cached in redis, get it from there if we can
redisdb = lib.get_redis_db()
cached_map = redisdb.get(payout_estimate_map_key)
#print("Get the pickled map: {}".format(cached_map))
sys.stdout.flush()
if cached_map is None:
return None
json_map = pickle.loads(cached_map)
return json_map
except Exception as e:
logger.warn("block_payout_map Lookup Error: {} - {}".format(payout_estimate_map_key, repr(e)))
def RigDataCommitScheduler(max_lag, logger):
global RIGDATA_MUTEX
global RIGDATA
global REDIS_RIGDATA_KEY
global REDIS_RIGDATA_EXPIRETIME
while True:
try:
redisdb = lib.get_redis_db()
while True:
database = lib.get_db()
chain_height = Blocks.get_latest().height
logger.warn(
"RIGDATA commit scheduler - chain_height = {}".format(
chain_height))
RIGDATA_MUTEX.acquire()
try:
for height in [
h for h in RIGDATA.keys()
if h < (chain_height - max_lag)
]:
logger.warn(
"Commit RIGDATA for height: {}".format(height))
# Picke RIGDATA and write to redis
for user, rigdata in RIGDATA[height].items():
key = "{}.{}.{}".format(REDIS_RIGDATA_KEY, height,
user)
if redisdb.exists(key):
logger.warn(
"XXX TODO - MERGE THIS ADDITIONAL SHARE DATA"
)
else:
redisdb.set(key,
json.dumps(rigdata),
ex=REDIS_RIGDATA_EXPIRETIME)
RIGDATA.pop(height, None)
finally:
RIGDATA_MUTEX.release()
lib.teardown_db()
time.sleep(30)
except Exception as e:
logger.error("Something went wrong: {}\n{}".format(
e,
traceback.format_exc().splitlines()))
lib.teardown_db()
time.sleep(10)
def ShareCommitScheduler(max_lag, commit_interval, logger):
global SHARES_MUTEX
while True:
try:
database = lib.get_db()
latest_block = Blocks.get_latest()
while latest_block is None:
logger.warn("Waiting for first block")
time.sleep(10)
latest_block = Blocks.get_latest()
chain_height = latest_block.height
SHARES_MUTEX.acquire()
try:
logger.warn("= Begin ShareCommitScheduler")
# Itterate over each sharedata key in redis
redisdb = lib.get_redis_db()
redis_key = "{}-*".format(REDIS_SHAREDATA_KEY)
keys = []
for key in redisdb.scan_iter(match=redis_key, count=100):
keys.append(key.decode())
for key in sorted(keys):
share_height = int(key.split("-")[1])
if share_height < chain_height - max_lag:
# Commit this record
logger.warn(
"-- ShareCommitScheduler processing record at height: {}"
.format(share_height))
redis_sharedata = redisdb.get(key)
redis_sharedata = json.loads(redis_sharedata.decode())
ts_str = redis_sharedata.pop("timestamp",
str(datetime.utcnow()))
ts = datetime.strptime(
ts_str.split(".")[0], '%Y-%m-%d %H:%M:%S')
for worker, worker_shares in redis_sharedata.items():
# Get any existing record
worker_shares_rec = Worker_shares.get_by_height_and_id(
share_height, worker)
if worker_shares_rec is None:
# No existing record for this worker at this height, create it
logger.warn(
"New share record for worker {} at height {}"
.format(worker, share_height))
worker_shares_rec = Worker_shares(
height=share_height,
user_id=worker,
timestamp=ts,
)
database.db.createDataObj(worker_shares_rec)
else:
logger.warn(
"Add to existing record for worker {} at height {}"
.format(worker, share_height))
for edge_bits, shares_count in worker_shares.items(
):
worker_shares_rec.add_shares(
edge_bits, shares_count["difficulty"],
shares_count["accepted"],
shares_count["rejected"],
shares_count["stale"])
# Debug
logger.warn("Worker Shares: {}".format(
worker_shares_rec))
# We wrote this record to mysql, so remove the redis cache
database.db.getSession().commit()
redisdb.delete(key)
# Write fillter record if needed
share_height = Worker_shares.get_latest_height()
if share_height is None:
share_height = grin.blocking_get_current_height()
share_height = share_height + 1
while share_height < (chain_height - max_lag):
logger.warn(
"Processed 0 shares in block {} - Creating filler record"
.format(share_height))
filler_worker_shares_rec = Worker_shares(
height=share_height,
user_id=1, # Pool User
timestamp=datetime.utcnow(),
)
database.db.createDataObj(filler_worker_shares_rec)
share_height += 1
finally:
database.db.getSession().commit()
SHARES_MUTEX.release()
lib.teardown_db()
logger.warn("= End ShareCommitScheduler")
time.sleep(commit_interval)
except Exception as e:
lib.teardown_db()
logger.exception("Something went wrong: {} ".format(
traceback.format_exc()))
time.sleep(10)
def addWorkerShares(logger, channel, delivery_tag, timestamp, height, worker,
workerid, rig_id, agent, difficulty, shares_data):
global debug
global SHARES_MUTEX
global SHARE_RE
global REDIS_SHAREDATA_KEY
global REDIS_SHAREDATA_EXPIRETIME
global RIGDATA_MUTEX
global REDIS_RIGDATA_KEY
global RIGDATA_GROUPSIZE
global REDIS_RIGDATA_CACHE_EXPIRETIME
logger.warn(
"New Worker Shares: Height: {}, Worker: {}, RigID: {}, Agent: {}, Difficulty: {}, SharesData: {}"
.format(height, worker, rig_id, agent, difficulty, shares_data))
# Parse the sharedata
# TEST: shares_data = "{29: Shares { edge_bits: 29, accepted: 7, rejected: 0, stale: 0 }, 31: Shares { edge_bits: 31, accepted: 4, rejected: 0, stale: 0 }}"
all_shares = SHARE_RE.findall(shares_data)
#logger.warn("addWorkerShares processing all_shares: {}".format(all_shares))
# God Damn REDIS - Convert all keys to strings
worker = str(worker)
rig_id = str(rig_id)
workerid = str(workerid)
for shares in all_shares:
debug and logger.warn("shares: {}".format(shares))
edge_bits, accepted, rejected, stale = shares
# Adjust for minimum share difficulty
accepted = int(accepted) * difficulty
rejected = int(rejected) * difficulty
stale = int(stale) * difficulty
logger.warn(
"New Worker Shares: Difficulty: {}, Accepted: {}, Rejected: {}, Stale: {}"
.format(difficulty, accepted, rejected, stale))
##
# Add the data to Aggregated counts of *all rigs* for this worker
SHARES_MUTEX.acquire()
try:
redisdb = lib.get_redis_db()
redis_key = "{}-{}".format(REDIS_SHAREDATA_KEY, height)
# Check for exisintg redis record
sharedata = redisdb.get(redis_key)
if sharedata is None:
# Create a new record
sharedata = {"timestamp": str(timestamp)}
else:
# Load existing record
sharedata = json.loads(sharedata.decode())
# Merge in this shares data
if worker not in sharedata:
sharedata[worker] = {}
if edge_bits not in sharedata[worker]:
sharedata[worker][edge_bits] = {
'difficulty': 1,
'accepted': 0,
'rejected': 0,
'stale': 0
}
sharedata[worker][edge_bits]['accepted'] += accepted
sharedata[worker][edge_bits]['rejected'] += rejected
sharedata[worker][edge_bits]['stale'] += stale
# Write the record to REDIS
logger.debug("Write to REDIS: {} - {}".format(
redis_key, sharedata))
redisdb.set(redis_key,
json.dumps(sharedata),
ex=REDIS_SHAREDATA_EXPIRETIME)
# Debug
logger.debug("Adding to sharedata cache: {}[{}][{}] = {}".format(
redis_key, worker, edge_bits, sharedata[worker][edge_bits]))
finally:
SHARES_MUTEX.release()
##
# Add the *individual rig* share data for each worker (into redis cache record)
RIGDATA_MUTEX.acquire()
group_height = int((height - height % RIGDATA_GROUPSIZE) +
RIGDATA_GROUPSIZE)
try:
redisdb = lib.get_redis_db()
redis_key = "{}-{}".format(REDIS_RIGDATA_KEY, group_height)
# Check for exisintg redis record
rigdata = redisdb.get(redis_key)
if rigdata is None:
# Create a new record
rigdata = {}
else:
# Load existing record
rigdata = json.loads(rigdata.decode())
logger.debug(
"Existing rigdata cache record: {}".format(rigdata))
if worker not in rigdata:
rigdata[worker] = {}
if rig_id not in rigdata[worker]:
rigdata[worker][rig_id] = {}
if workerid not in rigdata[worker][rig_id]:
rigdata[worker][rig_id][workerid] = {}
if edge_bits not in rigdata[worker][rig_id][workerid]:
logger.debug(
"New rigdata cache record: height {} group_height {} worker_id {}"
.format(height, group_height, worker))
rigdata[worker][rig_id][workerid][edge_bits] = {
'difficulty': 1,
'agent': agent,
'accepted': 0,
'rejected': 0,
'stale': 0
}
rigdata[worker][rig_id][workerid][edge_bits][
'accepted'] += accepted
rigdata[worker][rig_id][workerid][edge_bits][
'rejected'] += rejected
rigdata[worker][rig_id][workerid][edge_bits]['stale'] += stale
# Write the record to REDIS
logger.debug("Write to REDIS: {} - {}".format(redis_key, rigdata))
redisdb.set(redis_key,
json.dumps(rigdata),
ex=REDIS_RIGDATA_CACHE_EXPIRETIME)
logger.debug(
"Adding to rigdata cache: {}[{}][{}][{}][{}] = {}".format(
redis_key, group_height, worker, rig_id, workerid,
edge_bits, rigdata[worker][rig_id][workerid][edge_bits]))
finally:
RIGDATA_MUTEX.release()
def calculate_block_payout_map(height, window_size, logger, estimate=False):
global reward_estimate_mutex
reward_estimate_mutex.acquire()
try:
if estimate == True:
cached_map = get_block_payout_map_estimate(height, logger)
if cached_map is not None:
return cached_map
# Get pool_block record and check block state
#print("getting the pool bloch: {}".format(height))
#sys.stdout.flush()
poolblock = Pool_blocks.get_by_height(height)
if poolblock is None:
return {}
#print("The pool block {}".format(poolblock.to_json()))
#sys.stdout.flush()
if estimate == True:
if poolblock.state != "new" and poolblock.state != "unlocked":
return {}
else:
if poolblock.state != "unlocked":
return {}
# Get an array with share counts for each block in the window
shares = Worker_shares.get_by_height(height, window_size)
#print("share data in window = {}".format(shares))
#sys.stdout.flush()
# Get total value of this block: reward + fees
reward = get_reward_by_block(height)
#print("Reward for block {} = {}".format(height, reward))
#sys.stdout.flush()
# Get the "secondary_scaling" value for this block
scale = get_scale_by_block(height)
#print("Secondary Scaling value for block = {}".format(scale))
#sys.stdout.flush()
# build a map of total shares of each size for each user
shares_count_map = get_share_counts(shares)
# DUMMY DATA
# scale = 529
# shares_count_map = {
# 1: {29: 50},
# 2: {29: 25, 31: 10},
# 3: {32: 5},
# }
#print("Shares Count Map:")
#sys.stdout.flush()
#pp = pprint.PrettyPrinter(indent=4)
#pp.pprint(shares_count_map)
#sys.stdout.flush()
# Calcualte total value of all shares
total_value = calculate_total_share_value(shares_count_map, scale)
#print("total share value in payment window: {}".format(total_value))
#sys.stdout.flush()
block_payout_map = {}
# For each user with shares in the window, calculate payout and add to block_payout_map
for user_id, worker_shares_count in shares_count_map.items():
#print("xxx: {} {}".format(user_id, worker_shares_count))
#sys.stdout.flush()
# Calcualte the total share value from this worker
total_worker_value = calculate_total_share_value(
{user_id: worker_shares_count}, scale)
worker_payment = total_worker_value / total_value * reward
#print("worker_payment: {}".format(worker_payment/1000000000))
#sys.stdout.flush()
block_payout_map[user_id] = worker_payment
#print("block_payout_map = {}".format(block_payout_map))
#sys.stdout.flush()
if estimate == True:
payout_estimate_map_key = "payout-estimate-for-block-" + str(
height)
try:
# Estimates are cached in redis, save it there if we can
redisdb = lib.get_redis_db()
#redisdb.hmset(payout_estimate_map_key, json.dumps(block_payout_map))
redisdb.set(payout_estimate_map_key,
pickle.dumps(block_payout_map))
except Exception as e:
logger.warn(
"block_payout_map cache insert failed: {} - {}".format(
payout_estimate_map_key, repr(e)))
except Exception as e:
logger.error("Estimate went wrong: {} - {}".format(
e, traceback.print_stack()))
finally:
reward_estimate_mutex.release()
#logger.warn("calculate_map: {}".format(block_payout_map))
return block_payout_map
def main():
global CONFIG
global LOGGER
CONFIG = lib.get_config()
LOGGER = lib.get_logger(PROCESS)
LOGGER.warn("=== Starting {}".format(PROCESS))
# Number of blocks of share data used to calculate rewards
PPLNG_WINDOW_SIZE = 60
try:
PPLNG_WINDOW_SIZE = int(os.environ["PPLNG_WINDOW_SIZE"])
except Exception as e:
LOGGER.error(
"Failed to get PPLNG_WINDOW_SIZE from the environment: {}. Using default size of {}"
.format(e, PPLNG_WINDOW_SIZE))
POOL_FEE = 0.0075
try:
POOL_FEE = float(CONFIG[PROCESS]["pool_fee"])
except Exception as e:
LOGGER.error(
"Failed to get POOL_FEE from the config: {}. Using default fee of {}"
.format(e, POOL_FEE))
# Keep track of "next" block estimated
next_height_estimated = 0
# Connect to DB
database = lib.get_db()
while True:
# Generate pool block reward estimates for all new and unlocked blocks
try:
database.db.initializeSession()
next_height = Blocks.get_latest(
).height - 5 # A recent height which all worker shares are available
unlocked_blocks = Pool_blocks.get_all_unlocked()
new_blocks = Pool_blocks.get_all_new()
unlocked_blocks_h = [blk.height for blk in unlocked_blocks]
new_blocks_h = [blk.height for blk in new_blocks]
need_estimates = unlocked_blocks_h + new_blocks_h
LOGGER.warn(
"Will ensure estimate for blocks: {}".format(need_estimates))
redisdb = lib.get_redis_db()
# Generate Estimate
for height in need_estimates:
if height > next_height:
LOGGER.warn(
"Delay estimate until we have recent shares availalbe for block: {}"
.format(height))
else:
LOGGER.warn("Ensure estimate for block: {}".format(height))
# Check if we already have an estimate cached
payout_estimate_map_key = key_prefix + str(height)
cached_map = redisdb.get(payout_estimate_map_key)
if cached_map is None:
# We dont have it cached, we need to calcualte it and cache it now
payout_map = pool.calculate_block_payout_map(
height, PPLNG_WINDOW_SIZE, POOL_FEE, LOGGER, True)
payout_map_json = json.dumps(payout_map)
redisdb.set(payout_estimate_map_key,
payout_map_json,
ex=cache_expire)
LOGGER.warn(
"Created estimate for block {} with key {}".format(
height, payout_estimate_map_key))
else:
LOGGER.warn(
"There is an exiting estimate for block: {}".
format(height))
# Generate estimate for "next" block
LOGGER.warn(
"Ensure estimate for next block: {}".format(next_height))
if next_height_estimated != next_height:
payout_map = pool.calculate_block_payout_map(
next_height, PPLNG_WINDOW_SIZE, POOL_FEE, LOGGER, True)
payout_map_json = json.dumps(payout_map)
payout_estimate_map_key = key_prefix + "next"
redisdb.set(payout_estimate_map_key,
payout_map_json,
ex=cache_expire)
next_height_estimated = next_height
LOGGER.warn("Created estimate for block {} with key {}".format(
next_height, payout_estimate_map_key))
else:
LOGGER.warn(
"There is an exiting next block estimate for : {}".format(
next_height))
LOGGER.warn("Completed estimates")
database.db.destroySession()
# Flush debug print statements
sys.stdout.flush()
except Exception as e: # AssertionError as e:
LOGGER.error("Something went wrong: {} - {}".format(
e, traceback.format_exc()))
database.db.destroySession()
LOGGER.warn("=== Completed {}".format(PROCESS))
sleep(check_interval)
def calculate_block_payout_map(height, window_size, pool_fee, logger, estimate=False):
block_payout_map = {}
# Get the grinpool admin user ID for pool fee
pool_admin_user_id = 1
# Total the payments for sanity check
total_payments_this_block = 0
try:
admin_user = os.environ["GRIN_POOL_ADMIN_USER"]
pool_admin_user_id = Users.get_id_by_username(admin_user)
logger.warn("Pool Fee goes to admin account with id={}".format(pool_admin_user_id))
except Exception as e:
logger.warn("We dont have Admin account info, using default id={}: {}".format(pool_admin_user_id, e))
# Create the payout map
try:
if estimate == True:
cached_map = get_block_payout_map_estimate(height, logger)
if cached_map is not None:
return cached_map
# Get pool_block record and check block state
print("getting the pool block: {}".format(height))
sys.stdout.flush()
poolblock = Pool_blocks.get_by_height(height)
if poolblock is None:
return {}
print("The pool block {}".format(poolblock.to_json()))
sys.stdout.flush()
if estimate == True:
if poolblock.state != "new" and poolblock.state != "unlocked":
return {}
else:
if poolblock.state != "unlocked":
return {}
# Get total value of this block: reward + tx fees
reward = get_reward_by_block(height)
print("Reward for block {} = {}".format(height, reward))
sys.stdout.flush()
# The Pools Fee
the_pools_fee = reward * pool_fee
block_payout_map[pool_admin_user_id] = the_pools_fee
reward = reward - the_pools_fee
logger.warn("Pool Fee = {}".format(block_payout_map))
# Get the "secondary_scaling" value for this block
scale = get_scale_by_block(height)
print("Secondary Scaling value for block = {}".format(scale))
sys.stdout.flush()
# build a map of total shares of each size for each user
shares_count_map = get_share_counts(height, window_size)
# DUMMY DATA
# scale = 529
# shares_count_map = {
# 1: {29: 50},
# 2: {29: 25, 31: 10},
# 3: {32: 5},
# }
#print("Shares Count Map:")
#sys.stdout.flush()
#pp = pprint.PrettyPrinter(indent=4)
#pp.pprint(shares_count_map)
#sys.stdout.flush()
# Calcualte total value of all shares
total_value = calculate_total_share_value(shares_count_map, scale)
print("total share value in payment window: {}".format(total_value))
sys.stdout.flush()
# For each user with shares in the window, calculate payout and add to block_payout_map
for user_id, worker_shares_count in shares_count_map.items():
print("xxx: {} {}".format(user_id, worker_shares_count))
sys.stdout.flush()
# Calcualte the total share value from this worker
total_worker_value = calculate_total_share_value({user_id:worker_shares_count}, scale)
worker_payment = total_worker_value / total_value * reward
total_payments_this_block += worker_payment
print("worker_payment: {}".format(worker_payment/1000000000))
sys.stdout.flush()
if user_id in block_payout_map.keys():
block_payout_map[user_id] += worker_payment
else:
block_payout_map[user_id] = worker_payment
logger.warn("Total Grin Paid Out this block: {} + the_pools_fee: {} ".format(total_payments_this_block, the_pools_fee))
print("block_payout_map = {}".format(block_payout_map))
#sys.stdout.flush()
if estimate == True:
payout_estimate_map_key = "payout-estimate-for-block-" + str(height)
try:
# Estimates are cached in redis, save it there if we can
redisdb = lib.get_redis_db()
#redisdb.hmset(payout_estimate_map_key, json.dumps(block_payout_map))
redisdb.set(payout_estimate_map_key, pickle.dumps(block_payout_map))
except Exception as e:
logger.warn("block_payout_map cache insert failed: {} - {}".format(payout_estimate_map_key, repr(e)))
except Exception as e:
logger.error("Estimate went wrong: {} - {}".format(e, traceback.print_exc(e)))
raise e
#logger.warn("calculate_map: {}".format(block_payout_map))
return block_payout_map
