def processDeltaPerf(conn, validator, hPoolId, currHourValDetails,
prevHourValDetails, currentEpoch, currentTime,
deltaInserts, deltaUpdates):
logger.info("in processDeltaPerf")
# if validator["status"] != hConstants.H_ELECTED:
# return
startTime = commonUtils.getCurrentTime()
# logger.info("currentEpoch")
# logger.info(currentEpoch)
# logger.info("validator")
# logger.info(validator)
if not prevHourValDetails:
prevHourValDetails = getLatestDataForVal(conn, hPoolId,
constants.H_HOUR_MODE)
commonUtils.logTimeDiff(
startTime, "time spent after getting latest data for validator")
askedToSignDelta = getGap(validator["lifetimePerf"]["lifetimeToSign"],
prevHourValDetails, "lifetimeToSign")
signedDelta = getGap(validator["lifetimePerf"]["lifetimeSigned"],
prevHourValDetails, "lifetimeSigned")
signPer = 0
if askedToSignDelta != 0:
signPer = signedDelta * 100 / askedToSignDelta
rewards = getGap(validator["lifetimePerf"]["totalRewards"],
prevHourValDetails, "lifetimeRewards")
# logger.info("askedToSignDelta - {}, signedDelta - {}, lifetimeSigned - {}, lifetimeToSign - {}, prevHourValDetails - {}".format(
# askedToSignDelta, signedDelta, validator["lifetimePerf"]["lifetimeToSign"],
# validator["lifetimePerf"]["lifetimeSigned"], prevHourValDetails))
currentData = {
"signed": signedDelta,
"askedToSign": askedToSignDelta,
"signPer": signPer,
"blsKeyCount": None,
"rewards": rewards,
"bidPerSeat": None,
"syncTime": currentTime,
"epochSyncTime": currentTime.strftime('%s')
}
# logger.info("currentData")
# logger.info(currentData)
commonUtils.logTimeDiff(startTime, "before creating pool perf record")
if currHourValDetails:
record = getUpdateHPoolPerf(validator, currHourValDetails, currentData)
deltaUpdates.append(record)
else:
record = getCreateHPoolPerf(validator, hPoolId, constants.H_HOUR_MODE,
currentData, currentEpoch)
deltaInserts.append(record)
def listHPoolPerfByEpoch(conn, app, epochNumber, mode):
sql = " select * "
sql += " from " + tables.hpoolperf + " p "
sql += " where p.epochNumber =%s and mode=%s "
# logger.info(sql)
startTime = commonUtils.getCurrentTime()
data = dbUtil.listResultsWithConn(sql, conn, (epochNumber, mode))
commonUtils.logTimeDiff(startTime, "after processing listHPoolPerfByEpoch")
return data
def listVersionStats(conn):
startTime = commonUtils.getCurrentTime()
nodeVersionSummary = listNodeVersionSummary(conn)
commonUtils.logTimeDiff(startTime, "after nodeVersionSummary")
nodeVersions = listNodeVersions(conn)
commonUtils.logTimeDiff(startTime, "after nodeVersions")
return dbUtil.combineResults2("nodeVersionSummary", nodeVersionSummary,
"nodeVersions", nodeVersions)
def getAverageAprForCurrentEpoch(conn, app, epochNumber):
sql = "select sum(totalStaked * apr)/sum(totalStaked) as averageApr "
sql += " from " + tables.hpoolperf
sql += " where mode=%s "
sql += " and epochNumber=%s and elected='True' "
args = (constants.H_EPOCH_MODE, epochNumber)
# logger.info(sql)
# logger.info(args)
startTime = commonUtils.getCurrentTime()
data = dbUtil.getSingleRecordNoJsonWithConn(sql, conn, args)
commonUtils.logTimeDiff(startTime,
"after processing getAverageAprForCurrentEpoch")
return data["averageApr"]
def listCurrentValues(conn, epochNumber, hourTime):
sql = "select * "
sql += " from " + tables.hpoolperf
sql += " where mode=%s "
if epochNumber and hourTime:
sql += " and epochNumber=%s and syncTime = date_format(%s,'%%Y-%%m-%%d %%H') "
args = (constants.H_HOUR_MODE, epochNumber, hourTime)
elif epochNumber:
sql += " and epochNumber=%s "
args = (constants.H_EPOCH_MODE, epochNumber)
else:
# sql += " and date_format(syncTime,'%%Y-%%m-%%d %%H')= date_format(%s,'%%Y-%%m-%%d %%H') "
sql += " and syncTime = date_format(%s,'%%Y-%%m-%%d %%H') "
args = (constants.H_HOUR_MODE, hourTime)
# logger.info(sql)
# logger.info(args)
startTime = commonUtils.getCurrentTime()
list = dbUtil.listResultsWithConn(sql, conn, args)
commonUtils.logTimeDiff(startTime, "after processing listCurrentValues")
return list
def listAddressEvents(conn, app, address):
startTime = commonUtils.getCurrentTime()
addressDetails = getAddressDetails(conn, app, address, None)
commonUtils.logTimeDiff(startTime, "after getting address details")
lastUpdated = commonUtils.getEventLastUpdated(
conn, eventName.syncHarmonyDelegations)
commonUtils.logTimeDiff(startTime, "after getting event last updated")
addressEvents = harmonyEvents.listEventsByAddress(conn, address)
commonUtils.logTimeDiff(startTime, "after getting event details")
return dbUtil.combineResults3("addressDetails", addressDetails,
"lastUpdated", lastUpdated, "data",
addressEvents)
def listNetworkHistory(conn, app):
startTime = commonUtils.getCurrentTime()
sql = "select epoch as title, value1 as totalStake, "
sql += " round(value1/1000000, 3) as totalStakeInMillions, "
sql += " round(value2/1000000, 1) as circulatingSupplyInMillions, "
sql += " value2 as circulatingSupply, "
sql += " value3 as rewardRate, value4 as totalValidators, "
sql += " value5 as btcPrice, value6 as medianRawStake, "
sql += " round(value6/1000000, 1) as medianRawStakeInMillions, "
sql += " value7 as uniqueDelegates, value8 as totalAddresses, "
sql += " value9 as signRate "
sql += " from " + tables.hhistory
sql += " where dataType='Network' "
sql += " order by epoch "
# logger.info(sql)
commonUtils.logTimeDiff(startTime, "after network stats")
stats = dbUtil.listResultsWithConn(sql, conn)
commonUtils.logTimeDiff(startTime, "after stats")
coinStat = commonUtils.getCoinStat(conn, app)
commonUtils.logTimeDiff(startTime, "after coinStat")
txSummary = harmonyTransactionSync.listDailyTransactions(conn)
commonUtils.logTimeDiff(startTime, "after tx summary")
shardTxSummary = harmonyTransactionSync.listShardSummaryTransactions(conn)
commonUtils.logTimeDiff(startTime, "after shard tx summary")
# dailyShardTxSummary = harmonyTransactionSync.listDailyShardTransactions(conn)
blockRate = harmonyShard.getBlockRateHistory(conn)
commonUtils.logTimeDiff(startTime, "after block rate history")
nodeVersionSummary = harmonyVersionSync.listNodeVersionSummary(conn)
commonUtils.logTimeDiff(startTime, "after nodeVersionSummary")
# logger.info("shardTxSummary: {}".format(shardTxSummary))
return dbUtil.combineResults6("data", stats, "coinStat", coinStat,
"txSummary", txSummary, "shardTxSummary", shardTxSummary,
"blockRate", blockRate, "nodeVersionSummary", nodeVersionSummary)
def getHPool(conn, app, hPoolId, more):
startTime = commonUtils.getCurrentTime()
coinStat = commonUtils.getCoinStat(conn, app)
commonUtils.logTimeDiff(startTime, "after getting coinStat:")
validator = getHPoolById(conn, hPoolId)
commonUtils.logTimeDiff(startTime, "after getting validator details:")
hourlyChartData = getPerfChartData(conn, hPoolId, True,
constants.H_HOUR_MODE, more)
commonUtils.logTimeDiff(startTime, "after getting hourlyChartData:")
dailyChartData = getEpochChartData(conn, hPoolId, more,
coinStat["currentEpoch"])
commonUtils.logTimeDiff(startTime, "after getting dailyChartData:")
# events = harmonyEvents.listEventsByPoolId(conn, hPoolId, 10)
events = []
commonUtils.logTimeDiff(startTime, "after getting events:")
notification = commonUtils.getNotification(conn, app, hPoolId)
commonUtils.logTimeDiff(startTime, "after getting notifications:")
# logger.info(notification)
return dbUtil.combineResults6("val", validator, "hourlyChartData",
hourlyChartData, "dailyChartData",
dailyChartData, "notification", notification,
"coinStat", coinStat, "events", events)
def getAddressDisplayDetails(conn, app, address, alias, subType=None):
startTime = commonUtils.getCurrentTime()
# logger.info("subType : " + subType)
if address is None and alias is not None:
record = aliasUtils.getAddressFromAlias(conn, app, alias)
commonUtils.logTimeDiff(startTime, "after getting address from alias")
if not record:
response = "'" + alias + "' - alias not found in server. Please enter the address, alias and use the 'Save Alias in Server' button to save the alias."
return dbUtil.getResponse(dbUtil.jsondumps({"error": response}))
address = record["address"]
coinStats = dbUtil.getSingleRecordNoJsonWithConn(
commonUtils.getCoinStatSql(), conn, app)
commonUtils.logTimeDiff(startTime, "after getting coinstat")
addressDetails = getAddressDetails(conn, app, address, alias)
commonUtils.logTimeDiff(startTime, "after getting address details")
lastUpdated = commonUtils.getEventLastUpdated(
conn, eventName.syncHarmonyDelegations)
commonUtils.logTimeDiff(startTime, "after getting event last updated")
delegations = getDelegationDetails(conn, address)
commonUtils.logTimeDiff(startTime, "after getting delegations details")
smartStake = None
if subType == "myAccount":
# logger.info("checking if smart stake user")
smartStake = isUsingSmartStake(delegations)
commonUtils.logTimeDiff(startTime, "after getting is smart stake")
rewards = harmonyEvents.getRewardsSummary(conn, address)
commonUtils.logTimeDiff(startTime, "after getting rewards summary")
stakeHistory = listAddressStakeHistoryChartData(conn, address,
coinStats["currentEpoch"])
stakeHistory.sort(key=lambda item: item['epoch'], reverse=False)
commonUtils.logTimeDiff(startTime, "after getting stake history chart")
# addressEvents = harmonyEvents.listEventsByAddress(conn, address)
addressEvents = []
commonUtils.logTimeDiff(startTime, "after getting event details")
# logger.info("after calling reportData.listRewardComparisonReportData")
return dbUtil.combineResults8("addressDetails", addressDetails,
"lastUpdated", lastUpdated, "delegations",
delegations, "rewards", rewards,
"stakeHistory", stakeHistory, "events",
addressEvents, "coinStats", coinStats,
"smartStake", smartStake)
def syncValidators(conn, app, data, event):
# logger.info("in harmony validator sync")
startTime = commonUtils.getCurrentTime()
startTimeAudit = datetime.datetime.now()
# conn = dbUtil.getConnection()
# currentTime = datetime.datetime.now()
currentHour = datetime.datetime.now().replace(microsecond=0,
second=0,
minute=0)
previousHour = currentHour - datetime.timedelta(hours=1)
# logger.info("currentTime: {}".format(currentTime))
# logger.info("loading request")
# body = json.loads(event["body"])
validators = data["allValidators"]
stakingInfo = data["stakingInfo"]
epochInfo = data["epochInfo"]
currentEpoch = epochInfo["currentEpoch"]
logger.info("total number of validators returned is: " +
str(len(validators)))
if len(validators) <= 100:
logger.info(
"number of validators returned is much less than what was expected. skipping the processing for now."
)
return getResponse(jsondumps({"result": "successful"}))
currentEpochValMap = getCurrentValMap(conn, currentEpoch, None)
previousEpochValMap = getCurrentValMap(conn, currentEpoch - 1, None)
currentHourValMap = getCurrentValMap(conn, currentEpoch, currentHour)
previousHourValMap = getCurrentValMap(conn, None, previousHour)
commonUtils.logTimeDiff(startTime, "before coinstats")
currentCoinStat = commonUtils.getCoinStat(conn, app)
commonUtils.logTimeDiff(startTime, "before getBlockCompletionFactor")
completionFactor, enoughDataForCurrentApr = getBlockCompletionFactor(
epochInfo, stakingInfo, currentCoinStat, currentEpoch)
# logger.info("is enough data captured for current apr: {}".format(enoughDataForCurrentApr))
valMap = harmonyData.listHPoolsAsMap(conn)
networkStake = getNetworkStake(validators)
commonUtils.logTimeDiff(
startTime, "time spent before starting validators processing")
i = 0
deltaInserts, deltaUpdates = [], []
stakeWeights = []
for validator in validators:
address = validator["address"]
# logger.info("{} - processing validator: {}".format(i, address))
# logger.info(validator)
i += 1
commonUtils.logTimeDiff(startTime,
"processing validator #: {}".format(i))
blsKeyCount = len(validator["blsPublicKeys"])
optimalBlsKeyCount, bidPerSeat = getOptimalBlsKeyCount(
stakingInfo, blsKeyCount, validator)
commonUtils.logTimeDiff(
startTime, "before processing syncValidator #: {}".format(i))
stakeWeight = getStakeWeight(validator, networkStake)
stakeWeights.append(stakeWeight)
dbValidator = None
if address in valMap:
dbValidator = valMap[address]
hPoolId = syncValidator(conn, validator, blsKeyCount,
optimalBlsKeyCount, bidPerSeat, stakeWeight,
currentEpoch, dbValidator)
commonUtils.logTimeDiff(
startTime, "after processing syncValidator #: {}".format(i))
currEpochSummary = getValidatorDetails(currentEpochValMap, hPoolId)
commonUtils.logTimeDiff(
startTime, "after getting currEpochSummary #: {}".format(i))
prevEpochSummary = getValidatorDetails(previousEpochValMap, hPoolId)
commonUtils.logTimeDiff(
startTime, "after getting prevEpochSummary #: {}".format(i))
syncValidatorEpochSummary(conn, validator, blsKeyCount, bidPerSeat,
hPoolId, currEpochSummary, prevEpochSummary,
currentEpoch, completionFactor,
enoughDataForCurrentApr)
commonUtils.logTimeDiff(
startTime,
"after processing syncValidatorEpochSummary #: {}".format(i))
# logger.info("processing hourly data")
currHourValDetails = getValidatorDetails(currentHourValMap, hPoolId)
commonUtils.logTimeDiff(
startTime, "after obtaining currHourValDetails #: {}".format(i))
prevHourValDetails = getValidatorDetails(previousHourValMap, hPoolId)
# logger.info("current hour existing details")
# logger.info(currHourValDetails)
# logger.info("previous hour existing details")
# logger.info(prevHourValDetails)
# logger.info("previous hourly delta performance")
commonUtils.logTimeDiff(
startTime,
"before delta perf - processing validator #: {}".format(i))
processDeltaPerf(conn, validator, hPoolId, currHourValDetails,
prevHourValDetails, currentEpoch, currentHour,
deltaInserts, deltaUpdates)
commonUtils.logTimeDiff(startTime,
"after processDeltaPerf #: {}".format(i))
# logger.info("after processing deltas")
commonUtils.logTimeDiff(startTime,
"before processing hpoolperf inserts and updates")
batchCreateHPoolPerf(conn, deltaInserts)
batchUpdateHPoolPerf(conn, deltaUpdates)
conn.commit()
# processDeltaPerf(conn, app, validators, currentHour)
commonUtils.logTimeDiff(startTime, "before processing perf index")
if enoughDataForCurrentApr:
processPerfIndex(conn, app, epochInfo)
conn.commit()
# uniqueDelegates = len(delegates)
commonUtils.logTimeDiff(startTime, "before processing perf index")
processEpochPerf(conn, app, epochInfo, currentCoinStat,
enoughDataForCurrentApr)
commonUtils.logTimeDiff(startTime, "before update coinstat")
valForNetworkHalt = getValCountForNetworkHalt(stakeWeights)
updateCoinStats(conn, app, stakingInfo, epochInfo, len(validators),
valForNetworkHalt)
conn.commit()
harmonyValEvents.generateValSyncEvents(conn, currentEpoch,
enoughDataForCurrentApr)
commonUtils.logTimeDiff(startTime, "before creating event and audit")
auditUtils.createEvent(conn, app, eventName.syncHarmonyValidators)
auditUtils.audit(conn, app, event, eventName.syncHarmonyValidators,
"service", startTimeAudit)
# conn.close()
commonUtils.logTimeDiff(startTime, "total time spent")
return getResponse(jsondumps({"result": "successful"}))