def processDeltaPerf(conn, validator, hPoolId, currHourValDetails, prevHourValDetails, currentEpoch, currentTime, deltaInserts, deltaUpdates): logger.info("in processDeltaPerf") # if validator["status"] != hConstants.H_ELECTED: # return startTime = commonUtils.getCurrentTime() # logger.info("currentEpoch") # logger.info(currentEpoch) # logger.info("validator") # logger.info(validator) if not prevHourValDetails: prevHourValDetails = getLatestDataForVal(conn, hPoolId, constants.H_HOUR_MODE) commonUtils.logTimeDiff( startTime, "time spent after getting latest data for validator") askedToSignDelta = getGap(validator["lifetimePerf"]["lifetimeToSign"], prevHourValDetails, "lifetimeToSign") signedDelta = getGap(validator["lifetimePerf"]["lifetimeSigned"], prevHourValDetails, "lifetimeSigned") signPer = 0 if askedToSignDelta != 0: signPer = signedDelta * 100 / askedToSignDelta rewards = getGap(validator["lifetimePerf"]["totalRewards"], prevHourValDetails, "lifetimeRewards") # logger.info("askedToSignDelta - {}, signedDelta - {}, lifetimeSigned - {}, lifetimeToSign - {}, prevHourValDetails - {}".format( # askedToSignDelta, signedDelta, validator["lifetimePerf"]["lifetimeToSign"], # validator["lifetimePerf"]["lifetimeSigned"], prevHourValDetails)) currentData = { "signed": signedDelta, "askedToSign": askedToSignDelta, "signPer": signPer, "blsKeyCount": None, "rewards": rewards, "bidPerSeat": None, "syncTime": currentTime, "epochSyncTime": currentTime.strftime('%s') } # logger.info("currentData") # logger.info(currentData) commonUtils.logTimeDiff(startTime, "before creating pool perf record") if currHourValDetails: record = getUpdateHPoolPerf(validator, currHourValDetails, currentData) deltaUpdates.append(record) else: record = getCreateHPoolPerf(validator, hPoolId, constants.H_HOUR_MODE, currentData, currentEpoch) deltaInserts.append(record)
def listHPoolPerfByEpoch(conn, app, epochNumber, mode): sql = " select * " sql += " from " + tables.hpoolperf + " p " sql += " where p.epochNumber =%s and mode=%s " # logger.info(sql) startTime = commonUtils.getCurrentTime() data = dbUtil.listResultsWithConn(sql, conn, (epochNumber, mode)) commonUtils.logTimeDiff(startTime, "after processing listHPoolPerfByEpoch") return data
def listVersionStats(conn): startTime = commonUtils.getCurrentTime() nodeVersionSummary = listNodeVersionSummary(conn) commonUtils.logTimeDiff(startTime, "after nodeVersionSummary") nodeVersions = listNodeVersions(conn) commonUtils.logTimeDiff(startTime, "after nodeVersions") return dbUtil.combineResults2("nodeVersionSummary", nodeVersionSummary, "nodeVersions", nodeVersions)
def getAverageAprForCurrentEpoch(conn, app, epochNumber): sql = "select sum(totalStaked * apr)/sum(totalStaked) as averageApr " sql += " from " + tables.hpoolperf sql += " where mode=%s " sql += " and epochNumber=%s and elected='True' " args = (constants.H_EPOCH_MODE, epochNumber) # logger.info(sql) # logger.info(args) startTime = commonUtils.getCurrentTime() data = dbUtil.getSingleRecordNoJsonWithConn(sql, conn, args) commonUtils.logTimeDiff(startTime, "after processing getAverageAprForCurrentEpoch") return data["averageApr"]
def listCurrentValues(conn, epochNumber, hourTime): sql = "select * " sql += " from " + tables.hpoolperf sql += " where mode=%s " if epochNumber and hourTime: sql += " and epochNumber=%s and syncTime = date_format(%s,'%%Y-%%m-%%d %%H') " args = (constants.H_HOUR_MODE, epochNumber, hourTime) elif epochNumber: sql += " and epochNumber=%s " args = (constants.H_EPOCH_MODE, epochNumber) else: # sql += " and date_format(syncTime,'%%Y-%%m-%%d %%H')= date_format(%s,'%%Y-%%m-%%d %%H') " sql += " and syncTime = date_format(%s,'%%Y-%%m-%%d %%H') " args = (constants.H_HOUR_MODE, hourTime) # logger.info(sql) # logger.info(args) startTime = commonUtils.getCurrentTime() list = dbUtil.listResultsWithConn(sql, conn, args) commonUtils.logTimeDiff(startTime, "after processing listCurrentValues") return list
def listAddressEvents(conn, app, address): startTime = commonUtils.getCurrentTime() addressDetails = getAddressDetails(conn, app, address, None) commonUtils.logTimeDiff(startTime, "after getting address details") lastUpdated = commonUtils.getEventLastUpdated( conn, eventName.syncHarmonyDelegations) commonUtils.logTimeDiff(startTime, "after getting event last updated") addressEvents = harmonyEvents.listEventsByAddress(conn, address) commonUtils.logTimeDiff(startTime, "after getting event details") return dbUtil.combineResults3("addressDetails", addressDetails, "lastUpdated", lastUpdated, "data", addressEvents)
def listNetworkHistory(conn, app): startTime = commonUtils.getCurrentTime() sql = "select epoch as title, value1 as totalStake, " sql += " round(value1/1000000, 3) as totalStakeInMillions, " sql += " round(value2/1000000, 1) as circulatingSupplyInMillions, " sql += " value2 as circulatingSupply, " sql += " value3 as rewardRate, value4 as totalValidators, " sql += " value5 as btcPrice, value6 as medianRawStake, " sql += " round(value6/1000000, 1) as medianRawStakeInMillions, " sql += " value7 as uniqueDelegates, value8 as totalAddresses, " sql += " value9 as signRate " sql += " from " + tables.hhistory sql += " where dataType='Network' " sql += " order by epoch " # logger.info(sql) commonUtils.logTimeDiff(startTime, "after network stats") stats = dbUtil.listResultsWithConn(sql, conn) commonUtils.logTimeDiff(startTime, "after stats") coinStat = commonUtils.getCoinStat(conn, app) commonUtils.logTimeDiff(startTime, "after coinStat") txSummary = harmonyTransactionSync.listDailyTransactions(conn) commonUtils.logTimeDiff(startTime, "after tx summary") shardTxSummary = harmonyTransactionSync.listShardSummaryTransactions(conn) commonUtils.logTimeDiff(startTime, "after shard tx summary") # dailyShardTxSummary = harmonyTransactionSync.listDailyShardTransactions(conn) blockRate = harmonyShard.getBlockRateHistory(conn) commonUtils.logTimeDiff(startTime, "after block rate history") nodeVersionSummary = harmonyVersionSync.listNodeVersionSummary(conn) commonUtils.logTimeDiff(startTime, "after nodeVersionSummary") # logger.info("shardTxSummary: {}".format(shardTxSummary)) return dbUtil.combineResults6("data", stats, "coinStat", coinStat, "txSummary", txSummary, "shardTxSummary", shardTxSummary, "blockRate", blockRate, "nodeVersionSummary", nodeVersionSummary)
def getHPool(conn, app, hPoolId, more): startTime = commonUtils.getCurrentTime() coinStat = commonUtils.getCoinStat(conn, app) commonUtils.logTimeDiff(startTime, "after getting coinStat:") validator = getHPoolById(conn, hPoolId) commonUtils.logTimeDiff(startTime, "after getting validator details:") hourlyChartData = getPerfChartData(conn, hPoolId, True, constants.H_HOUR_MODE, more) commonUtils.logTimeDiff(startTime, "after getting hourlyChartData:") dailyChartData = getEpochChartData(conn, hPoolId, more, coinStat["currentEpoch"]) commonUtils.logTimeDiff(startTime, "after getting dailyChartData:") # events = harmonyEvents.listEventsByPoolId(conn, hPoolId, 10) events = [] commonUtils.logTimeDiff(startTime, "after getting events:") notification = commonUtils.getNotification(conn, app, hPoolId) commonUtils.logTimeDiff(startTime, "after getting notifications:") # logger.info(notification) return dbUtil.combineResults6("val", validator, "hourlyChartData", hourlyChartData, "dailyChartData", dailyChartData, "notification", notification, "coinStat", coinStat, "events", events)
def getAddressDisplayDetails(conn, app, address, alias, subType=None): startTime = commonUtils.getCurrentTime() # logger.info("subType : " + subType) if address is None and alias is not None: record = aliasUtils.getAddressFromAlias(conn, app, alias) commonUtils.logTimeDiff(startTime, "after getting address from alias") if not record: response = "'" + alias + "' - alias not found in server. Please enter the address, alias and use the 'Save Alias in Server' button to save the alias." return dbUtil.getResponse(dbUtil.jsondumps({"error": response})) address = record["address"] coinStats = dbUtil.getSingleRecordNoJsonWithConn( commonUtils.getCoinStatSql(), conn, app) commonUtils.logTimeDiff(startTime, "after getting coinstat") addressDetails = getAddressDetails(conn, app, address, alias) commonUtils.logTimeDiff(startTime, "after getting address details") lastUpdated = commonUtils.getEventLastUpdated( conn, eventName.syncHarmonyDelegations) commonUtils.logTimeDiff(startTime, "after getting event last updated") delegations = getDelegationDetails(conn, address) commonUtils.logTimeDiff(startTime, "after getting delegations details") smartStake = None if subType == "myAccount": # logger.info("checking if smart stake user") smartStake = isUsingSmartStake(delegations) commonUtils.logTimeDiff(startTime, "after getting is smart stake") rewards = harmonyEvents.getRewardsSummary(conn, address) commonUtils.logTimeDiff(startTime, "after getting rewards summary") stakeHistory = listAddressStakeHistoryChartData(conn, address, coinStats["currentEpoch"]) stakeHistory.sort(key=lambda item: item['epoch'], reverse=False) commonUtils.logTimeDiff(startTime, "after getting stake history chart") # addressEvents = harmonyEvents.listEventsByAddress(conn, address) addressEvents = [] commonUtils.logTimeDiff(startTime, "after getting event details") # logger.info("after calling reportData.listRewardComparisonReportData") return dbUtil.combineResults8("addressDetails", addressDetails, "lastUpdated", lastUpdated, "delegations", delegations, "rewards", rewards, "stakeHistory", stakeHistory, "events", addressEvents, "coinStats", coinStats, "smartStake", smartStake)
def syncValidators(conn, app, data, event): # logger.info("in harmony validator sync") startTime = commonUtils.getCurrentTime() startTimeAudit = datetime.datetime.now() # conn = dbUtil.getConnection() # currentTime = datetime.datetime.now() currentHour = datetime.datetime.now().replace(microsecond=0, second=0, minute=0) previousHour = currentHour - datetime.timedelta(hours=1) # logger.info("currentTime: {}".format(currentTime)) # logger.info("loading request") # body = json.loads(event["body"]) validators = data["allValidators"] stakingInfo = data["stakingInfo"] epochInfo = data["epochInfo"] currentEpoch = epochInfo["currentEpoch"] logger.info("total number of validators returned is: " + str(len(validators))) if len(validators) <= 100: logger.info( "number of validators returned is much less than what was expected. skipping the processing for now." ) return getResponse(jsondumps({"result": "successful"})) currentEpochValMap = getCurrentValMap(conn, currentEpoch, None) previousEpochValMap = getCurrentValMap(conn, currentEpoch - 1, None) currentHourValMap = getCurrentValMap(conn, currentEpoch, currentHour) previousHourValMap = getCurrentValMap(conn, None, previousHour) commonUtils.logTimeDiff(startTime, "before coinstats") currentCoinStat = commonUtils.getCoinStat(conn, app) commonUtils.logTimeDiff(startTime, "before getBlockCompletionFactor") completionFactor, enoughDataForCurrentApr = getBlockCompletionFactor( epochInfo, stakingInfo, currentCoinStat, currentEpoch) # logger.info("is enough data captured for current apr: {}".format(enoughDataForCurrentApr)) valMap = harmonyData.listHPoolsAsMap(conn) networkStake = getNetworkStake(validators) commonUtils.logTimeDiff( startTime, "time spent before starting validators processing") i = 0 deltaInserts, deltaUpdates = [], [] stakeWeights = [] for validator in validators: address = validator["address"] # logger.info("{} - processing validator: {}".format(i, address)) # logger.info(validator) i += 1 commonUtils.logTimeDiff(startTime, "processing validator #: {}".format(i)) blsKeyCount = len(validator["blsPublicKeys"]) optimalBlsKeyCount, bidPerSeat = getOptimalBlsKeyCount( stakingInfo, blsKeyCount, validator) commonUtils.logTimeDiff( startTime, "before processing syncValidator #: {}".format(i)) stakeWeight = getStakeWeight(validator, networkStake) stakeWeights.append(stakeWeight) dbValidator = None if address in valMap: dbValidator = valMap[address] hPoolId = syncValidator(conn, validator, blsKeyCount, optimalBlsKeyCount, bidPerSeat, stakeWeight, currentEpoch, dbValidator) commonUtils.logTimeDiff( startTime, "after processing syncValidator #: {}".format(i)) currEpochSummary = getValidatorDetails(currentEpochValMap, hPoolId) commonUtils.logTimeDiff( startTime, "after getting currEpochSummary #: {}".format(i)) prevEpochSummary = getValidatorDetails(previousEpochValMap, hPoolId) commonUtils.logTimeDiff( startTime, "after getting prevEpochSummary #: {}".format(i)) syncValidatorEpochSummary(conn, validator, blsKeyCount, bidPerSeat, hPoolId, currEpochSummary, prevEpochSummary, currentEpoch, completionFactor, enoughDataForCurrentApr) commonUtils.logTimeDiff( startTime, "after processing syncValidatorEpochSummary #: {}".format(i)) # logger.info("processing hourly data") currHourValDetails = getValidatorDetails(currentHourValMap, hPoolId) commonUtils.logTimeDiff( startTime, "after obtaining currHourValDetails #: {}".format(i)) prevHourValDetails = getValidatorDetails(previousHourValMap, hPoolId) # logger.info("current hour existing details") # logger.info(currHourValDetails) # logger.info("previous hour existing details") # logger.info(prevHourValDetails) # logger.info("previous hourly delta performance") commonUtils.logTimeDiff( startTime, "before delta perf - processing validator #: {}".format(i)) processDeltaPerf(conn, validator, hPoolId, currHourValDetails, prevHourValDetails, currentEpoch, currentHour, deltaInserts, deltaUpdates) commonUtils.logTimeDiff(startTime, "after processDeltaPerf #: {}".format(i)) # logger.info("after processing deltas") commonUtils.logTimeDiff(startTime, "before processing hpoolperf inserts and updates") batchCreateHPoolPerf(conn, deltaInserts) batchUpdateHPoolPerf(conn, deltaUpdates) conn.commit() # processDeltaPerf(conn, app, validators, currentHour) commonUtils.logTimeDiff(startTime, "before processing perf index") if enoughDataForCurrentApr: processPerfIndex(conn, app, epochInfo) conn.commit() # uniqueDelegates = len(delegates) commonUtils.logTimeDiff(startTime, "before processing perf index") processEpochPerf(conn, app, epochInfo, currentCoinStat, enoughDataForCurrentApr) commonUtils.logTimeDiff(startTime, "before update coinstat") valForNetworkHalt = getValCountForNetworkHalt(stakeWeights) updateCoinStats(conn, app, stakingInfo, epochInfo, len(validators), valForNetworkHalt) conn.commit() harmonyValEvents.generateValSyncEvents(conn, currentEpoch, enoughDataForCurrentApr) commonUtils.logTimeDiff(startTime, "before creating event and audit") auditUtils.createEvent(conn, app, eventName.syncHarmonyValidators) auditUtils.audit(conn, app, event, eventName.syncHarmonyValidators, "service", startTimeAudit) # conn.close() commonUtils.logTimeDiff(startTime, "total time spent") return getResponse(jsondumps({"result": "successful"}))