def getHPool(conn, app, hPoolId, more): startTime = commonUtils.getCurrentTime() coinStat = commonUtils.getCoinStat(conn, app) commonUtils.logTimeDiff(startTime, "after getting coinStat:") validator = getHPoolById(conn, hPoolId) commonUtils.logTimeDiff(startTime, "after getting validator details:") hourlyChartData = getPerfChartData(conn, hPoolId, True, constants.H_HOUR_MODE, more) commonUtils.logTimeDiff(startTime, "after getting hourlyChartData:") dailyChartData = getEpochChartData(conn, hPoolId, more, coinStat["currentEpoch"]) commonUtils.logTimeDiff(startTime, "after getting dailyChartData:") # events = harmonyEvents.listEventsByPoolId(conn, hPoolId, 10) events = [] commonUtils.logTimeDiff(startTime, "after getting events:") notification = commonUtils.getNotification(conn, app, hPoolId) commonUtils.logTimeDiff(startTime, "after getting notifications:") # logger.info(notification) return dbUtil.combineResults6("val", validator, "hourlyChartData", hourlyChartData, "dailyChartData", dailyChartData, "notification", notification, "coinStat", coinStat, "events", events)
def getEpochSummary(conn): coinStat = commonUtils.getCoinStat(conn, hTgBot.app) message = "Current Epoch: <b>" + str(coinStat["currentEpoch"]) + "</b>.\n" message += "Next epoch starts in: <b>" + str( convertEpochToMinHr(coinStat["nextEpochTime"])) + "</b>.\n" return message
def listRewardsHistory(conn, app, event, showHistory, startTime): # logger.info("in listRewardsHistory") address = event["queryStringParameters"]["address"] # logger.info("obtaining getRewardsDetails for address: {}".format(address)) history = getRewardsDetails(conn, address) # logger.info("obtaining getRewardsHistorySummary for address: {}".format(address)) historySummary = getRewardsHistorySummary(conn, address, '%%Y-%%m') # logger.info("obtaining getRewardsHistorySummary for address: {}".format(address)) historySummaryAnnual = getRewardsHistorySummary(conn, address, '%%Y') coinStat = commonUtils.getCoinStat(conn, app) data = harmonyAddress.getDelegationDetails(conn, address) smartStake = harmonyAddress.isUsingSmartStake(data) auditUtils.audit(conn, app, event, "rewardsHistory", "get", startTime, None, address=address) return dbUtil.combineResults6("history", history, "data", data, "coinStat", coinStat, "historySummary", historySummary, "historySummaryAnnual", historySummaryAnnual, "smartStake", smartStake)
def listPoolsSummary(conn, app): sql, args = listHPoolsSql(None) # logger.info(args) data = dbUtil.listResultsWithConn(sql, conn, args) # logger.info(data) coinStat = commonUtils.getCoinStat(conn, app) return dbUtil.combineResults2("data", data, "coinStat", coinStat)
def syncEvents(conn, app, data, startTime): # logger.info("syncing harmony events") poolMap = harmonyData.listHPoolsAsMap(conn) # startBlock = data["startBlockHeight"] endBlock = data["blockHeight"] events = data["events"] inserts = [] coinStat = commonUtils.getCoinStat(conn, app) for e in events: if "validatorAddress" in e and e["validatorAddress"] not in poolMap: # logger.info("the validator {} is not known. skipping event.".format(e["validatorAddress"])) continue eventType = e["type"] if eventType == hConstants.H_EVENT_DELEGATE or eventType == hConstants.H_EVENT_UNDELEGATE: record = (eventType, e["blockNumber"], e["txHash"], e["address"], poolMap[e["validatorAddress"]]["hPoolId"], e["amount"], e["epochTimestamp"], None) inserts.append(record) elif eventType == hConstants.H_EVENT_COLLECT_REWARDS: record = (eventType, e["blockNumber"], e["txHash"], e["address"], None, e["amount"], e["epochTimestamp"], None) inserts.append(record) elif eventType == hConstants.H_EVENT_EDIT_VALIDATOR: record = (eventType, e["blockNumber"], e["txHash"], e["address"], poolMap[e["validatorAddress"]]["hPoolId"], None, e["epochTimestamp"], None) inserts.append(record) specialRecords = processEditValDetails(e, poolMap, coinStat) if specialRecords: inserts.extend(specialRecords) elif eventType == hConstants.H_CREATE_VALIDATOR: record = (eventType, e["blockNumber"], e["txHash"], e["address"], poolMap[e["validatorAddress"]]["hPoolId"], None, e["epochTimestamp"], None) inserts.append(record) # hTgNotify.handleEvent(conn, e, poolMap) # logger.info("inserts are:") # logger.info(inserts) if len(inserts) > 0: batchCreateEvents(conn, inserts) auditUtils.createEvent(conn, app, hConstants.EVENT_LAST_SYNCED_EVENT_BLOCK_HEIGHT, endBlock) auditUtils.createEvent(conn, app, eventName.syncHarmonyEvents) # logger.info("finished processing all events") conn.commit()
def getRichList(conn, app, count): # logger.info(sql) sql = getRichListSql() data = dbUtil.listResultsWithConn(sql, conn, count) # logger.info(data) coinStat = commonUtils.getCoinStat(conn, app) lastUpdated = commonUtils.getEventLastUpdated( conn, eventName.syncHarmonyAddresses) return dbUtil.combineResults3("data", data, "lastUpdated", lastUpdated, "coinStat", coinStat)
def getEpoch(conn): coinStat = commonUtils.getCoinStat(conn, hTgBot.app) message = "Current Epoch: <b>" + str(coinStat["currentEpoch"]) + "</b>.\n" message += "Next epoch starts in: <b>" + str( convertEpochToMinHr(coinStat["nextEpochTime"])) + "</b>.\n" message += "Upper median raw stake: " + str( round(coinStat["medianRawStake"] * 1.15)) + ".\n" message += "Median raw stake: " + str(round( coinStat["medianRawStake"])) + ".\n" message += "Lower median raw stake: " + str( round(coinStat["medianRawStake"] * 0.85)) + ".\n" message += "/menu" return message
def listHPoolsApr(conn, app): sql = " select hPoolId as code, " sql += " concat(trim(replace(left(name, 29), '?','')), ' - ', status, ' - ', " sql += " (case when avgApr is null then 0 else avgApr end)) as description, " sql += " avgApr, avgNetApr, (fee * 100) as fee " sql += " from " + tables.hpool + " p " sql += " where status != %s " sql += " having description is not null " sql += " order by description " # logger.info(sql) data = dbUtil.listResultsWithConn(sql, conn, hConstants.H_NOT_ELIGIBLE) coinStat = commonUtils.getCoinStat(conn, app) return dbUtil.combineResults2("data", data, "coinStat", coinStat)
def getSignatureSummary(conn, hPoolId): coinStat = commonUtils.getCoinStat(conn, app) hourlyChartData = harmonyData.getPerfChartData(conn, hPoolId, True, constants.H_HOUR_MODE, 'false') message = "Hourly Performance:\n" message += getHourlyPerfSummary(hourlyChartData) epochChartData = harmonyData.getEpochChartData(conn, hPoolId, 'false', coinStat["currentEpoch"]) message += "\nEpoch Performance:\n" message += getEpochPerfSummary(epochChartData) return message
def listKeys(conn, app, event, hPoolId): # logger.info("in listKeys method") val = harmonyData.getHPoolById(conn, hPoolId) coinStat = commonUtils.getCoinStat(conn, app) epoch = coinStat["currentEpoch"] if "epoch" in event["queryStringParameters"]: epoch = int(event["queryStringParameters"]["epoch"]) data = listLatestKeys(conn, hPoolId, epoch) lastUpdated = commonUtils.getEventLastUpdated(conn, eventName.syncHarmonyBlsKeys) return dbUtil.combineResults5("val", val, "data", data, "epoch", epoch, "coinStat", coinStat, "lastUpdated", lastUpdated)
def processKeyNodeVersionAlert(conn, app): effectiveEpoch = propUtils.getIntPropValue( conn, app, hConstants.VERSION_ALERT_EPOCH_PROP) startingEpoch = effectiveEpoch - hConstants.VERSION_ALERT_RANGE endingEpoch = effectiveEpoch + hConstants.VERSION_ALERT_RANGE currentCoinStat = commonUtils.getCoinStat(conn, app) currentEpoch = currentCoinStat["currentEpoch"] logger.info( "effectiveEpoch: {}, startingEpoch: {}, endingEpoch: {}".format( effectiveEpoch, startingEpoch, endingEpoch)) if startingEpoch <= currentEpoch <= endingEpoch: version = propUtils.getPropValue( conn, app, hConstants.VERSION_ALERT_MANDATORY_PROP) logger.info("processing version alert") generateNodeVersionAlert(conn, version, effectiveEpoch, currentEpoch) else: logger.info("skipping version alert")
def listHPools(conn, app, status): # logger.info("listing harmony pools") sql, args = listHPoolsSql(status) # logger.info(args) data = dbUtil.listResultsWithConn(sql, conn, args) # logger.info(data) coinStat = commonUtils.getCoinStat(conn, app) statusSummary = getHStatusMap(conn) statusSummary[constants.ALL_FEE_INCREASE] = getFeeIncreaseCount(conn) # logger.info(statusSummary) lastUpdated = commonUtils.getEventLastUpdated( conn, eventName.syncHarmonyValidators) notification = commonUtils.getNotification(conn, app) return dbUtil.combineResults5("data", data, "lastUpdated", lastUpdated, "notification", notification, "statusSummary", statusSummary, "coinStat", coinStat)
def getRichDelegates(conn, app): sql = " select ad.ranking, pd.address, round(sum(pd.stake),0) as totalStake, " sql += " group_concat(p.name) as validators " sql += " from " + tables.hpool + " p " sql += " inner join " + tables.hpooldel + " pd on pd.hPoolId=p.hPoolId" sql += " inner join " + tables.haddress + " ad on ad.address=pd.address" sql += " where pd.stake > " + MIN_AMOUNT sql += " group by pd.address order by totalStake desc " sql += " limit 2000 " # logger.info(sql) data = dbUtil.listResultsWithConn(sql, conn) coinStat = commonUtils.getCoinStat(conn, app) lastUpdated = commonUtils.getEventLastUpdated( conn, eventName.syncHarmonyAddresses) return dbUtil.combineResults3("data", data, "lastUpdated", lastUpdated, "coinStat", coinStat)
def listNetworkHistory(conn, app): startTime = commonUtils.getCurrentTime() sql = "select epoch as title, value1 as totalStake, " sql += " round(value1/1000000, 3) as totalStakeInMillions, " sql += " round(value2/1000000, 1) as circulatingSupplyInMillions, " sql += " value2 as circulatingSupply, " sql += " value3 as rewardRate, value4 as totalValidators, " sql += " value5 as btcPrice, value6 as medianRawStake, " sql += " round(value6/1000000, 1) as medianRawStakeInMillions, " sql += " value7 as uniqueDelegates, value8 as totalAddresses, " sql += " value9 as signRate " sql += " from " + tables.hhistory sql += " where dataType='Network' " sql += " order by epoch " # logger.info(sql) commonUtils.logTimeDiff(startTime, "after network stats") stats = dbUtil.listResultsWithConn(sql, conn) commonUtils.logTimeDiff(startTime, "after stats") coinStat = commonUtils.getCoinStat(conn, app) commonUtils.logTimeDiff(startTime, "after coinStat") txSummary = harmonyTransactionSync.listDailyTransactions(conn) commonUtils.logTimeDiff(startTime, "after tx summary") shardTxSummary = harmonyTransactionSync.listShardSummaryTransactions(conn) commonUtils.logTimeDiff(startTime, "after shard tx summary") # dailyShardTxSummary = harmonyTransactionSync.listDailyShardTransactions(conn) blockRate = harmonyShard.getBlockRateHistory(conn) commonUtils.logTimeDiff(startTime, "after block rate history") nodeVersionSummary = harmonyVersionSync.listNodeVersionSummary(conn) commonUtils.logTimeDiff(startTime, "after nodeVersionSummary") # logger.info("shardTxSummary: {}".format(shardTxSummary)) return dbUtil.combineResults6("data", stats, "coinStat", coinStat, "txSummary", txSummary, "shardTxSummary", shardTxSummary, "blockRate", blockRate, "nodeVersionSummary", nodeVersionSummary)
def processKeyPerf(conn, perfData, currentEpoch, currentBlock): logger.info("processing key perf") if not perfData: return badKeyMap = notificationUtils.getKeyMapForBadPerfCheck( conn, currentEpoch, hConstants.NOTIFICATION_TYPE_KEY_PERF_BAD) goodKeyMap = notificationUtils.getKeyMapForGoodPerfCheck( conn, currentEpoch, hConstants.NOTIFICATION_TYPE_KEY_PERF_BAD) logger.info("processing perfData: {}".format(len(perfData))) keyPerfInserts, notifications, watchEndList = [], [], [] for keyPerf in perfData: # logger.info("inserting key perf: {}".format(keyPerf)) record = (keyPerf["blsKey"], keyPerf["epochNumber"], keyPerf["effectiveStake"], keyPerf["rawStake"], keyPerf["groupPercentStake"], keyPerf["overallPercentStake"], keyPerf["groupPercentReward"], keyPerf["overallPercentReward"], keyPerf["reward"], keyPerf["groupRewardRatio"], keyPerf["isBadPerf"], keyPerf["keyPerfIndex"], int(time.time())) keyPerfInserts.append(record) processNotificationsForBadPerf(keyPerf, badKeyMap, notifications) processNotificationsForGoodPerf(keyPerf, goodKeyMap, notifications, watchEndList) batchCreateKeyPerf(conn, keyPerfInserts) # logger.info("creating bls key perf notifications for: {}".format(notifications)) if len(notifications) > 0 or len(watchEndList) > 0: coinStat = commonUtils.getCoinStat(conn, app) # logger.info("shardDetails: {}".format(shardDetails)) # currentBlock = shardDetails["blockNumber"] if harmonyUtils.isEnoughDataForBlsKeys(coinStat, currentEpoch, currentBlock): logger.info("enough blocks have passed for bls key notifications") notificationUtils.batchCreateNotifications(conn, notifications) notificationUtils.updateEndWatchList(conn, watchEndList) conn.commit()
def updateAddressHistory(conn): coinStat = commonUtils.getCoinStat(conn, app) syncStatus = commonUtils.getSyncStatus(conn, app, constants.H_ADDRESS_HISTORY_SYNC) epoch = coinStat["currentEpoch"] # logger.info("addresses are synced till epoch: {}. checking with current epoch: {}".format( # syncStatus["syncedTillEpoch"], epoch)) if syncStatus["syncedTillEpoch"] == epoch: logger.info( "skipping. addresses are already synced till current epoch: {}". format(syncStatus["syncedTillEpoch"])) return False logger.info("syncing address history") sql = harmonyData.getRichListSql() # logger.info(sql) currentAddresses = dbUtil.listResultsWithConn(sql, conn, 10000000) # logger.info(currentAddresses) # logger.info("processing address history") inserts = [] for address in currentAddresses: # logger.info("processing address history for: ".format(address["address"])) record = harmonyHistory.getCreateHistoryData( epoch, constants.H_HISTORY_ADDRESS, address["totalStake"], None, None, address["address"], address["totalBalance"], address["totalRewards"]) inserts.append(record) harmonyHistory.batchCreateHistory(conn, inserts) # logger.info("after processing address history") commonUtils.updateSyncStatus(conn, app, constants.H_ADDRESS_HISTORY_SYNC, None, epoch) auditUtils.createEvent(conn, app, eventName.syncHarmonyAddressHistory) logger.info("after syncing address history") conn.commit() return True
def processElections(conn, app): validators = harmonyData.listAllValidators(conn) # logger.info(validators) valByAddMap = commonUtils.getMapFromList(validators, "address") data = commonUtils.getDataByUrl(hConstants.ELECTIONS_URL) currentEpoch = data["current_epoch"] currentBlock = data["current_block_number"] coinStat = commonUtils.getCoinStat(conn, constants.APP) timeLeft, eventToUpdate = getEpochEndTimeLeftForEpochAlert( conn, coinStat, currentEpoch, currentBlock) nextSlots = getSlots(data["live_table"], valByAddMap) processForWhatIf(nextSlots) nextEpochNotMap = notificationUtils.getNotMapForElectionCheck( conn, currentEpoch, True) notificationsNext, watchEndListNext = processNextEpochNotifications( nextSlots, nextEpochNotMap, currentEpoch, timeLeft) slots = getSlots(data["table"], valByAddMap) epochNotMap = notificationUtils.getNotMapForElectionCheck( conn, currentEpoch, False) notifications = processEpochNotifications(slots, epochNotMap, currentEpoch) notifications.extend(notificationsNext) notificationsUnelectedVal = processUnelectedValNotifications( conn, epochNotMap, currentEpoch) notifications.extend(notificationsUnelectedVal) # logger.info("notifications: {}, watchEndListNext: {}".format(notifications, watchEndListNext)) processForUnderbid(conn, currentEpoch, nextSlots, notifications, watchEndListNext, currentBlock, coinStat, timeLeft) if eventToUpdate: # logger.info("creating event: {}".format(eventToUpdate)) auditUtils.createEvent(conn, app, eventToUpdate, currentEpoch) notificationUtils.batchCreateNotifications(conn, notifications) notificationUtils.updateEndWatchList(conn, watchEndListNext)
def processTransactions(conn, app, txs, poolMap): inserts, notifications = [], [] coinStat = commonUtils.getCoinStat(conn, app) txNotAddresses = tgBotUtils.getAllAddressesForTxAlert(conn, app) logger.info("txNotAddresses are: {}".format(txNotAddresses)) for tx in txs: if tx["txCategory"] == hConstants.TX_REGULAR: insertData = getRegularTransaction(tx) processTxForNotification(tx, notifications, txNotAddresses) else: insertData = getStakingTransaction(tx, coinStat, poolMap) inserts.append(insertData) # logger.info("inserts are:") # logger.info(inserts) if len(inserts) > 0: batchCreateTx(conn, inserts) logger.info("notifications are: {}".format(len(notifications))) if len(notifications) > 0: notificationUtils.batchCreateNotificationsWithOtherData( conn, notifications)
def listNetworkStats(conn, app): coinStat = commonUtils.getCoinStat(conn, app) shardData = harmonyShard.getShardDataFromDB(conn) return dbUtil.combineResults2("coinStat", coinStat, "shardData", shardData)
def syncValidators(conn, app, data, event): # logger.info("in harmony validator sync") startTime = commonUtils.getCurrentTime() startTimeAudit = datetime.datetime.now() # conn = dbUtil.getConnection() # currentTime = datetime.datetime.now() currentHour = datetime.datetime.now().replace(microsecond=0, second=0, minute=0) previousHour = currentHour - datetime.timedelta(hours=1) # logger.info("currentTime: {}".format(currentTime)) # logger.info("loading request") # body = json.loads(event["body"]) validators = data["allValidators"] stakingInfo = data["stakingInfo"] epochInfo = data["epochInfo"] currentEpoch = epochInfo["currentEpoch"] logger.info("total number of validators returned is: " + str(len(validators))) if len(validators) <= 100: logger.info( "number of validators returned is much less than what was expected. skipping the processing for now." ) return getResponse(jsondumps({"result": "successful"})) currentEpochValMap = getCurrentValMap(conn, currentEpoch, None) previousEpochValMap = getCurrentValMap(conn, currentEpoch - 1, None) currentHourValMap = getCurrentValMap(conn, currentEpoch, currentHour) previousHourValMap = getCurrentValMap(conn, None, previousHour) commonUtils.logTimeDiff(startTime, "before coinstats") currentCoinStat = commonUtils.getCoinStat(conn, app) commonUtils.logTimeDiff(startTime, "before getBlockCompletionFactor") completionFactor, enoughDataForCurrentApr = getBlockCompletionFactor( epochInfo, stakingInfo, currentCoinStat, currentEpoch) # logger.info("is enough data captured for current apr: {}".format(enoughDataForCurrentApr)) valMap = harmonyData.listHPoolsAsMap(conn) networkStake = getNetworkStake(validators) commonUtils.logTimeDiff( startTime, "time spent before starting validators processing") i = 0 deltaInserts, deltaUpdates = [], [] stakeWeights = [] for validator in validators: address = validator["address"] # logger.info("{} - processing validator: {}".format(i, address)) # logger.info(validator) i += 1 commonUtils.logTimeDiff(startTime, "processing validator #: {}".format(i)) blsKeyCount = len(validator["blsPublicKeys"]) optimalBlsKeyCount, bidPerSeat = getOptimalBlsKeyCount( stakingInfo, blsKeyCount, validator) commonUtils.logTimeDiff( startTime, "before processing syncValidator #: {}".format(i)) stakeWeight = getStakeWeight(validator, networkStake) stakeWeights.append(stakeWeight) dbValidator = None if address in valMap: dbValidator = valMap[address] hPoolId = syncValidator(conn, validator, blsKeyCount, optimalBlsKeyCount, bidPerSeat, stakeWeight, currentEpoch, dbValidator) commonUtils.logTimeDiff( startTime, "after processing syncValidator #: {}".format(i)) currEpochSummary = getValidatorDetails(currentEpochValMap, hPoolId) commonUtils.logTimeDiff( startTime, "after getting currEpochSummary #: {}".format(i)) prevEpochSummary = getValidatorDetails(previousEpochValMap, hPoolId) commonUtils.logTimeDiff( startTime, "after getting prevEpochSummary #: {}".format(i)) syncValidatorEpochSummary(conn, validator, blsKeyCount, bidPerSeat, hPoolId, currEpochSummary, prevEpochSummary, currentEpoch, completionFactor, enoughDataForCurrentApr) commonUtils.logTimeDiff( startTime, "after processing syncValidatorEpochSummary #: {}".format(i)) # logger.info("processing hourly data") currHourValDetails = getValidatorDetails(currentHourValMap, hPoolId) commonUtils.logTimeDiff( startTime, "after obtaining currHourValDetails #: {}".format(i)) prevHourValDetails = getValidatorDetails(previousHourValMap, hPoolId) # logger.info("current hour existing details") # logger.info(currHourValDetails) # logger.info("previous hour existing details") # logger.info(prevHourValDetails) # logger.info("previous hourly delta performance") commonUtils.logTimeDiff( startTime, "before delta perf - processing validator #: {}".format(i)) processDeltaPerf(conn, validator, hPoolId, currHourValDetails, prevHourValDetails, currentEpoch, currentHour, deltaInserts, deltaUpdates) commonUtils.logTimeDiff(startTime, "after processDeltaPerf #: {}".format(i)) # logger.info("after processing deltas") commonUtils.logTimeDiff(startTime, "before processing hpoolperf inserts and updates") batchCreateHPoolPerf(conn, deltaInserts) batchUpdateHPoolPerf(conn, deltaUpdates) conn.commit() # processDeltaPerf(conn, app, validators, currentHour) commonUtils.logTimeDiff(startTime, "before processing perf index") if enoughDataForCurrentApr: processPerfIndex(conn, app, epochInfo) conn.commit() # uniqueDelegates = len(delegates) commonUtils.logTimeDiff(startTime, "before processing perf index") processEpochPerf(conn, app, epochInfo, currentCoinStat, enoughDataForCurrentApr) commonUtils.logTimeDiff(startTime, "before update coinstat") valForNetworkHalt = getValCountForNetworkHalt(stakeWeights) updateCoinStats(conn, app, stakingInfo, epochInfo, len(validators), valForNetworkHalt) conn.commit() harmonyValEvents.generateValSyncEvents(conn, currentEpoch, enoughDataForCurrentApr) commonUtils.logTimeDiff(startTime, "before creating event and audit") auditUtils.createEvent(conn, app, eventName.syncHarmonyValidators) auditUtils.audit(conn, app, event, eventName.syncHarmonyValidators, "service", startTimeAudit) # conn.close() commonUtils.logTimeDiff(startTime, "total time spent") return getResponse(jsondumps({"result": "successful"}))