def listRewardsHistory(conn, app, event, showHistory, startTime):
# logger.info("in listRewardsHistory")
address = event["queryStringParameters"]["address"]
# logger.info("obtaining getRewardsDetails for address: {}".format(address))
history = getRewardsDetails(conn, address)
# logger.info("obtaining getRewardsHistorySummary for address: {}".format(address))
historySummary = getRewardsHistorySummary(conn, address, '%%Y-%%m')
# logger.info("obtaining getRewardsHistorySummary for address: {}".format(address))
historySummaryAnnual = getRewardsHistorySummary(conn, address, '%%Y')
coinStat = commonUtils.getCoinStat(conn, app)
data = harmonyAddress.getDelegationDetails(conn, address)
smartStake = harmonyAddress.isUsingSmartStake(data)
auditUtils.audit(conn,
app,
event,
"rewardsHistory",
"get",
startTime,
None,
address=address)
return dbUtil.combineResults6("history", history, "data", data, "coinStat",
coinStat, "historySummary", historySummary,
"historySummaryAnnual", historySummaryAnnual,
"smartStake", smartStake)
def syncDelegates(conn, app, data, event):
# logger.info("in harmony delegation sync")
startTime = datetime.datetime.now()
# conn = dbUtil.getConnection()
# currentTime = datetime.datetime.now()
# logger.info(body)
logger.info("processing delegation deletes")
if data["deletes"] is not None:
processDelegationDeletes(conn, data["deletes"])
logger.info("processing delegation inserts")
if data["inserts"] is not None:
processDelegationInserts(conn, data["inserts"])
logger.info("processing delegation updates")
if data["updates"] is not None:
processDelegationUpdates(conn, data["updates"])
conn.commit()
logger.info("updating balances")
updateBalances(conn, True)
conn.commit()
updateRank(conn, 500)
conn.commit()
auditUtils.createEvent(conn, app, eventName.syncHarmonyDelegations)
auditUtils.audit(conn, app, event, eventName.syncHarmonyDelegations,
"service", startTime)
logger.info("processing finished")
def lambda_handler(event, context):
logger.info("in updateData for harmony")
startTime = datetime.datetime.now()
conn, app, error = securityUtils.performSecurityCheckAndGetDetails(event)
if error:
return error
body = json.loads(event["body"])
# logger.info("body: {}".format(body))
data = body["data"]
if "type" in data:
dataType = data["type"]
else:
dataType = body["type"]
logger.info("dataType: {}".format(dataType))
if dataType == "valSync":
harmonyValSync.syncValidators(conn, app, data, event)
elif dataType == "addressSync":
harmonyDelegationSync.syncAddresses(conn, app, data, event)
elif dataType == "delegateSync":
harmonyDelegationSync.syncDelegates(conn, app, data, event)
elif dataType == "sendNotifications":
hTgNotificationSync.sendAllNotifications(conn)
elif dataType == "eventsSync":
harmonyEvents.syncEvents(conn, app, data, event)
elif dataType == "txSync":
harmonyTransactionSync.syncTransactions(conn, app, data, event)
elif dataType == "versionSync":
harmonyVersionSync.syncVersion(conn, app, data, startTime)
elif dataType == "blsKeySync":
harmonyBlsKeySync.blsKeySync(conn, app, data, event)
elif dataType == "epochSignSync":
harmonyEpochSign.epochSignSync(conn, app, data, event)
elif dataType == "electionSync":
harmonyElectionProcess.electionSync(conn, app, data, event)
elif dataType == "shardSync":
harmonyShard.syncShardDetails(conn, app, data, event)
elif dataType == "saveHealthCheck":
saveHealthCheck.createHealthCheck(app, body["data"], conn)
auditUtils.audit(conn, app, event, "saveHealthCheck", "service",
startTime)
elif dataType == "saveAlias":
# data = body["data"]
result = aliasUtils.saveAlias(conn, app, event, data, startTime)
if result:
conn.close()
return result
elif dataType == "clearAlias":
# data = body["data"]
aliasUtils.clearAlias(conn, app, data, startTime)
else:
logger.info(
"dataType didnt match supported datatypes: {}".format(dataType))
conn.close()
return utilities.getSuccessResponse()
def electionSync(conn, app, data, event):
logger.info("in electionSync for harmony")
startTimeAudit = datetime.datetime.now()
processElections(conn, app)
auditUtils.createEvent(conn, app, eventName.syncHarmonyElections)
auditUtils.audit(conn, app, event, eventName.syncHarmonyElections,
"service", startTimeAudit)
def syncAddresses(conn, app, data, event):
# logger.info("in harmony address sync")
startTime = datetime.datetime.now()
updated = updateAddressHistory(conn)
conn.commit()
if updated:
logger.info("skipping address sync as address history was processed")
return
inserts = data["inserts"]
updates = data["updates"]
# logger.info("processing address inserts")
if inserts is not None:
# count = 0
insertArgs = []
for i in inserts:
# logger.info("inserting address: {}".format(i))
# , i["txCount"]
record = (i["address"], i["balance"])
insertArgs.append(record)
# count += 1
# if count % 1000 == 0:
# conn.commit()
batchCreateAddress(conn, insertArgs)
conn.commit()
# logger.info("processing address updates")
if updates is not None:
updateArgs = []
for i in updates:
# logger.info("updating address: {}".format(i))
# logger.info(i)
# i["txCount"],
updateArgs.append((i["balance"], i["addressId"]))
batchUpdateAddress(conn, updateArgs)
conn.commit()
balanceIncludesStake = data["balanceIncludesStake"]
# logger.info("balanceIncludesStake: {}".format(balanceIncludesStake))
updateBalances(conn, balanceIncludesStake)
conn.commit()
updateRank(conn, 50000)
conn.commit()
auditUtils.createEvent(conn, app, eventName.syncHarmonyAddresses)
auditUtils.audit(conn, app, event, eventName.syncHarmonyAddresses,
"service", startTime)
def blsKeySync(conn, app, data, event):
logger.info("in blsKeySync")
startTime = datetime.datetime.now()
blockNumber = processShards(conn, data["shardDetails"])
processKeyInserts(conn, data["inserts"])
processKeyPerf(conn, data["blsPerfData"], data["currentEpoch"],
blockNumber)
auditUtils.createEvent(conn, app, eventName.syncHarmonyBlsKeys)
auditUtils.audit(conn, app, event, eventName.syncHarmonyBlsKeys, "service",
startTime)
logger.info("processing finished")
def saveAlias(conn, app, event, data, startTime):
address = data["alias"]["address"]
# addressList = addresses.split(",")
alias = data["alias"]["alias"]
# logger.info("creating alias")
resp = createAlias(conn, alias, address, app)
if resp is not None:
return resp
auditUtils.audit(conn,
app,
event,
"saveAlias",
"post",
startTime,
None,
alias=alias)
def listHealthCheckData(conn, app, event, startTime):
# logger.info("obtaining health check data from database - " + app)
sql = " select healthid, nodeName, symbol, UNIX_TIMESTAMP(checkupTime) as cTime, "
sql += " DATE_FORMAT(checkupTime,'%%Y-%%m-%%d %%H:%%i:%%s') as checkupTime, "
sql += " networkBlockHeight, nodeBlockHeight, heightGap, shardId, epochTimestamp, "
sql += " round(time_to_sec((TIMEDIFF(NOW(), checkupTime))) / 60,0) as elapsedTime "
sql += " from " + tables.nodehealth + " w "
sql += " where symbol = %s "
sql += " order by cTime desc"
sql += " limit 50"
# logger.info(sql)
activities = dbUtil.listResultsWithConn(sql, conn, app)
pool = None
auditUtils.audit(conn, app, event, "listHealthCheckData", "get", startTime, None)
return dbUtil.combineResults2("pool", pool, "activities", activities)
def lambda_handler(event, context):
logger.info("in updateData for harmony")
startTime = datetime.datetime.now()
conn, app, error = securityUtils.performSecurityCheckAndGetDetails(event)
if error:
return error
# body = json.loads(event["body"])
# logger.info("body: {}".format(body))
sendAllNotifications(conn)
auditUtils.createEvent(conn, app, eventName.syncHarmonyNotifications)
auditUtils.audit(conn, app, event, eventName.syncHarmonyNotifications,
"service", startTime)
# harmonyBlsKeys.processKeyNodeVersionAlert(conn, app)
conn.close()
return utilities.getSuccessResponse()
def lambda_handler(event, context):
# logger.info(event)
startTime = datetime.datetime.now()
conn, app, error = securityUtils.performSecurityCheckAndGetDetails(event)
if error:
return error
# logger.info(app)
body = json.loads(event["body"])
createHealthCheck(app, body, conn)
conn.commit()
# logger.info("record created/modified")
auditUtils.audit(conn, app, event, "saveHealthCheck", "service", startTime)
conn.close()
return utilities.getSuccessResponse()
def epochSignSync(conn, app, data, event):
logger.info("in blsKeySync")
startTime = datetime.datetime.now()
allData = data["allData"]
currentTime = datetime.datetime.now()
epochSignUpdates = []
epochSignSummaryUpdates = []
maxEpoch, signRate = 0, 0
for epochData in allData:
logger.info("processing epoch: {}".format(epochData["epoch"]))
summary = (epochData["signRate"], epochData["epoch"])
epochSignSummaryUpdates.append(summary)
if epochData["epoch"] > maxEpoch:
signRate = epochData["signRate"]
maxEpoch = epochData["epoch"]
for validator in epochData["validators"]:
valSign = (validator["signed"], validator["askedToSign"],
validator["signPer"], currentTime,
currentTime.strftime('%s'), epochData["epoch"],
validator["hPoolId"])
epochSignUpdates.append(valSign)
batchUpdateEpochSigns(conn, epochSignUpdates)
batchUpdateHistory(conn, epochSignSummaryUpdates)
updateSignRate(conn, signRate, app)
epoch = data["epoch"]
commonUtils.updateSyncStatus(conn, app,
constants.SYNC_STATUS_EPOCH_FOR_SIGN, None,
epoch)
auditUtils.createEvent(conn, app, eventName.syncEpochSign)
auditUtils.audit(conn, app, event, eventName.syncEpochSign, "service",
startTime)
logger.info("processing finished")
def listHData(conn, app, event, context, startTime, dataType):
logger.info("in harmony list data - {}".format(dataType))
data = None
if dataType == "listPools":
status = None
if "status" in event["queryStringParameters"]:
status = event["queryStringParameters"]["status"]
data = listHPools(conn, app, status)
auditUtils.audit(conn, app, event, "listData_" + dataType, "get",
startTime)
elif dataType == "listPoolsSummary":
data = listPoolsSummary(conn, app)
elif dataType == "event":
data = getEventDetails(conn, event)
elif dataType == "keys":
hPoolId = getHPoolId(event)
data = harmonyBlsKeys.listKeys(conn, app, event, hPoolId)
elif dataType == "key":
data = harmonyBlsKeys.listKeyDetails(conn, app, event)
elif dataType == "listApr":
data = listHPoolsApr(conn, app)
elif dataType == "getPool":
hPoolId = getHPoolId(event)
# logger.info("hPoolId: %s", hPoolId)
more = False
if "showMore" in event["queryStringParameters"]:
# logger.info(event["queryStringParameters"])
more = event["queryStringParameters"]["showMore"]
# logger.info("show more: {}".format(more))
data = getHPool(conn, app, hPoolId, more)
auditUtils.audit(conn, app, event, "listData_" + dataType, "get",
startTime)
elif dataType == "health":
data = getHealthSummary.listHealthCheckData(conn, app, event,
startTime)
# elif dataType == "listValPerf":
# data = listValPerformanceData(conn, app, event, startTime)
elif dataType == "listAllDelegates":
data = listAllDelegates(conn, app)
elif dataType == "listAllAddresses":
data = harmonyAddress.listAllAddresses(conn, app)
elif dataType == "listAllAddressesBasic":
data = harmonyAddress.listAllAddressesBasic(conn, app)
elif dataType == "blsKeySyncDetails":
data = harmonyBlsKeys.listBlsKeySyncDetails(conn, app)
elif dataType == "listAllValidatorsBasic":
data = listAllValidatorsBasic(conn)
elif dataType == "listDataForEpochSign":
data = listDataForEpochSign(conn, app)
elif dataType == "richlist":
count = 200
if "count" in event["queryStringParameters"]:
count = int(event["queryStringParameters"]["count"])
if count == 100:
count = 200
data = getRichList(conn, app, count)
elif dataType == "delegates":
hPoolId = getHPoolId(event)
data = listDelegates(conn, app, hPoolId)
elif dataType == "richDelegates":
data = getRichDelegates(conn, app)
elif dataType == "networkStakeHistory":
data = harmonyHistory.listNetworkStakeHistory(conn, app, event)
elif dataType == "rewardsHistory":
data = harmonyRewards.listRewardsHistory(conn, app, event, context,
startTime)
# elif dataType == "valStakeHistory":
# data = harmonyHistory.listValStakeHistory(conn, app, event)
elif dataType == "addressHistory":
data = harmonyHistory.listAddressHistory(conn, app, event)
elif dataType == "valPerf":
hPoolId = getHPoolId(event)
more = False
if "showMore" in event["queryStringParameters"]:
# logger.info(event["queryStringParameters"])
more = event["queryStringParameters"]["showMore"]
data = listPerf(conn, app, hPoolId, more)
elif dataType == "networkHistory":
data = harmonyHistory.listNetworkHistory(conn, app)
elif dataType == "versionStats":
data = harmonyVersionSync.listVersionStats(conn)
elif dataType == "networkStats":
data = harmonyHistory.listNetworkStats(conn, app)
elif dataType == "addressDetails":
alias, address, subType = None, None, None
if "address" in event["queryStringParameters"]:
address = event["queryStringParameters"]["address"]
elif "alias" in event["queryStringParameters"]:
alias = event["queryStringParameters"]["alias"]
if "subType" in event["queryStringParameters"]:
subType = event["queryStringParameters"]["subType"]
data = harmonyAddress.getAddressDisplayDetails(conn, app, address,
alias, subType)
auditUtils.audit(conn, app, event, "address", "get", startTime)
elif dataType == "events":
hPoolId = paramUtils.getOptionalIntParam(event, "hPoolId")
data = harmonyEvents.listEvents(conn, app, event, hPoolId, startTime)
elif dataType == "addressEvents":
address = None
if "address" in event["queryStringParameters"]:
address = event["queryStringParameters"]["address"]
data = harmonyAddress.listAddressEvents(conn, app, address)
elif dataType == "coinStat":
data = getCoinStat(conn, app)
elif dataType == "shardSyncDetails":
data = harmonyShard.getShardSyncInputDetails(conn)
elif dataType == "valKeys":
data = harmonyBlsKeys.listValKeys(conn)
return data
def syncValidators(conn, app, data, event):
# logger.info("in harmony validator sync")
startTime = commonUtils.getCurrentTime()
startTimeAudit = datetime.datetime.now()
# conn = dbUtil.getConnection()
# currentTime = datetime.datetime.now()
currentHour = datetime.datetime.now().replace(microsecond=0,
second=0,
minute=0)
previousHour = currentHour - datetime.timedelta(hours=1)
# logger.info("currentTime: {}".format(currentTime))
# logger.info("loading request")
# body = json.loads(event["body"])
validators = data["allValidators"]
stakingInfo = data["stakingInfo"]
epochInfo = data["epochInfo"]
currentEpoch = epochInfo["currentEpoch"]
logger.info("total number of validators returned is: " +
str(len(validators)))
if len(validators) <= 100:
logger.info(
"number of validators returned is much less than what was expected. skipping the processing for now."
)
return getResponse(jsondumps({"result": "successful"}))
currentEpochValMap = getCurrentValMap(conn, currentEpoch, None)
previousEpochValMap = getCurrentValMap(conn, currentEpoch - 1, None)
currentHourValMap = getCurrentValMap(conn, currentEpoch, currentHour)
previousHourValMap = getCurrentValMap(conn, None, previousHour)
commonUtils.logTimeDiff(startTime, "before coinstats")
currentCoinStat = commonUtils.getCoinStat(conn, app)
commonUtils.logTimeDiff(startTime, "before getBlockCompletionFactor")
completionFactor, enoughDataForCurrentApr = getBlockCompletionFactor(
epochInfo, stakingInfo, currentCoinStat, currentEpoch)
# logger.info("is enough data captured for current apr: {}".format(enoughDataForCurrentApr))
valMap = harmonyData.listHPoolsAsMap(conn)
networkStake = getNetworkStake(validators)
commonUtils.logTimeDiff(
startTime, "time spent before starting validators processing")
i = 0
deltaInserts, deltaUpdates = [], []
stakeWeights = []
for validator in validators:
address = validator["address"]
# logger.info("{} - processing validator: {}".format(i, address))
# logger.info(validator)
i += 1
commonUtils.logTimeDiff(startTime,
"processing validator #: {}".format(i))
blsKeyCount = len(validator["blsPublicKeys"])
optimalBlsKeyCount, bidPerSeat = getOptimalBlsKeyCount(
stakingInfo, blsKeyCount, validator)
commonUtils.logTimeDiff(
startTime, "before processing syncValidator #: {}".format(i))
stakeWeight = getStakeWeight(validator, networkStake)
stakeWeights.append(stakeWeight)
dbValidator = None
if address in valMap:
dbValidator = valMap[address]
hPoolId = syncValidator(conn, validator, blsKeyCount,
optimalBlsKeyCount, bidPerSeat, stakeWeight,
currentEpoch, dbValidator)
commonUtils.logTimeDiff(
startTime, "after processing syncValidator #: {}".format(i))
currEpochSummary = getValidatorDetails(currentEpochValMap, hPoolId)
commonUtils.logTimeDiff(
startTime, "after getting currEpochSummary #: {}".format(i))
prevEpochSummary = getValidatorDetails(previousEpochValMap, hPoolId)
commonUtils.logTimeDiff(
startTime, "after getting prevEpochSummary #: {}".format(i))
syncValidatorEpochSummary(conn, validator, blsKeyCount, bidPerSeat,
hPoolId, currEpochSummary, prevEpochSummary,
currentEpoch, completionFactor,
enoughDataForCurrentApr)
commonUtils.logTimeDiff(
startTime,
"after processing syncValidatorEpochSummary #: {}".format(i))
# logger.info("processing hourly data")
currHourValDetails = getValidatorDetails(currentHourValMap, hPoolId)
commonUtils.logTimeDiff(
startTime, "after obtaining currHourValDetails #: {}".format(i))
prevHourValDetails = getValidatorDetails(previousHourValMap, hPoolId)
# logger.info("current hour existing details")
# logger.info(currHourValDetails)
# logger.info("previous hour existing details")
# logger.info(prevHourValDetails)
# logger.info("previous hourly delta performance")
commonUtils.logTimeDiff(
startTime,
"before delta perf - processing validator #: {}".format(i))
processDeltaPerf(conn, validator, hPoolId, currHourValDetails,
prevHourValDetails, currentEpoch, currentHour,
deltaInserts, deltaUpdates)
commonUtils.logTimeDiff(startTime,
"after processDeltaPerf #: {}".format(i))
# logger.info("after processing deltas")
commonUtils.logTimeDiff(startTime,
"before processing hpoolperf inserts and updates")
batchCreateHPoolPerf(conn, deltaInserts)
batchUpdateHPoolPerf(conn, deltaUpdates)
conn.commit()
# processDeltaPerf(conn, app, validators, currentHour)
commonUtils.logTimeDiff(startTime, "before processing perf index")
if enoughDataForCurrentApr:
processPerfIndex(conn, app, epochInfo)
conn.commit()
# uniqueDelegates = len(delegates)
commonUtils.logTimeDiff(startTime, "before processing perf index")
processEpochPerf(conn, app, epochInfo, currentCoinStat,
enoughDataForCurrentApr)
commonUtils.logTimeDiff(startTime, "before update coinstat")
valForNetworkHalt = getValCountForNetworkHalt(stakeWeights)
updateCoinStats(conn, app, stakingInfo, epochInfo, len(validators),
valForNetworkHalt)
conn.commit()
harmonyValEvents.generateValSyncEvents(conn, currentEpoch,
enoughDataForCurrentApr)
commonUtils.logTimeDiff(startTime, "before creating event and audit")
auditUtils.createEvent(conn, app, eventName.syncHarmonyValidators)
auditUtils.audit(conn, app, event, eventName.syncHarmonyValidators,
"service", startTimeAudit)
# conn.close()
commonUtils.logTimeDiff(startTime, "total time spent")
return getResponse(jsondumps({"result": "successful"}))