def syncDelegations():
logger.info("obtaining db delegates")
session = Session()
dbDelegates = commonUtils.getDataFromGet(session,
constants.listDelegatesUrl)
# logger.info(dbDelegates)
logger.info("obtaining network delegates")
allDelegations = getAllDelegations(session)
# logger.info(allDelegations)
logger.info("processing delegates")
updates, inserts, deletes = processDelegations(allDelegations, dbDelegates)
logger.info("after processing delegates")
logger.info("deletes len is: {}".format(len(deletes)))
logger.info("syncing all delegations - delegateSync")
delReqDetails = {
"type": "delegateSync",
"updates": updates,
"inserts": inserts,
"deletes": deletes
}
commonUtils.postReq(constants.updateUrl, delReqDetails)
logger.info("after syncing all delegations - delegateSync")
def syncValVersions():
session = Session()
dbValKeys = commonUtils.getDataFromGet(session, constants.listValKeys)
# logger.info(dbValKeys)
dbKeyValMap = commonUtils.getMapFromList(dbValKeys, "blsKey")
metrics = commonUtils.getDataFromGet(session, constants.METRICS_URL)
# logger.info(metrics)
data = processValVersions(metrics, dbKeyValMap)
logger.info("version number data: {}".format(len(data)))
addReqDetails = {"type": "versionSync", "data": data}
# logger.info(addReqDetails)
commonUtils.postReq(constants.versionSync, addReqDetails)
def syncAllAddresses():
session = Session()
r = requests.get(constants.ADDRESS_URL)
addressesData = r.json()
# logger.info(addressesData)
networkAddresses = addressesData["address"]
logger.info("obtaining db address list")
dbAddresses = commonUtils.getDataFromGet(
session, constants.listAllAddressesBasicUrl)
logger.info("preparing db address map for {} addresses".format(
len(dbAddresses)))
addressMap = commonUtils.getMapFromList(dbAddresses, "address")
logger.info("processing addresses")
updates, inserts = processAddresses(networkAddresses, addressMap)
logger.info("syncing all addresses with database")
addReqDetails = {
"type": "addressSync",
"updates": updates,
"inserts": inserts,
"balanceIncludesStake": False,
"allMode": allMode,
}
commonUtils.postReq(constants.syncAddressesUrl, addReqDetails)
logger.info("after syncing all addresses with database")
def syncStats():
session = Session()
dbData = commonUtils.getDataFromGet(session,
constants.listDataForEpochSign)
dbValidators = dbData["validators"]
syncedTillEpoch = dbData["epoch"]
currentEpoch = commonUtils.getHarmonyResultDataFromPost(
session, "hmyv2_getEpoch", [])
prevEpoch = currentEpoch - 1
logger.info("prevEpoch: {} syncedTillEpoch: {}".format(
prevEpoch, syncedTillEpoch))
if prevEpoch <= syncedTillEpoch:
logger.info(
"skipping processing as data is already synced up. prevEpoch <= syncedTillEpoch"
)
return
allData = []
for epoch in range(syncedTillEpoch + 1, prevEpoch + 1):
# logger.info("processing epoch: {}".format(epoch))
data = processEpoch(session, epoch, dbValidators)
allData.append(data)
logger.info("# of epochs processed: {}".format(len(allData)))
addReqDetails = {
"type": "epochSignSync",
"allData": allData,
"epoch": prevEpoch
}
# logger.info(addReqDetails)
commonUtils.postReq(constants.syncEpochSignUrl, addReqDetails)
def syncValVersions():
session = Session()
dbValKeys = commonUtils.getDataFromGet(session, constants.listValKeys)
# logger.info(dbValKeys)
dbKeyValMap = commonUtils.getMapFromList(dbValKeys, "blsKey")
metrics = commonUtils.getDataFromGet(session, constants.METRICS_URL)
# logger.info(metrics)
keyVersionMap = processValVersions(metrics, dbKeyValMap)
# logger.info(keyVersionMap)
keyVersions = list(keyVersionMap.values())
# logger.info(keyVersions)
logger.info("version number data: {}".format(len(keyVersions)))
addReqDetails = {"type": "versionSync", "keyVersions": keyVersions}
# logger.info("final request is: {}".format(addReqDetails))
commonUtils.postReq(constants.versionSync, addReqDetails)
def processShardUtils():
# hmyv2_getValidatorInformation
# capture block production rate
# obtain oldest block production rate after current epoch
# capture pace of blocks
session = Session()
currentEpoch = commonUtils.getHarmonyResultDataFromPost(
session, "hmyv2_getEpoch", [])
dbDetails = commonUtils.getDataFromGet(session,
constants.shardSyncDetailsUrl)
logger.info("dbDetails: {}".format(dbDetails))
syncedEpoch = dbDetails["epochNumber"]
dbShardData = dbDetails["shardData"]
logger.info("currentEpoch: {}, syncedEpoch: {}".format(
currentEpoch, syncedEpoch))
logger.info("obtaining shard details")
shard0Details = getShardDetails(session, constants.HARMONY_BASE_URL, 0)
shard1Details = getShardDetails(session, constants.HARMONY_BASE_URL_S1, 1)
shard2Details = getShardDetails(session, constants.HARMONY_BASE_URL_S2, 2)
shard3Details = getShardDetails(session, constants.HARMONY_BASE_URL_S3, 3)
logger.info("after obtaining shard details")
shardDetails = []
if "0" not in dbShardData:
# this means this is the first execution of the job
logger.info(
"this means this is the first execution of the job. preparing initAllShardData"
)
dbShardData = initAllShardData(session, shard0Details, shard1Details,
shard2Details, shard3Details)
logger.info("after preparing initAllShardData")
# return
elif currentEpoch != syncedEpoch:
# update previous epoch first
logger.info(
"epochs have changed. this means we need to update previous epoch data first"
)
shardDetails.append(processShardEnd(dbShardData["0"]))
shardDetails.append(processShardEnd(dbShardData["1"]))
shardDetails.append(processShardEnd(dbShardData["2"]))
shardDetails.append(processShardEnd(dbShardData["3"]))
# init shard data for this epoch
dbShardData = initAllShardData(session, shard0Details, shard1Details,
shard2Details, shard3Details)
# lastBlock = getEpochBlockDetails(session)
shardDetails.append(processShard0(shard0Details, dbShardData["0"]))
shardDetails.append(
processOtherShard(shard1Details, dbShardData["1"], dbShardData["0"]))
shardDetails.append(
processOtherShard(shard2Details, dbShardData["2"], dbShardData["0"]))
shardDetails.append(
processOtherShard(shard3Details, dbShardData["3"], dbShardData["0"]))
# get latest shard 0 record for calculations
# get first block of epoch to calculate pace or may be do it in db
# shardDetails = {}
# shardDetails["0"] = getShardDetails(session, constants.HARMONY_BASE_URL, 0)
# shardDetails["0"]["epochLastBlock"] = epochLastBlock
#
# shardDetails["1"] = getShardDetails(session, constants.HARMONY_BASE_URL_S1, 1)
# shardDetails["2"] = getShardDetails(session, constants.HARMONY_BASE_URL_S2, 2)
# shardDetails["3"] = getShardDetails(session, constants.HARMONY_BASE_URL_S3, 3)
reqDetails = {
"type": "shardSync",
"shardDetails": shardDetails,
"currentEpoch": currentEpoch
}
# logger.info(reqDetails)
commonUtils.postReq(constants.syncShardUrl, reqDetails)
def processBlsUtils():
# hmyv2_getValidatorInformation
# capture block production rate
# obtain oldest block production rate after current epoch
# capture pace of blocks
# get latest next election slots
# get all validators
session = Session()
logger.info("obtaining blsKeySyncDetailsUrl data")
blsKeySyncDetails = commonUtils.getDataFromGet(
session, constants.blsKeySyncDetailsUrl)
# logger.info(blsKeySyncDetails)
dbValidators = blsKeySyncDetails["validators"]
# dbShards = blsKeySyncDetails["shards"]
keys = blsKeySyncDetails["keys"]
dbKeyMap = commonUtils.getMapFromList(keys, "blsKey")
logger.info("obtaining epoch details")
currentEpoch = commonUtils.getHarmonyResultDataFromPost(
session, "hmyv2_getEpoch", [])
shardDetails = getAllShardDetails(session, currentEpoch)
# latestBlock = commonUtils.getHarmonyResultDataFromPost(session, "hmyv2_blockNumber", [])
#note down block for which data is being captured ... record block number as well along with bls data
logger.info("obtaining validators")
# dbValidators = commonUtils.getDataFromGet(session, constants.listAllValidatorsBasicUrl)
blsKeyInserts, blsPerfData = [], []
rewardsDetails = {"0": 0, "1": 0, "2": 0, "3": 0, "totalRewards": 0}
i = 0
for validator in dbValidators:
#exclude the ones not elected
# status = commonUtils.getStatus(validatorDetails["eposStatus"])
# logger.info(validator)
status = validator["status"]
if status != constants.H_STATUS_ELECTED:
# logger.info("skipping validator, {}, as status is not elected, it is: {}".format(
# validator["name"], status))
continue
address = validator["address"]
i += 1
logger.info("{} - obtaining all keys for validator: {}".format(
i, address))
validatorDetails = getValidatorDetails(session, address)
perfData, keyInserts = processKeys(validatorDetails["perfByKeys"],
dbKeyMap, validator["hPoolId"],
currentEpoch, rewardsDetails)
blsPerfData.extend(perfData)
blsKeyInserts.extend(keyInserts)
# loop again here to calculate overall averages
# logger.info("blsPerfData before overall percentages: {}".format(blsPerfData))
processForOverallPercentages(blsPerfData, rewardsDetails)
# logger.info("blsPerfData after overall percentages: {}".format(blsPerfData))
logger.info("blsPerfData # of keys is: {}".format(len(blsPerfData)))
logger.info("syncing all addresses with database")
addReqDetails = {
"type": "blsKeySync",
"blsPerfData": blsPerfData,
"inserts": blsKeyInserts,
"shardDetails": shardDetails,
"currentEpoch": currentEpoch
}
commonUtils.postReq(constants.syncBlsPerfUrl, addReqDetails)
logger.info("after syncing all addresses with database")