def syncDelegations(): logger.info("obtaining db delegates") session = Session() dbDelegates = commonUtils.getDataFromGet(session, constants.listDelegatesUrl) # logger.info(dbDelegates) logger.info("obtaining network delegates") allDelegations = getAllDelegations(session) # logger.info(allDelegations) logger.info("processing delegates") updates, inserts, deletes = processDelegations(allDelegations, dbDelegates) logger.info("after processing delegates") logger.info("deletes len is: {}".format(len(deletes))) logger.info("syncing all delegations - delegateSync") delReqDetails = { "type": "delegateSync", "updates": updates, "inserts": inserts, "deletes": deletes } commonUtils.postReq(constants.updateUrl, delReqDetails) logger.info("after syncing all delegations - delegateSync")
def syncValVersions(): session = Session() dbValKeys = commonUtils.getDataFromGet(session, constants.listValKeys) # logger.info(dbValKeys) dbKeyValMap = commonUtils.getMapFromList(dbValKeys, "blsKey") metrics = commonUtils.getDataFromGet(session, constants.METRICS_URL) # logger.info(metrics) data = processValVersions(metrics, dbKeyValMap) logger.info("version number data: {}".format(len(data))) addReqDetails = {"type": "versionSync", "data": data} # logger.info(addReqDetails) commonUtils.postReq(constants.versionSync, addReqDetails)
def syncAllAddresses(): session = Session() r = requests.get(constants.ADDRESS_URL) addressesData = r.json() # logger.info(addressesData) networkAddresses = addressesData["address"] logger.info("obtaining db address list") dbAddresses = commonUtils.getDataFromGet( session, constants.listAllAddressesBasicUrl) logger.info("preparing db address map for {} addresses".format( len(dbAddresses))) addressMap = commonUtils.getMapFromList(dbAddresses, "address") logger.info("processing addresses") updates, inserts = processAddresses(networkAddresses, addressMap) logger.info("syncing all addresses with database") addReqDetails = { "type": "addressSync", "updates": updates, "inserts": inserts, "balanceIncludesStake": False, "allMode": allMode, } commonUtils.postReq(constants.syncAddressesUrl, addReqDetails) logger.info("after syncing all addresses with database")
def syncStats(): session = Session() dbData = commonUtils.getDataFromGet(session, constants.listDataForEpochSign) dbValidators = dbData["validators"] syncedTillEpoch = dbData["epoch"] currentEpoch = commonUtils.getHarmonyResultDataFromPost( session, "hmyv2_getEpoch", []) prevEpoch = currentEpoch - 1 logger.info("prevEpoch: {} syncedTillEpoch: {}".format( prevEpoch, syncedTillEpoch)) if prevEpoch <= syncedTillEpoch: logger.info( "skipping processing as data is already synced up. prevEpoch <= syncedTillEpoch" ) return allData = [] for epoch in range(syncedTillEpoch + 1, prevEpoch + 1): # logger.info("processing epoch: {}".format(epoch)) data = processEpoch(session, epoch, dbValidators) allData.append(data) logger.info("# of epochs processed: {}".format(len(allData))) addReqDetails = { "type": "epochSignSync", "allData": allData, "epoch": prevEpoch } # logger.info(addReqDetails) commonUtils.postReq(constants.syncEpochSignUrl, addReqDetails)
def syncValVersions(): session = Session() dbValKeys = commonUtils.getDataFromGet(session, constants.listValKeys) # logger.info(dbValKeys) dbKeyValMap = commonUtils.getMapFromList(dbValKeys, "blsKey") metrics = commonUtils.getDataFromGet(session, constants.METRICS_URL) # logger.info(metrics) keyVersionMap = processValVersions(metrics, dbKeyValMap) # logger.info(keyVersionMap) keyVersions = list(keyVersionMap.values()) # logger.info(keyVersions) logger.info("version number data: {}".format(len(keyVersions))) addReqDetails = {"type": "versionSync", "keyVersions": keyVersions} # logger.info("final request is: {}".format(addReqDetails)) commonUtils.postReq(constants.versionSync, addReqDetails)
def processShardUtils(): # hmyv2_getValidatorInformation # capture block production rate # obtain oldest block production rate after current epoch # capture pace of blocks session = Session() currentEpoch = commonUtils.getHarmonyResultDataFromPost( session, "hmyv2_getEpoch", []) dbDetails = commonUtils.getDataFromGet(session, constants.shardSyncDetailsUrl) logger.info("dbDetails: {}".format(dbDetails)) syncedEpoch = dbDetails["epochNumber"] dbShardData = dbDetails["shardData"] logger.info("currentEpoch: {}, syncedEpoch: {}".format( currentEpoch, syncedEpoch)) logger.info("obtaining shard details") shard0Details = getShardDetails(session, constants.HARMONY_BASE_URL, 0) shard1Details = getShardDetails(session, constants.HARMONY_BASE_URL_S1, 1) shard2Details = getShardDetails(session, constants.HARMONY_BASE_URL_S2, 2) shard3Details = getShardDetails(session, constants.HARMONY_BASE_URL_S3, 3) logger.info("after obtaining shard details") shardDetails = [] if "0" not in dbShardData: # this means this is the first execution of the job logger.info( "this means this is the first execution of the job. preparing initAllShardData" ) dbShardData = initAllShardData(session, shard0Details, shard1Details, shard2Details, shard3Details) logger.info("after preparing initAllShardData") # return elif currentEpoch != syncedEpoch: # update previous epoch first logger.info( "epochs have changed. this means we need to update previous epoch data first" ) shardDetails.append(processShardEnd(dbShardData["0"])) shardDetails.append(processShardEnd(dbShardData["1"])) shardDetails.append(processShardEnd(dbShardData["2"])) shardDetails.append(processShardEnd(dbShardData["3"])) # init shard data for this epoch dbShardData = initAllShardData(session, shard0Details, shard1Details, shard2Details, shard3Details) # lastBlock = getEpochBlockDetails(session) shardDetails.append(processShard0(shard0Details, dbShardData["0"])) shardDetails.append( processOtherShard(shard1Details, dbShardData["1"], dbShardData["0"])) shardDetails.append( processOtherShard(shard2Details, dbShardData["2"], dbShardData["0"])) shardDetails.append( processOtherShard(shard3Details, dbShardData["3"], dbShardData["0"])) # get latest shard 0 record for calculations # get first block of epoch to calculate pace or may be do it in db # shardDetails = {} # shardDetails["0"] = getShardDetails(session, constants.HARMONY_BASE_URL, 0) # shardDetails["0"]["epochLastBlock"] = epochLastBlock # # shardDetails["1"] = getShardDetails(session, constants.HARMONY_BASE_URL_S1, 1) # shardDetails["2"] = getShardDetails(session, constants.HARMONY_BASE_URL_S2, 2) # shardDetails["3"] = getShardDetails(session, constants.HARMONY_BASE_URL_S3, 3) reqDetails = { "type": "shardSync", "shardDetails": shardDetails, "currentEpoch": currentEpoch } # logger.info(reqDetails) commonUtils.postReq(constants.syncShardUrl, reqDetails)
def processBlsUtils(): # hmyv2_getValidatorInformation # capture block production rate # obtain oldest block production rate after current epoch # capture pace of blocks # get latest next election slots # get all validators session = Session() logger.info("obtaining blsKeySyncDetailsUrl data") blsKeySyncDetails = commonUtils.getDataFromGet( session, constants.blsKeySyncDetailsUrl) # logger.info(blsKeySyncDetails) dbValidators = blsKeySyncDetails["validators"] # dbShards = blsKeySyncDetails["shards"] keys = blsKeySyncDetails["keys"] dbKeyMap = commonUtils.getMapFromList(keys, "blsKey") logger.info("obtaining epoch details") currentEpoch = commonUtils.getHarmonyResultDataFromPost( session, "hmyv2_getEpoch", []) shardDetails = getAllShardDetails(session, currentEpoch) # latestBlock = commonUtils.getHarmonyResultDataFromPost(session, "hmyv2_blockNumber", []) #note down block for which data is being captured ... record block number as well along with bls data logger.info("obtaining validators") # dbValidators = commonUtils.getDataFromGet(session, constants.listAllValidatorsBasicUrl) blsKeyInserts, blsPerfData = [], [] rewardsDetails = {"0": 0, "1": 0, "2": 0, "3": 0, "totalRewards": 0} i = 0 for validator in dbValidators: #exclude the ones not elected # status = commonUtils.getStatus(validatorDetails["eposStatus"]) # logger.info(validator) status = validator["status"] if status != constants.H_STATUS_ELECTED: # logger.info("skipping validator, {}, as status is not elected, it is: {}".format( # validator["name"], status)) continue address = validator["address"] i += 1 logger.info("{} - obtaining all keys for validator: {}".format( i, address)) validatorDetails = getValidatorDetails(session, address) perfData, keyInserts = processKeys(validatorDetails["perfByKeys"], dbKeyMap, validator["hPoolId"], currentEpoch, rewardsDetails) blsPerfData.extend(perfData) blsKeyInserts.extend(keyInserts) # loop again here to calculate overall averages # logger.info("blsPerfData before overall percentages: {}".format(blsPerfData)) processForOverallPercentages(blsPerfData, rewardsDetails) # logger.info("blsPerfData after overall percentages: {}".format(blsPerfData)) logger.info("blsPerfData # of keys is: {}".format(len(blsPerfData))) logger.info("syncing all addresses with database") addReqDetails = { "type": "blsKeySync", "blsPerfData": blsPerfData, "inserts": blsKeyInserts, "shardDetails": shardDetails, "currentEpoch": currentEpoch } commonUtils.postReq(constants.syncBlsPerfUrl, addReqDetails) logger.info("after syncing all addresses with database")