def ContribIdCompletion(pCtx, pData, *pArgs):
pServerName, pLogDate, pLogTime, pContribId, pConnection, pConnectionIP, pLVUuid, pHandle, pStatus, pDummy, pSizeTransferred, pProcessingTime = pArgs[
0]
# Add data to the appropriate contribid entry
l_ContribIdEntry = pData["Handles"][pHandle]["Connections"][pConnection][
"LVUuids"][pLVUuid]["ContribIds"][pContribId]
l_ContribIdEntry["LogDate"] = pLogDate
l_ContribIdEntry["LogTime"] = pLogTime
l_ContribIdEntry["Status"] = pStatus
if pSizeTransferred != None:
l_ContribIdEntry["SizeTransferred"] = pSizeTransferred
else:
l_ContribIdEntry["SizeTransferred"] = 0
l_FileEntry = pData["Handles"][pHandle]["Connections"][pConnection][
"LVUuids"][pLVUuid]["ContribIds"][pContribId]["Files"]
for l_FileData in l_FileEntry.values():
l_ContribIdEntry["SizeTransferred"] += l_FileData["TransferSize"]
l_ContribIdEntry["ProcessingTime"] = pProcessingTime
# If necessary, add to the elapsed time data
l_ElapsedTimeEntry = pCtx["ElapsedTimeData"]
if "JobId" in pData["Handles"][pHandle]["Connections"][pConnection][
"LVUuids"][pLVUuid]["ContribIds"][pContribId]:
l_JobId = pData["Handles"][pHandle]["Connections"][pConnection][
"LVUuids"][pLVUuid]["ContribIds"][pContribId]["JobId"]
if "JobStepId" in pData["Handles"][pHandle]["Connections"][
pConnection]["LVUuids"][pLVUuid]["ContribIds"][pContribId]:
l_JobStepId = pData["Handles"][pHandle]["Connections"][
pConnection]["LVUuids"][pLVUuid]["ContribIds"][pContribId][
"JobStepId"]
if cmn.compareTimes(
(pLogDate, pLogTime), l_ElapsedTimeEntry["jobIds"][l_JobId]
["jobStepIds"][l_JobStepId]["EndDateTime"]) == 1:
l_ElapsedTimeEntry["jobIds"][l_JobId]["jobStepIds"][
l_JobStepId]["EndDateTime"] = (pLogDate, pLogTime)
if cmn.compareTimes(
(pLogDate, pLogTime),
l_ElapsedTimeEntry["jobIds"][l_JobId]["jobStepIds"]
[l_JobStepId]["servers"][pServerName]["EndDateTime"]) == 1:
l_ElapsedTimeEntry["jobIds"][l_JobId]["jobStepIds"][
l_JobStepId]["servers"][pServerName]["EndDateTime"] = (
pLogDate, pLogTime)
if cmn.compareTimes(
(pLogDate, pLogTime), l_ElapsedTimeEntry["jobIds"][l_JobId]
["jobStepIds"][l_JobStepId]["servers"][pServerName]
["connections"][pConnection]["EndDateTime"]) == 1:
l_ElapsedTimeEntry["jobIds"][l_JobId]["jobStepIds"][
l_JobStepId]["servers"][pServerName]["connections"][
pConnection]["EndDateTime"] = (pLogDate, pLogTime)
l_ElapsedTimeEntry["jobIds"][l_JobId]["jobStepIds"][l_JobStepId][
"NumberOfContribIds"] += 1
l_ElapsedTimeEntry["jobIds"][l_JobId]["jobStepIds"][l_JobStepId][
"SizeTransferred"] += l_ContribIdEntry["SizeTransferred"]
l_ElapsedTimeEntry["jobIds"][l_JobId]["jobStepIds"][l_JobStepId][
"servers"][pServerName]["NumberOfContribIds"] += 1
l_ElapsedTimeEntry["jobIds"][l_JobId]["jobStepIds"][l_JobStepId][
"servers"][pServerName]["SizeTransferred"] += l_ContribIdEntry[
"SizeTransferred"]
l_ElapsedTimeEntry["jobIds"][l_JobId]["jobStepIds"][l_JobStepId][
"servers"][pServerName]["connections"][pConnection][
"NumberOfContribIds"] += 1
l_ElapsedTimeEntry["jobIds"][l_JobId]["jobStepIds"][l_JobStepId][
"servers"][pServerName]["connections"][pConnection][
"SizeTransferred"] += l_ContribIdEntry["SizeTransferred"]
else:
print "JobId was not found for handle %d, connection %s, LVUuid %s, contribid %d. Related transfer rate(s) will not be accurate."
return
def StartTransfer(pCtx, pData, *pArgs):
pServerName, pLogDate, pLogTime, pConnection, pConnectionIP, pLVUuid, pHostName, pJobId, pJobStepId, pHandle, pContribId, pRestart = pArgs[
0]
# Add jobid and jobstepid to the appropriate handle entry
l_GetHandleEntry = pData["Handles"][pHandle]
if "JobId" not in l_GetHandleEntry:
l_GetHandleEntry["JobId"] = pJobId
if "JobStepId" not in l_GetHandleEntry:
l_GetHandleEntry["JobStepId"] = pJobStepId
if "LogDate" not in l_GetHandleEntry:
l_GetHandleEntry["LogDate"] = pLogDate
# Add data to the appropriate start transfer entry
l_StartTransferEntry = pData["Handles"][pHandle]["Connections"][
pConnection]["LVUuids"][pLVUuid]["ContribIds"][pContribId]
l_StartTransferEntry["LogDate"] = pLogDate
l_StartTransferEntry["LogTime"] = pLogTime
l_StartTransferEntry["JobId"] = pJobId
l_StartTransferEntry["JobStepId"] = pJobStepId
l_StartTransferEntry["Restart"] = pRestart
# If necessary, add to the elapsed time data
l_ElapsedTimeEntry = pCtx["ElapsedTimeData"]["jobIds"]
l_SizeTransferred = 0
if pJobId not in l_ElapsedTimeEntry:
addJobIdData(pCtx, pLogDate, pLogTime, pJobId, pJobStepId, pServerName,
pConnection, l_SizeTransferred)
else:
l_ElapsedTimeEntry = pCtx["ElapsedTimeData"]["jobIds"][pJobId][
"jobStepIds"]
if pJobStepId not in l_ElapsedTimeEntry:
addJobStepIdData(pCtx, pLogDate, pLogTime, pJobId, pJobStepId,
pServerName, pConnection, l_SizeTransferred)
if cmn.compareTimes(
(pLogDate, pLogTime),
l_ElapsedTimeEntry[pJobStepId]["StartDateTime"]) == -1:
l_ElapsedTimeEntry[pJobStepId]["StartDateTime"] = (pLogDate,
pLogTime)
if pServerName not in l_ElapsedTimeEntry[pJobStepId]["servers"]:
addServerData(pCtx, pLogDate, pLogTime, pJobId, pJobStepId,
pServerName, pConnection, l_SizeTransferred)
else:
if cmn.compareTimes((pLogDate, pLogTime),
l_ElapsedTimeEntry[pJobStepId]["servers"]
[pServerName]["StartDateTime"]) == -1:
l_ElapsedTimeEntry[pJobStepId]["servers"][pServerName][
"StartDateTime"] = (pLogDate, pLogTime)
if pConnection not in l_ElapsedTimeEntry[pJobStepId]["servers"][
pServerName]["connections"]:
addConnectionData(pCtx, pLogDate, pLogTime, pJobId, pJobStepId,
pServerName, pConnection, l_SizeTransferred)
else:
if cmn.compareTimes(
(pLogDate, pLogTime),
l_ElapsedTimeEntry[pJobStepId]["servers"][pServerName]
["connections"][pConnection]["StartDateTime"]) == -1:
l_ElapsedTimeEntry[pJobStepId]["servers"][pServerName][
"connections"][pConnection]["StartDateTime"] = (
pLogDate, pLogTime)
return
def performBSCFS_Analysis(pCtx):
print("%sStart: BSCFS transfer rate calculations..." % (os.linesep))
if "BSCFS" in pCtx:
for l_JobId in pCtx["BSCFS"]:
for l_JobStepId in pCtx["BSCFS"][l_JobId]:
for l_Server in pCtx["BSCFS"][l_JobId][l_JobStepId]:
for l_Connection in pCtx["BSCFS"][l_JobId][l_JobStepId][l_Server]:
l_Order = 1
l_Handles = pCtx["BSCFS"][l_JobId][l_JobStepId][l_Server][l_Connection]
l_AllDone = False
while not l_AllDone:
l_StartTime = [None, None]
for l_Handle in l_Handles.keys():
if l_Handles[l_Handle][0] == 0:
if l_StartTime[1] is not None:
if (cmn.compareTimes(l_StartTime, (l_Handle, l_Handles[l_Handle][3])) == 1):
l_StartTime = [l_Handle, l_Handles[l_Handle][3]]
else:
l_StartTime = [l_Handle, l_Handles[l_Handle][3]]
if l_StartTime[1] is not None:
l_Handles[l_StartTime[0]][0] = l_Order
l_Order += 1
else:
l_AllDone = True
pCtx["BSCFS_TransferRates"] = {}
pCtx["BSCFS_TransferRates"][0] = ("BSCFS Iteration:", "(StartTime", "EndTime", "TotalTransferSize", "TransferRate (GB/sec))")
l_Order = 1
l_AllDone = False
while not l_AllDone:
l_AllDone = True
pCtx["BSCFS_TransferRates"][l_Order] = [None, None, 0, 0.0]
for l_Server in pCtx["BSCFS"][l_JobId][l_JobStepId]:
for l_Connection in pCtx["BSCFS"][l_JobId][l_JobStepId][l_Server]:
l_Handles = pCtx["BSCFS"][l_JobId][l_JobStepId][l_Server][l_Connection]
for l_Handle in l_Handles.keys():
if l_Handles[l_Handle][0] == l_Order:
if pCtx["BSCFS_TransferRates"][l_Order][0] is not None:
if (cmn.compareTimes((None, pCtx["BSCFS_TransferRates"][l_Order][0]), (None, l_Handles[l_Handle][3])) == 1):
pCtx["BSCFS_TransferRates"][l_Order][0] = l_Handles[l_Handle][3]
else:
pCtx["BSCFS_TransferRates"][l_Order][0] = l_Handles[l_Handle][3]
if pCtx["BSCFS_TransferRates"][l_Order][1] is not None:
if (cmn.compareTimes((None, pCtx["BSCFS_TransferRates"][l_Order][1]), (None, l_Handles[l_Handle][4])) == -1):
pCtx["BSCFS_TransferRates"][l_Order][1] = l_Handles[l_Handle][4]
else:
pCtx["BSCFS_TransferRates"][l_Order][1] = l_Handles[l_Handle][4]
pCtx["BSCFS_TransferRates"][l_Order][2] += l_Handles[l_Handle][5]
l_AllDone = False
break
if l_AllDone == False:
l_ElapsedTime = float(cmn.calculateTimeDifferenceInSeconds((None, pCtx["BSCFS_TransferRates"][l_Order][1]), (None, pCtx["BSCFS_TransferRates"][l_Order][0])))
pCtx["BSCFS_TransferRates"][l_Order][3] = round((float(pCtx["BSCFS_TransferRates"][l_Order][2]) / float(l_ElapsedTime)) / float(10**9),6)
l_Order += 1
# Remove the last, as that has no data
if "BSCFS_TransferRates" in pCtx:
pCtx["BSCFS_TransferRates"].pop(max(pCtx["BSCFS_TransferRates"].keys()), None)
if len(pCtx["BSCFS_TransferRates"].keys()) > 0:
# Print the results to a file
l_PathFileName = os.path.join(pCtx["ROOTDIR"], "Analysis", "BSCFS_TransferRates.txt")
cmn.printFormattedFile(pCtx, l_PathFileName, pCtx["BSCFS_TransferRates"])
l_PathFileName = os.path.join(pCtx["ROOTDIR"], "Analysis", "BSCFS_TransferRates.json")
cmn.printFormattedFileAsJson(pCtx, l_PathFileName, pCtx["BSCFS_TransferRates"])
else:
print(" No BSCFS transfers found...")
else:
print(" No BSCFS transfers found...")
print("%s End: BSCFS transfer rate calculations..." % (os.linesep))
return