def addServerData(pCtx, pLogDate, pLogTime, pJobId, pJobStepId, pServerName,
pConnection, pSizeTransferred):
l_ElapsedTimeJobStepEntry = pCtx["ElapsedTimeData"]["jobIds"][pJobId][
"jobStepIds"][pJobStepId]
l_ElapsedTimeJobStepEntry["NumberOfServers"] += 1
l_ElapsedTimeEntry = l_ElapsedTimeJobStepEntry["servers"]
l_ElapsedTimeEntry[pServerName] = {}
l_ElapsedTimeEntry[pServerName]["StartDateTime"] = (pLogDate, pLogTime)
l_ElapsedTimeEntry[pServerName]["EndDateTime"] = (pLogDate, pLogTime)
l_ElapsedTimeEntry[pServerName]["ElapsedTime (secs)"] = float(
cmn.calculateTimeDifferenceInSeconds(
l_ElapsedTimeEntry[pServerName]["EndDateTime"],
l_ElapsedTimeEntry[pServerName]["StartDateTime"]))
l_ElapsedTimeEntry[pServerName]["SizeTransferred"] = pSizeTransferred
l_ElapsedTimeEntry[pServerName]["NumberOfConnections"] = 0
l_ElapsedTimeEntry[pServerName]["NumberOfContribIds"] = 0
if "BSCFS" in pCtx:
# Second pass through console logs
if pJobId in pCtx["BSCFS"]:
if pJobStepId in pCtx["BSCFS"][pJobId]:
if pServerName not in pCtx["BSCFS"][pJobId][pJobStepId]:
pCtx["BSCFS"][pJobId][pJobStepId][pServerName] = {}
l_ElapsedTimeEntry[pServerName]["connections"] = {}
addConnectionData(pCtx, pLogDate, pLogTime, pJobId, pJobStepId,
pServerName, pConnection, pSizeTransferred)
return
def addConnectionData(pCtx, pLogDate, pLogTime, pJobId, pJobStepId,
pServerName, pConnection, pSizeTransferred):
l_JobStepIdEntry = pCtx["ElapsedTimeData"]["jobIds"][pJobId]["jobStepIds"][
pJobStepId]
l_JobStepIdEntry["NumberOfConnections"] += 1
l_ElapsedTimeServerEntry = l_JobStepIdEntry["servers"][pServerName]
l_ElapsedTimeServerEntry["NumberOfConnections"] += 1
l_ElapsedTimeEntry = l_ElapsedTimeServerEntry["connections"]
l_ElapsedTimeEntry[pConnection] = {}
l_ElapsedTimeEntry[pConnection]["NumberOfContribIds"] = 0
l_ElapsedTimeEntry[pConnection]["StartDateTime"] = (pLogDate, pLogTime)
l_ElapsedTimeEntry[pConnection]["EndDateTime"] = (pLogDate, pLogTime)
l_ElapsedTimeEntry[pConnection]["ElapsedTime (secs)"] = float(
cmn.calculateTimeDifferenceInSeconds(
l_ElapsedTimeEntry[pConnection]["EndDateTime"],
l_ElapsedTimeEntry[pConnection]["StartDateTime"]))
l_ElapsedTimeEntry[pConnection]["SizeTransferred"] = pSizeTransferred
return
def calculateTransferRates(pCtx):
print("%sStart: Transfer rate calculations..." % (os.linesep))
if "jobIds" in pCtx["ElapsedTimeData"]:
for l_JobId in pCtx["ElapsedTimeData"]["jobIds"]:
if "jobStepIds" in pCtx["ElapsedTimeData"]["jobIds"][l_JobId]:
for l_JobStepId in pCtx["ElapsedTimeData"]["jobIds"][l_JobId][
"jobStepIds"]:
l_JobIdEntry = pCtx["ElapsedTimeData"]["jobIds"][l_JobId][
"jobStepIds"][l_JobStepId]
l_ElapsedTime = float(
cmn.calculateTimeDifferenceInSeconds(
l_JobIdEntry["EndDateTime"],
l_JobIdEntry["StartDateTime"]))
l_JobIdEntry["ElapsedTime (secs)"] = l_ElapsedTime
if l_ElapsedTime:
l_JobIdEntry[
"JobId/JobStepId TransferRate (GB/sec)"] = round(
(float(l_JobIdEntry["SizeTransferred"]) /
float(l_ElapsedTime)) / float(10**9), 6)
else:
l_JobIdEntry[
"JobId/JobStepId TransferRate (GB/sec)"] = None
l_JobIdEntry["NumberOfConnections"] = cmn.numericFormat(
l_JobIdEntry["NumberOfConnections"])
l_JobIdEntry["NumberOfContribIds"] = cmn.numericFormat(
l_JobIdEntry["NumberOfContribIds"])
l_JobIdEntry["SizeTransferred"] = cmn.numericFormat(
l_JobIdEntry["SizeTransferred"])
if "servers" in pCtx["ElapsedTimeData"]["jobIds"][l_JobId][
"jobStepIds"][l_JobStepId]:
for l_Server in pCtx["ElapsedTimeData"]["jobIds"][
l_JobId]["jobStepIds"][l_JobStepId]["servers"]:
l_ServerEntry = pCtx["ElapsedTimeData"]["jobIds"][
l_JobId]["jobStepIds"][l_JobStepId]["servers"][
l_Server]
l_ElapsedTime = float(
cmn.calculateTimeDifferenceInSeconds(
l_ServerEntry["EndDateTime"],
l_ServerEntry["StartDateTime"]))
l_ServerEntry["ElapsedTime (secs)"] = l_ElapsedTime
if l_ElapsedTime:
l_ServerEntry[
"Server TransferRate (GB/sec)"] = round(
(float(
l_ServerEntry["SizeTransferred"]) /
float(l_ElapsedTime)) / float(10**9),
6)
else:
l_ServerEntry[
"Server TransferRate (GB/sec)"] = None
l_ServerEntry[
"NumberOfContribIds"] = cmn.numericFormat(
l_ServerEntry["NumberOfContribIds"])
l_ServerEntry[
"SizeTransferred"] = cmn.numericFormat(
l_ServerEntry["SizeTransferred"])
if "connections" in l_ServerEntry:
for l_Connection in l_ServerEntry[
"connections"]:
l_ConnectionEntry = l_ServerEntry[
"connections"][l_Connection]
l_ElapsedTime = float(
cmn.calculateTimeDifferenceInSeconds(
l_ConnectionEntry["EndDateTime"],
l_ConnectionEntry["StartDateTime"])
)
l_ConnectionEntry[
"ElapsedTime (secs)"] = l_ElapsedTime
if l_ElapsedTime:
l_ConnectionEntry[
"Connection TransferRate (GB/sec)"] = round(
(float(l_ConnectionEntry[
"SizeTransferred"]) /
float(l_ElapsedTime)) /
float(10**9), 6)
else:
l_ConnectionEntry[
"Connection TransferRate (GB/sec)"] = None
l_ConnectionEntry[
"NumberOfContribIds"] = cmn.numericFormat(
l_ConnectionEntry[
"NumberOfContribIds"])
l_ConnectionEntry[
"SizeTransferred"] = cmn.numericFormat(
l_ConnectionEntry[
"SizeTransferred"])
# Print the results to a file
l_PathFileName = os.path.join(pCtx["ROOTDIR"], "Analysis",
"TransferRates.txt")
cmn.printFormattedFile(pCtx, l_PathFileName, pCtx["ElapsedTimeData"])
l_PathFileName = os.path.join(pCtx["ROOTDIR"], "Analysis",
"TransferRates.json")
cmn.printFormattedFileAsJson(pCtx, l_PathFileName, pCtx["ElapsedTimeData"])
print("%s End: Transfer rate calculations..." % (os.linesep))
return
def performBSCFS_Analysis(pCtx):
print("%sStart: BSCFS transfer rate calculations..." % (os.linesep))
if "BSCFS" in pCtx:
for l_JobId in pCtx["BSCFS"]:
for l_JobStepId in pCtx["BSCFS"][l_JobId]:
for l_Server in pCtx["BSCFS"][l_JobId][l_JobStepId]:
for l_Connection in pCtx["BSCFS"][l_JobId][l_JobStepId][l_Server]:
l_Order = 1
l_Handles = pCtx["BSCFS"][l_JobId][l_JobStepId][l_Server][l_Connection]
l_AllDone = False
while not l_AllDone:
l_StartTime = [None, None]
for l_Handle in l_Handles.keys():
if l_Handles[l_Handle][0] == 0:
if l_StartTime[1] is not None:
if (cmn.compareTimes(l_StartTime, (l_Handle, l_Handles[l_Handle][3])) == 1):
l_StartTime = [l_Handle, l_Handles[l_Handle][3]]
else:
l_StartTime = [l_Handle, l_Handles[l_Handle][3]]
if l_StartTime[1] is not None:
l_Handles[l_StartTime[0]][0] = l_Order
l_Order += 1
else:
l_AllDone = True
pCtx["BSCFS_TransferRates"] = {}
pCtx["BSCFS_TransferRates"][0] = ("BSCFS Iteration:", "(StartTime", "EndTime", "TotalTransferSize", "TransferRate (GB/sec))")
l_Order = 1
l_AllDone = False
while not l_AllDone:
l_AllDone = True
pCtx["BSCFS_TransferRates"][l_Order] = [None, None, 0, 0.0]
for l_Server in pCtx["BSCFS"][l_JobId][l_JobStepId]:
for l_Connection in pCtx["BSCFS"][l_JobId][l_JobStepId][l_Server]:
l_Handles = pCtx["BSCFS"][l_JobId][l_JobStepId][l_Server][l_Connection]
for l_Handle in l_Handles.keys():
if l_Handles[l_Handle][0] == l_Order:
if pCtx["BSCFS_TransferRates"][l_Order][0] is not None:
if (cmn.compareTimes((None, pCtx["BSCFS_TransferRates"][l_Order][0]), (None, l_Handles[l_Handle][3])) == 1):
pCtx["BSCFS_TransferRates"][l_Order][0] = l_Handles[l_Handle][3]
else:
pCtx["BSCFS_TransferRates"][l_Order][0] = l_Handles[l_Handle][3]
if pCtx["BSCFS_TransferRates"][l_Order][1] is not None:
if (cmn.compareTimes((None, pCtx["BSCFS_TransferRates"][l_Order][1]), (None, l_Handles[l_Handle][4])) == -1):
pCtx["BSCFS_TransferRates"][l_Order][1] = l_Handles[l_Handle][4]
else:
pCtx["BSCFS_TransferRates"][l_Order][1] = l_Handles[l_Handle][4]
pCtx["BSCFS_TransferRates"][l_Order][2] += l_Handles[l_Handle][5]
l_AllDone = False
break
if l_AllDone == False:
l_ElapsedTime = float(cmn.calculateTimeDifferenceInSeconds((None, pCtx["BSCFS_TransferRates"][l_Order][1]), (None, pCtx["BSCFS_TransferRates"][l_Order][0])))
pCtx["BSCFS_TransferRates"][l_Order][3] = round((float(pCtx["BSCFS_TransferRates"][l_Order][2]) / float(l_ElapsedTime)) / float(10**9),6)
l_Order += 1
# Remove the last, as that has no data
if "BSCFS_TransferRates" in pCtx:
pCtx["BSCFS_TransferRates"].pop(max(pCtx["BSCFS_TransferRates"].keys()), None)
if len(pCtx["BSCFS_TransferRates"].keys()) > 0:
# Print the results to a file
l_PathFileName = os.path.join(pCtx["ROOTDIR"], "Analysis", "BSCFS_TransferRates.txt")
cmn.printFormattedFile(pCtx, l_PathFileName, pCtx["BSCFS_TransferRates"])
l_PathFileName = os.path.join(pCtx["ROOTDIR"], "Analysis", "BSCFS_TransferRates.json")
cmn.printFormattedFileAsJson(pCtx, l_PathFileName, pCtx["BSCFS_TransferRates"])
else:
print(" No BSCFS transfers found...")
else:
print(" No BSCFS transfers found...")
print("%s End: BSCFS transfer rate calculations..." % (os.linesep))
return
