def setup(self, channels, bandwithds, failedFiles):
""" prepare fts graph
:param dict channels: { channelID : { "Files" : long , Size = long, "ChannelName" : str,
"Source" : str, "Destination" : str , "ChannelName" : str, "Status" : str } }
:param dict bandwidths: { channelID { "Throughput" : float, "Fileput" : float, "SucessfulFiles" : long, "FailedFiles" : long } }
:param dict failedFiles: { channelID : int }
channelInfo { channelName : { "ChannelID" : int, "TimeToStart" : float} }
"""
graph = FTSGraph("sites")
result = getStorageElementSiteMapping()
if not result["OK"]:
return result
sitesDict = result["Value"]
## create nodes
for site, ses in sitesDict.items():
rwDict = self.__getRWAccessForSE(ses)
if not rwDict["OK"]:
return rwDict
siteName = site
if "." in site:
siteName = site.split(".")[1]
graph.addNode(LCGSite(siteName, {"SEs": rwDict["Value"]}))
## channels { channelID : { "Files" : long , Size = long, "ChannelName" : str,
## "Source" : str, "Destination" : str ,
## "ChannelName" : str, "Status" : str } }
## bandwidths { channelID { "Throughput" : float, "Fileput" : float,
## "SucessfulFiles" : long, "FailedFiles" : long } }
## channelInfo { channelName : { "ChannelID" : int, "TimeToStart" : float} }
for channelID, channelDict in channels.items():
sourceName = channelDict["Source"]
destName = channelDict["Destination"]
fromNode = graph.getNode(sourceName)
toNode = graph.getNode(destName)
if fromNode and toNode:
rwAttrs = {
"status": channels[channelID]["Status"],
"files": channelDict["Files"],
"size": channelDict["Size"],
"successfulAttempts": bandwithds[channelID]["SuccessfulFiles"],
"failedAttempts": bandwithds[channelID]["FailedFiles"],
"distinctFailedFiles": failedFiles.get(channelID, 0),
"fileput": bandwithds[channelID]["Fileput"],
"throughput": bandwithds[channelID]["Throughput"],
}
roAttrs = {
"channelID": channelID,
"channelName": channelDict["ChannelName"],
"acceptableFailureRate": self.acceptableFailureRate,
"acceptableFailedFiles": self.acceptableFailedFiles,
"schedulingType": self.schedulingType,
}
ftsChannel = FTSChannel(fromNode, toNode, rwAttrs, roAttrs)
graph.addEdge(ftsChannel)
self.ftsGraph = graph
self.lastRssUpdate = datetime.datetime.now()
return S_OK()
def initialize( self, ftsHistoryViews = None ):
""" initialize FTSGraph given FTSSites and FTSHistoryViews
:param list ftsSites: list with FTSSites instances
:param list ftsHistoryViews: list with FTSHistoryViews instances
"""
self.log.debug( "initializing FTS graph..." )
ftsSites = self.ftsSites()
if ftsSites["OK"]:
ftsSites = ftsSites["Value"]
else:
ftsSites = []
ftsHistoryViews = ftsHistoryViews if ftsHistoryViews else []
sitesDict = getStorageElementSiteMapping() # [ ftsSite.Name for ftsSite in ftsSites ] )
if not sitesDict["OK"]:
self.log.error( sitesDict["Message"] )
# raise Exception( sitesDict["Message"] )
sitesDict = sitesDict["Value"] if "Value" in sitesDict else {}
# # revert to resources helper
# sitesDict = self.resources().getEligibleResources( "Storage" )
# if not sitesDict["OK"]:
# return sitesDict
# sitesDict = sitesDict["Value"]
# # create nodes
for ftsSite in ftsSites:
rwSEsDict = dict.fromkeys( sitesDict.get( ftsSite.Name, [] ), {} )
for se in rwSEsDict:
rwSEsDict[se] = { "read": False, "write": False }
rwAttrs = { "SEs": rwSEsDict }
roAttrs = { "FTSServer": ftsSite.FTSServer,
"MaxActiveJobs": ftsSite.MaxActiveJobs }
site = Site( ftsSite.Name, rwAttrs, roAttrs )
self.log.debug( "adding site %s using FTSServer %s" % ( ftsSite.Name, ftsSite.FTSServer ) )
self.addNode( site )
for sourceSite in self.nodes():
for destSite in self.nodes():
rwAttrs = { "WaitingFiles": 0, "WaitingSize": 0,
"SuccessfulFiles": 0, "SuccessfulSize": 0,
"FailedFiles": 0, "FailedSize": 0,
"FilePut": 0.0, "ThroughPut": 0.0,
"ActiveJobs": 0, "FinishedJobs": 0 }
roAttrs = { "routeName": "%s#%s" % ( sourceSite.name, destSite.name ),
"AcceptableFailureRate": self.accFailureRate,
"AcceptableFailedFiles": self.accFailedFiles,
"SchedulingType": self.schedulingType }
route = Route( sourceSite, destSite, rwAttrs, roAttrs )
self.log.debug( "adding route between %s and %s" % ( route.fromNode.name, route.toNode.name ) )
self.addEdge( route )
for ftsHistory in ftsHistoryViews:
route = self.findRoute( ftsHistory.SourceSE, ftsHistory.TargetSE )
if not route["OK"]:
self.log.warn( "route between %s and %s not found" % ( ftsHistory.SourceSE, ftsHistory.TargetSE ) )
continue
route = route["Value"]
if ftsHistory.Status in FTSJob.INITSTATES:
route.ActiveJobs += ftsHistory.FTSJobs
route.WaitingFiles += ftsHistory.Files
route.WaitingSize += ftsHistory.Size
elif ftsHistory.Status in FTSJob.TRANSSTATES:
route.ActiveJobs += ftsHistory.FTSJobs
route.WaitingSize += ftsHistory.Completeness * ftsHistory.Size / 100.0
route.WaitingFiles += int( ftsHistory.Completeness * ftsHistory.Files / 100.0 )
elif ftsHistory.Status in FTSJob.FAILEDSTATES:
route.FinishedJobs += ftsHistory.FTSJobs
route.FailedFiles += ftsHistory.FailedFiles
route.FailedSize += ftsHistory.FailedSize
else: # # FINISHEDSTATES
route.FinishedJobs += ftsHistory.FTSJobs
route.SuccessfulFiles += ( ftsHistory.Files - ftsHistory.FailedFiles )
route.SuccessfulSize += ( ftsHistory.Size - ftsHistory.FailedSize )
route.FilePut = float( route.SuccessfulFiles - route.FailedFiles ) / FTSHistoryView.INTERVAL
route.ThroughPut = float( route.SuccessfulSize - route.FailedSize ) / FTSHistoryView.INTERVAL
self.updateRWAccess()
self.log.debug( "init done!" )
def setup(self, channels, bandwithds, failedFiles):
""" prepare fts graph
:param dict channels: { channelID : { "Files" : long , Size = long, "ChannelName" : str,
"Source" : str, "Destination" : str , "ChannelName" : str, "Status" : str } }
:param dict bandwidths: { channelID { "Throughput" : float, "Fileput" : float, "SucessfulFiles" : long, "FailedFiles" : long } }
:param dict failedFiles: { channelID : int }
channelInfo { channelName : { "ChannelID" : int, "TimeToStart" : float} }
"""
graph = FTSGraph("sites")
result = getStorageElementSiteMapping()
if not result['OK']:
return result
sitesDict = result['Value']
## create nodes
for site, ses in sitesDict.items():
rwDict = self.__getRWAccessForSE(ses)
if not rwDict["OK"]:
return rwDict
siteName = site
if '.' in site:
siteName = site.split('.')[1]
graph.addNode(LCGSite(siteName, {"SEs": rwDict["Value"]}))
## channels { channelID : { "Files" : long , Size = long, "ChannelName" : str,
## "Source" : str, "Destination" : str ,
## "ChannelName" : str, "Status" : str } }
## bandwidths { channelID { "Throughput" : float, "Fileput" : float,
## "SucessfulFiles" : long, "FailedFiles" : long } }
## channelInfo { channelName : { "ChannelID" : int, "TimeToStart" : float} }
for channelID, channelDict in channels.items():
sourceName = channelDict["Source"]
destName = channelDict["Destination"]
fromNode = graph.getNode(sourceName)
toNode = graph.getNode(destName)
if fromNode and toNode:
rwAttrs = {
"status": channels[channelID]["Status"],
"files": channelDict["Files"],
"size": channelDict["Size"],
"successfulAttempts":
bandwithds[channelID]["SuccessfulFiles"],
"failedAttempts": bandwithds[channelID]["FailedFiles"],
"distinctFailedFiles": failedFiles.get(channelID, 0),
"fileput": bandwithds[channelID]["Fileput"],
"throughput": bandwithds[channelID]["Throughput"]
}
roAttrs = {
"channelID": channelID,
"channelName": channelDict["ChannelName"],
"acceptableFailureRate": self.acceptableFailureRate,
"acceptableFailedFiles": self.acceptableFailedFiles,
"schedulingType": self.schedulingType
}
ftsChannel = FTSChannel(fromNode, toNode, rwAttrs, roAttrs)
graph.addEdge(ftsChannel)
self.ftsGraph = graph
self.lastRssUpdate = datetime.datetime.now()
return S_OK()