def getFTSSitesList( self ):
""" get list of FTSSites """
getFTSSitesList = self.ftsManager().getFTSSitesList()
if not getFTSSitesList["OK"]:
self.log.error( "getFTSSitesList: %s" % getFTSSitesList["Message"] )
return getFTSSitesList
getFTSSitesList = getFTSSitesList["Value"]
return S_OK( [ FTSSite( ftsSite ) for ftsSite in getFTSSitesList ] )
def getFTSSite( self, ftsSiteID ):
""" get FTSSite given FTSSiteID """
getFTSSite = self.ftsManager().getFTSSite( ftsSiteID )
if not getFTSSite["OK"]:
self.log.error( "getFTSSite: %s" % getFTSSite["Message"] )
return getFTSSite
getFTSSite = FTSSite( getFTSSite["Value"] ) if getFTSSite["Value"] else None
return S_OK( getFTSSite )
def ftsSites(self):
""" get fts site list """
sites = getSites()
if not sites["OK"]:
return sites
sites = sites["Value"]
ftsServers = getFTSServersForSites(sites)
if not ftsServers["OK"]:
return ftsServers
ftsServers = ftsServers["Value"]
ftsSites = []
for site, ftsServerURL in ftsServers.items():
ftsSite = FTSSite()
ftsSite.Name, ftsSite.FTSServer = site, ftsServerURL
## should be read from CS as well
ftsSite.MaxActiveJobs = 50
ftsSites.append(ftsSite)
return S_OK(ftsSites)
def ftsSites( self ):
""" get fts site list """
sites = getSites()
if not sites["OK"]:
return sites
sites = sites["Value"]
ftsServers = getFTSServersForSites( sites )
if not ftsServers["OK"]:
return ftsServers
ftsServers = ftsServers["Value"]
ftsSites = []
for site, ftsServerURL in ftsServers.items():
ftsSite = FTSSite()
ftsSite.Name, ftsSite.FTSServer = site, ftsServerURL
## should be read from CS as well
ftsSite.MaxActiveJobs = 50
ftsSites.append( ftsSite )
return S_OK( ftsSites )
def test( self ):
""" test case """
ftsSite = FTSSite( self.fromDict )
self.assertEqual( type( ftsSite ), FTSSite, "wrong type" )
for k, v in self.fromDict.items():
self.assertEqual( hasattr( ftsSite, k ), True, "%s attr is missing" % k )
self.assertEqual( getattr( ftsSite, k ), v, "wrong value for attr %s" % k )
# # serialization
# # to JSON
toJSON = ftsSite.toJSON()
self.assertEqual( toJSON["OK"], True, "toJSON failed" )
toJSON = toJSON["Value"]
# # to SQL
toSQL = ftsSite.toSQL()
self.assertEqual( toSQL["OK"], True, "toSQL failed" )
self.assertEqual( toSQL["Value"].startswith( "INSERT" ), True, "toSQL should start with INSERT" )
# # FTSSiteID set
ftsSite.FTSSiteID = 10
self.assertEqual( ftsSite.FTSSiteID, 10, "wrong value for FTSSite" )
# # to SQL again
toSQL = ftsSite.toSQL()
self.assertEqual( toSQL["OK"], True, "toSQL failed" )
self.assertEqual( toSQL["Value"].startswith( "UPDATE" ), True, "toSQL should start with UPDATE" )
def test(self):
""" test case """
ftsSite = FTSSite(
name="something.somewhere.org",
ftsServer="https://something.somewhere.org/FTSService",
maxActiveJobs=100)
self.assertEqual(type(ftsSite), FTSSite, "wrong type")
for k, v in self.fromDict.items():
self.assertEqual(hasattr(ftsSite, k), True,
"%s attr is missing" % k)
self.assertEqual(getattr(ftsSite, k), v,
"wrong value for attr %s" % k)
def setUp(self):
""" test case set up """
# ## set some defaults
gConfig.setOptionValue('DIRAC/Setup', 'Test')
gConfig.setOptionValue('/DIRAC/Setups/Test/DataManagement', 'Test')
gConfig.setOptionValue(
'/Systems/DataManagement/Test/Databases/FTSDB/Host', 'localhost')
gConfig.setOptionValue(
'/Systems/DataManagement/Test/Databases/FTSDB/DBName', 'FTSDB')
gConfig.setOptionValue(
'/Systems/DataManagement/Test/Databases/FTSDB/User', 'Dirac')
self.ftsSites = [
FTSSite({
"FTSServer":
"https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "CERN.ch"
}),
FTSSite({
"FTSServer":
"https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "PIC.es"
}),
FTSSite({
"FTSServer":
"https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "RAL.uk"
}),
FTSSite({
"FTSServer":
"https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "SARA.nl"
}),
FTSSite({
"FTSServer":
"https://fts.cr.cnaf.infn.it:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "CNAF.it"
}),
FTSSite({
"FTSServer":
"https://fts.grid.sara.nl:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "NIKHEF.nl"
}),
FTSSite({
"FTSServer":
"https://fts-fzk.gridka.de:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "GRIDKA.de"
}),
FTSSite({
"FTSServer":
"https://cclcgftsprod.in2p3.fr:8443/glite-data-transfer-fts/services/FileTransfer",
"Name": "IN2P3.fr"
})
]
self.ftsFiles = []
for i in range(100):
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = 9999
ftsFile.LFN = "/a/b/c/%d" % i
ftsFile.Size = 10
ftsFile.SourceSE = "CERN-USER"
ftsFile.TargetSE = "PIC-USER"
ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
ftsFile.Status = "Waiting"
self.ftsFiles.append(ftsFile)
ses = ["CERN-USER", "RAL-USER"]
statuses = [
"Submitted", "Finished", "FinishedDirty", "Active", "Ready"
]
self.submitted = 0
self.ftsJobs = []
for i in range(7200):
ftsJob = FTSJob()
ftsJob.FTSGUID = str(uuid.uuid4())
ftsJob.FTSServer = self.ftsSites[0].FTSServer
ftsJob.Status = statuses[i % len(statuses)]
if ftsJob.Status in FTSJob.FINALSTATES:
ftsJob.Completeness = 100
if ftsJob.Status == "Active":
ftsJob.Completeness = 90
ftsJob.SourceSE = ses[i % len(ses)]
ftsJob.TargetSE = "PIC-USER"
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = i + 1
ftsFile.LFN = "/a/b/c/%d" % i
ftsFile.Size = 1000000
ftsFile.SourceSE = ftsJob.SourceSE
ftsFile.TargetSE = ftsJob.TargetSE
ftsFile.SourceSURL = "foo://source.bar.baz/%s" % ftsFile.LFN
ftsFile.TargetSURL = "foo://target.bar.baz/%s" % ftsFile.LFN
ftsFile.Status = "Waiting" if ftsJob.Status != "FinishedDirty" else "Failed"
ftsFile.FTSGUID = ftsJob.FTSGUID
if ftsJob.Status == "FinishedDirty":
ftsJob.FailedFiles = 1
ftsJob.FailedSize = ftsFile.Size
ftsJob.addFile(ftsFile)
self.ftsJobs.append(ftsJob)
self.submitted = len(
[i for i in self.ftsJobs if i.Status == "Submitted"])
if not getSites["OK"]:
gLogger.error( "unable to read sites defined in CS!!!" )
DIRAC.exit( -1 )
getSites = getSites["Value"]
if "LCG." + ftsSite not in getSites:
gLogger.error( "Site '%s' is not defined in CS Resources/Sites section !!!" % ( "LCG.%s" % ftsSite ) )
DIRAC.exit( -1 )
SEs = gConfig.getOption( "/Resources/Sites/LCG/LCG.%s/SE" % ftsSite , [] )
if not SEs["OK"]:
gLogger.error( "unable to read SEs attached to site LCG.%s: %s" % ftsSite )
DIRAC.exit( -1 )
SEs = SEs["Value"]
newSite = FTSSite()
newSite.Name = ftsSite
newSite.FTSServer = ftsServer
newSite.MaxActiveJobs = maxActiveJobs
putSite = ftsClient.putFTSSite( newSite )
if not putSite["OK"]:
gLogger.error( "unable to put new FTSSite: %s" % putSite["Message"] )
DIRAC.exit( -1 )
gLogger.always( "FTSSite '%s' using FTS server %s and serving %s SEs created" % ( newSite.Name,
newSite.FTSServer,
SEs ) )
DIRAC.exit( 0 )
def setUp(self):
""" test case set up """
gLogger.setLevel('NOTICE')
self.ftsSites = [
FTSSite(
ftsServer=
'https://fts22-t0-export.cern.ch:8443/glite-data-transfer-fts/services/FileTransfer',
name='CERN.ch'),
FTSSite(
ftsServer=
'https://fts.pic.es:8443/glite-data-transfer-fts/services/FileTransfer',
name='PIC.es'),
FTSSite(
ftsServer=
'https://lcgfts.gridpp.rl.ac.uk:8443/glite-data-transfer-fts/services/FileTransfer',
name='RAL.uk'),
]
self.ses = ['CERN-USER', 'RAL-USER']
self.statuses = [
'Submitted', 'Finished', 'FinishedDirty', 'Active', 'Ready'
]
self.submitted = 0
self.numberOfJobs = 10
self.opIDs = []
self.ftsJobs = []
for i in xrange(self.numberOfJobs):
opID = i % 3
if opID not in self.opIDs:
self.opIDs.append(opID)
ftsJob = FTSJob()
ftsJob.FTSGUID = str(uuid.uuid4())
ftsJob.FTSServer = self.ftsSites[0].FTSServer
ftsJob.Status = self.statuses[i % len(self.statuses)]
ftsJob.OperationID = opID
if ftsJob.Status in FTSJob.FINALSTATES:
ftsJob.Completeness = 100
if ftsJob.Status == 'Active':
ftsJob.Completeness = 90
ftsJob.SourceSE = self.ses[i % len(self.ses)]
ftsJob.TargetSE = 'PIC-USER'
ftsJob.RequestID = 12345
ftsFile = FTSFile()
ftsFile.FileID = i + 1
ftsFile.OperationID = i + 1
ftsFile.LFN = '/a/b/c/%d' % i
ftsFile.Size = 1000000
ftsFile.OperationID = opID
ftsFile.SourceSE = ftsJob.SourceSE
ftsFile.TargetSE = ftsJob.TargetSE
ftsFile.SourceSURL = 'foo://source.bar.baz/%s' % ftsFile.LFN
ftsFile.TargetSURL = 'foo://target.bar.baz/%s' % ftsFile.LFN
ftsFile.Status = 'Waiting' if ftsJob.Status != 'FinishedDirty' else 'Failed'
ftsFile.RequestID = 12345
ftsFile.Checksum = 'addler'
ftsFile.ChecksumType = 'adler32'
ftsFile.FTSGUID = ftsJob.FTSGUID
if ftsJob.Status == 'FinishedDirty':
ftsJob.FailedFiles = 1
ftsJob.FailedSize = ftsFile.Size
ftsJob.addFile(ftsFile)
self.ftsJobs.append(ftsJob)
self.submitted = len(
[i for i in self.ftsJobs if i.Status == 'Submitted'])
self.ftsClient = FTSClient()
