def setUp(self):
#Creating test configuration file
self.testCfgFileName = 'test.cfg'
cfgContent = '''
DIRAC
{
Setup=TestSetup
Setups
{
TestSetup
{
WorkloadManagement=MyWM
}
}
}
Systems
{
WorkloadManagement
{
MyWM
{
URLs
{
Service1 = dips://server1:1234/WorkloadManagement/Service1
Service2 = dips://$MAINSERVERS$:5678/WorkloadManagement/Service2
}
FailoverURLs
{
Service2 = dips://failover1:5678/WorkloadManagement/Service2
}
}
}
}
Operations{
Defaults
{
MainServers = gw1, gw2
}
}
'''
with open(self.testCfgFileName, 'w') as f:
f.write(cfgContent)
gConfig = ConfigurationClient(fileToLoadList=[
self.testCfgFileName
]) #we replace the configuration by our own one.
self.setup = gConfig.getValue('/DIRAC/Setup', '')
self.wm = gConfig.getValue(
'DIRAC/Setups/' + self.setup + '/WorkloadManagement', '')
def setUp(self):
#Creating test configuration file
self.testCfgFileName = 'test.cfg'
cfgContent = '''
DIRAC
{
Setup=TestSetup
Setups
{
TestSetup
{
WorkloadManagement=MyWM
}
}
}
Systems
{
WorkloadManagement
{
MyWM
{
}
}
}
'''
with open(self.testCfgFileName, 'w') as f:
f.write(cfgContent)
gConfig = ConfigurationClient(fileToLoadList=[
self.testCfgFileName
]) #we replace the configuration by our own one.
self.setup = gConfig.getValue('/DIRAC/Setup', '')
self.wm = gConfig.getValue(
'DIRAC/Setups/' + self.setup + '/WorkloadManagement', '')
def setUp( self ):
#Creating test configuration file
self.testCfgFileName = 'test.cfg'
cfgContent='''
DIRAC
{
Setup=TestSetup
Setups
{
TestSetup
{
WorkloadManagement=MyWM
}
}
}
Systems
{
WorkloadManagement
{
MyWM
{
URLs
{
Service1 = dips://server1:1234/WorkloadManagement/Service1
Service2 = dips://$MAINSERVERS$:5678/WorkloadManagement/Service2
}
FailoverURLs
{
Service2 = dips://failover1:5678/WorkloadManagement/Service2
}
}
}
}
Operations{
Defaults
{
MainServers = gw1, gw2
}
}
'''
with open(self.testCfgFileName, 'w') as f:
f.write(cfgContent)
gConfig = ConfigurationClient(fileToLoadList = [self.testCfgFileName]) #we replace the configuration by our own one.
self.setup = gConfig.getValue( '/DIRAC/Setup', '' )
self.wm = gConfig.getValue('DIRAC/Setups/' + self.setup +'/WorkloadManagement', '')
def setUp(self):
# Creating test configuration file
self.testCfgFileName = 'test.cfg'
cfgContent = '''
DIRAC
{
Setup=TestSetup
Setups
{
TestSetup
{
WorkloadManagement=MyWM
}
}
}
Systems
{
WorkloadManagement
{
MyWM
{
URLs
{
Service1 = dips://server1:1234/WorkloadManagement/Service1
Service2 = dips://$MAINSERVERS$:5678/WorkloadManagement/Service2
}
FailoverURLs
{
Service2 = dips://failover1:5678/WorkloadManagement/Service2
}
}
}
}
Operations{
Defaults
{
Pilot
{
Project = LHCb
GenericPilotDN = /DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=doe/CN=111213/CN=Joe Doe
GenericPilotGroup = xxx_pilot
}
MainServers = gw1, gw2
}
}
Registry
{
Users
{
ttester
{
DN = /DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=ttester/CN=696969/CN=Thomas Tester
CA = /DC=ch/DC=cern/CN=CERN Grid Certification Authority
Email = [email protected]
}
franekbolek
{
DN = /DC=ch/DC=voodo/OU=Organic Units/OU=Users/CN=franekbolek/CN=111122/CN=Franek Bolek
CA = /DC=ch/DC=voodo/CN=Voodo Grid Certification Authority
Email = [email protected]
}
}
Groups
{
lhcb_pilot
{
#@@-host - /DC=ch/DC=voodo/OU=computers/CN=brabra.voodo.pl
Users = franekbolek
Users += ttester
Properties = GenericPilot
Properties += LimitedDelegation
VOMSRole = /lhcb/Role=pilot
#@@-ggg@diracAdmin - 2015-07-07 13:40:55
VO = lhcb
}
}
}
'''
with open(self.testCfgFileName, 'w') as f:
f.write(cfgContent)
gConfig = ConfigurationClient(fileToLoadList=[
self.testCfgFileName
]) # we replace the configuration by our own one.
self.setup = gConfig.getValue('/DIRAC/Setup', '')
self.wm = gConfig.getValue(
'DIRAC/Setups/' + self.setup + '/WorkloadManagement', '')
# $HeadURL$
"""
Instantiate the global Configuration Object
gConfig is used everywhere within DIRAC to access Configuration data
"""
__RCSID__ = "$Id$"
from DIRAC.ConfigurationSystem.private.ConfigurationClient import ConfigurationClient
gConfig = ConfigurationClient()
def getConfig():
return gConfig
def config(request):
"""
fixture is the pytest way to declare initalization function.
Scope = module significate that this function will be called only time for this file.
If no scope precised it call config for each test.
This function can have a return value, it will be the value of 'config' argument for the tests
"""
cfgContent = """
DIRAC
{
Setup=TestSetup
Setups
{
TestSetup
{
WorkloadManagement=MyWM
}
}
}
Systems
{
WorkloadManagement
{
MyWM
{
URLs
{
ServiceDips = dips://$MAINSERVERS$:1234/WorkloadManagement/ServiceDips
ServiceHttps = https://$MAINSERVERS$:1234/WorkloadManagement/ServiceHttps
}
}
}
}
Operations{
Defaults
{
MainServers = server1, server2
}
}
"""
with open(testCfgFileName, "w") as f:
f.write(cfgContent)
gConfig = ConfigurationClient(fileToLoadList=[testCfgFileName]) # we replace the configuration by our own one.
# def tearDown():
# Wait for teardown
yield config
"""
This function is called at the end of the test.
"""
try:
os.remove(testCfgFileName)
except OSError:
pass
# SUPER UGLY: one must recreate the CFG objects of gConfigurationData
# not to conflict with other tests that might be using a local dirac.cfg
gConfigurationData.localCFG = CFG()
gConfigurationData.remoteCFG = CFG()
gConfigurationData.mergedCFG = CFG()
gConfigurationData.generateNewVersion()
print("TearDown")
def setUp(self, _mk_generateStorage, _mk_isLocalSE,
_mk_addAccountingOperation):
# Creating test configuration file
self.testCfgFileName = os.path.join(tempfile.gettempdir(),
'test_StorageElement.cfg')
cfgContent = '''
DIRAC
{
Setup=TestSetup
}
Resources{
StorageElements{
DiskStorageA
{
BackendType = local
ReadAccess = Active
WriteAccess = Active
GFAL2_SRM2
{
Host = srm-diskandtape.cern.ch
SpaceToken = Disk
Protocol = srm
Path = /base/pathDisk
}
}
# Same end point as DiskStorageA, but with a different space token
# So they should be considered the same
TapeStorageA
{
BackendType = local
ReadAccess = Active
WriteAccess = Active
GFAL2_SRM2
{
Host = srm-diskandtape.cern.ch
Protocol = srm
SpaceToken = Tape
Path = /base/pathDisk
}
}
# Normally does not happen in practice, but this is the same as DiskStorageA with more plugins
DiskStorageAWithMoreProtocol
{
BackendType = local
ReadAccess = Active
WriteAccess = Active
GFAL2_SRM2
{
Host = srm-diskandtape.cern.ch
SpaceToken = Disk
Protocol = srm
Path = /base/pathDisk
}
GFAL2_GSIFTP
{
Host = gsiftp-diskandtape.cern.ch
SpaceToken = Disk
Protocol = gsiftp
Path = /base/pathDisk
}
}
# A different storage
StorageB
{
BackendType = local
ReadAccess = Active
WriteAccess = Active
GFAL2_GSIFTP
{
Host = otherstorage.cern.ch
SpaceToken = Disk
Protocol = gsiftp
Path = /base/pathDisk
}
}
# The same endpoint as StorageB but with differetn base path, so not the same
StorageBWithOtherBasePath
{
BackendType = local
ReadAccess = Active
WriteAccess = Active
GFAL2_GSIFTP
{
Host = otherstorage.cern.ch
SpaceToken = Disk
Protocol = gsiftp
Path = /base/otherPath
}
}
}
Operations{
Defaults
{
DataManagement{
AccessProtocols = fakeProto
AccessProtocols += root
WriteProtocols = srm
}
}
}
'''
with open(self.testCfgFileName, 'w') as f:
f.write(cfgContent)
# SUPER UGLY: one must recreate the CFG objects of gConfigurationData
# not to conflict with other tests that might be using a local dirac.cfg
gConfigurationData.localCFG = CFG()
gConfigurationData.remoteCFG = CFG()
gConfigurationData.mergedCFG = CFG()
gConfigurationData.generateNewVersion()
gConfig = ConfigurationClient(fileToLoadList=[
self.testCfgFileName
]) # we replace the configuration by our own one.
self.diskStorageA = StorageElementItem('DiskStorageA')
self.diskStorageA.vo = 'lhcb'
self.tapeStorageA = StorageElementItem('TapeStorageA')
self.tapeStorageA.vo = 'lhcb'
self.diskStorageAWithMoreProtocol = StorageElementItem(
'DiskStorageAWithMoreProtocol')
self.diskStorageAWithMoreProtocol.vo = 'lhcb'
self.storageB = StorageElementItem('StorageB')
self.storageB.vo = 'lhcb'
self.storageBWithOtherBasePath = StorageElementItem(
'StorageBWithOtherBasePath')
self.storageBWithOtherBasePath.vo = 'lhcb'
def setUp(self, _mk_generateStorage, _mk_isLocalSE,
_mk_addAccountingOperation):
# Creating test configuration file
self.testCfgFileName = os.path.join(tempfile.gettempdir(),
'test_StorageElement.cfg')
cfgContent = '''
DIRAC
{
Setup=TestSetup
}
Resources{
StorageElements{
StorageA
{
BackendType = local
ReadAccess = Active
WriteAccess = Active
AccessProtocol.0
{
Host =
PluginName = File
Protocol = file
Path =
}
}
StorageB
{
BackendType = local
ReadAccess = Active
WriteAccess = Active
AccessProtocol.0
{
Host =
PluginName = SRM2
Protocol = srm
Path =
}
}
StorageC
{
BackendType = local
ReadAccess = Active
WriteAccess = Active
AccessProtocol.0
{
Host =
PluginName = XROOT
Protocol = root
Path =
}
}
StorageD
{
BackendType = local
ReadAccess = Active
WriteAccess = Active
AccessProtocol.0
{
Host =
PluginName = SRM2
Protocol = srm
Path =
}
AccessProtocol.1
{
Host =
PluginName = XROOT
Protocol = root
Path =
}
}
StorageE
{
BackendType = local
ReadAccess = Active
WriteAccess = Active
WriteProtocols = root
WriteProtocols += srm
AccessProtocol.0
{
Host =
PluginName = SRM2
Protocol = srm
Path =
}
AccessProtocol.1
{
Host =
PluginName = XROOT
Protocol = root
Path =
}
}
StorageX
{
BackendType = local
ReadAccess = Active
WriteAccess = Active
WriteProtocols = gsiftp
AccessProtocols = root
AccessProtocol.0
{
Host =
PluginName = GSIFTP
Protocol = gsiftp
Path =
}
AccessProtocol.1
{
Host =
PluginName = XROOT
Protocol = root
Path =
}
}
StorageY
{
BackendType = local
ReadAccess = Active
WriteAccess = Active
AccessProtocols = gsiftp
AccessProtocols += srm
AccessProtocol.0
{
Host =
PluginName = GSIFTP
Protocol = gsiftp
Path =
}
AccessProtocol.1
{
Host =
PluginName = SRM2
Protocol = srm
Path =
}
}
StorageZ
{
BackendType = local
ReadAccess = Active
WriteAccess = Active
AccessProtocols = root
AccessProtocols += srm
WriteProtocols = root
WriteProtocols += srm
AccessProtocol.0
{
Host =
PluginName = ROOT
Protocol = root
Path =
}
AccessProtocol.1
{
Host =
PluginName = SRM2
Protocol = srm
Path =
}
}
}
}
Operations{
Defaults
{
DataManagement{
AccessProtocols = fakeProto
AccessProtocols += root
WriteProtocols = srm
}
}
}
'''
with open(self.testCfgFileName, 'w') as f:
f.write(cfgContent)
# SUPER UGLY: one must recreate the CFG objects of gConfigurationData
# not to conflict with other tests that might be using a local dirac.cfg
gConfigurationData.localCFG = CFG()
gConfigurationData.remoteCFG = CFG()
gConfigurationData.mergedCFG = CFG()
gConfigurationData.generateNewVersion()
gConfig = ConfigurationClient(fileToLoadList=[
self.testCfgFileName
]) # we replace the configuration by our own one.
self.seA = StorageElementItem('StorageA')
self.seA.vo = 'lhcb'
self.seB = StorageElementItem('StorageB')
self.seB.vo = 'lhcb'
self.seC = StorageElementItem('StorageC')
self.seC.vo = 'lhcb'
self.seD = StorageElementItem('StorageD')
self.seD.vo = 'lhcb'
self.seE = StorageElementItem('StorageE')
self.seE.vo = 'lhcb'
self.seX = StorageElementItem('StorageX')
self.seX.vo = 'lhcb'
self.seY = StorageElementItem('StorageY')
self.seY.vo = 'lhcb'
self.seZ = StorageElementItem('StorageZ')
self.seZ.vo = 'lhcb'
def generateConfig():
"""
Generate the configuration that will be used for all the test
"""
testCfgFileName = os.path.join(tempfile.gettempdir(), "test_FTS3Plugin.cfg")
cfgContent = """
DIRAC
{
VirtualOrganization = lhcb
}
Resources
{
# We define a few SEBases with various protocols
StorageElementBases
{
CERN-Disk
{
BackendType = EOS
GFAL2_HTTPS
{
Host = cerneos.cern.ch
Protocol = https
Path = /eos
Access = remote
}
GFAL2_XROOT
{
Host = cerneos.cern.ch
Protocol = root
Path = /eos
Access = remote
}
}
CERN-Tape
{
BackendType = CTA
SEType = T1D0
# This StageProtocol will triger some multihop staging cases
StageProtocols = root
CTA
{
Host = cerncta.cern.ch
Protocol = root
Path = /eos/ctalhcbpps/archivetest/
Access = remote
}
GFAL2_HTTPS
{
Host = cerncta.cern.ch
Protocol = https
Path = /eos/ctalhcbpps/archivetest/
Access = remote
}
}
RAL-Disk
{
BackendType = Echo
GFAL2_XROOT
{
Host = ralecho.gridpp.uk
Protocol = root
Path = /echo
Access = remote
}
}
RAL-Tape
{
BackendType = Castor
AccessProtocol.0
{
Host =
PluginName = File
Protocol = file
Path =
}
}
CNAF-Disk
{
BackendType = Storm
GFAL2_HTTPS
{
Host = cnafstorm.infc.it
Protocol = https
Path = /storm
Access = remote
}
}
}
StorageElements
{
CERN-DST
{
BaseSE = CERN-Disk
}
CERN-RAW
{
BaseSE = CERN-Tape
}
CNAF-DST
{
BaseSE = CNAF-Disk
}
RAL-DST
{
BaseSE = RAL-Disk
}
}
}
Operations{
Defaults
{
DataManagement
{
AccessProtocols=https,root
WriteProtocols=https,root
ThirdPartyProtocols = https,root
}
}
}
"""
with open(testCfgFileName, "w") as f:
f.write(cfgContent)
# Load the configuration
ConfigurationClient(fileToLoadList=[testCfgFileName]) # we replace the configuration by our own one.
yield
try:
os.remove(testCfgFileName)
except OSError:
pass
# SUPER UGLY: one must recreate the CFG objects of gConfigurationData
# not to conflict with other tests that might be using a local dirac.cfg
gConfigurationData.localCFG = CFG()
gConfigurationData.remoteCFG = CFG()
gConfigurationData.mergedCFG = CFG()
gConfigurationData.generateNewVersion()
def setUp(self):
# Creating test configuration file
self.testCfgFileName = 'test.cfg'
cfgContent = '''
DIRAC
{
Setup=TestSetup
Setups
{
TestSetup
{
WorkloadManagement=MyWM
}
}
}
Systems
{
WorkloadManagement
{
MyWM
{
URLs
{
Service1 = dips://server1:1234/WorkloadManagement/Service1
Service2 = dips://$MAINSERVERS$:5678/WorkloadManagement/Service2
}
FailoverURLs
{
Service2 = dips://failover1:5678/WorkloadManagement/Service2
}
}
}
}
Operations{
Defaults
{
Pilot
{
Project = LHCb
GenericPilotDN = /DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=doe/CN=111213/CN=Joe Doe
GenericPilotGroup = xxx_pilot
}
MainServers = gw1, gw2
}
}
Registry
{
Users
{
ttester
{
DN = /DC=ch/DC=cern/OU=Organic Units/OU=Users/CN=ttester/CN=696969/CN=Thomas Tester
CA = /DC=ch/DC=cern/CN=CERN Grid Certification Authority
Email = [email protected]
}
franekbolek
{
DN = /DC=ch/DC=voodo/OU=Organic Units/OU=Users/CN=franekbolek/CN=111122/CN=Franek Bolek
CA = /DC=ch/DC=voodo/CN=Voodo Grid Certification Authority
Email = [email protected]
}
}
Groups
{
lhcb_pilot
{
#@@-host - /DC=ch/DC=voodo/OU=computers/CN=brabra.voodo.pl
Users = franekbolek
Users += ttester
Properties = GenericPilot
Properties += LimitedDelegation
VOMSRole = /lhcb/Role=pilot
#@@-ggg@diracAdmin - 2015-07-07 13:40:55
VO = lhcb
}
}
}
'''
with open(self.testCfgFileName, 'w') as f:
f.write(cfgContent)
gConfig = ConfigurationClient(fileToLoadList=[self.testCfgFileName]) # we replace the configuration by our own one.
self.setup = gConfig.getValue('/DIRAC/Setup', '')
self.wm = gConfig.getValue('DIRAC/Setups/' + self.setup + '/WorkloadManagement', '')
def generateConfig():
"""
This generates the test configuration once for the module, and removes it when done
"""
# Clean first the config from potential other leaking tests
gConfigurationData.localCFG = CFG()
gConfigurationData.remoteCFG = CFG()
gConfigurationData.mergedCFG = CFG()
gConfigurationData.generateNewVersion()
testCfgFileName = os.path.join(tempfile.gettempdir(),
"test_FTS3Plugin.cfg")
cfgContent = """
Resources
{
StorageElementBases
{
CERN-Disk
{
BackendType = EOS
AccessProtocol.0
{
Host =
PluginName = File
Protocol = file
Path =
}
}
CERN-Tape
{
BackendType = CTA
AccessProtocol.0
{
Host =
PluginName = File
Protocol = file
Path =
}
}
RAL-Disk
{
BackendType = Echo
AccessProtocol.0
{
Host =
PluginName = File
Protocol = file
Path =
}
}
RAL-Tape
{
BackendType = Castor
AccessProtocol.0
{
Host =
PluginName = File
Protocol = file
Path =
}
}
CNAF-Disk
{
BackendType = EOS
AccessProtocol.0
{
Host =
PluginName = File
Protocol = file
Path =
}
}
CNAF-Tape
{
BackendType = CTA
AccessProtocol.0
{
Host =
PluginName = File
Protocol = file
Path =
}
}
IN2P3-Disk
{
BackendType = EOS
AccessProtocol.0
{
Host =
PluginName = File
Protocol = file
Path =
}
}
}
StorageElements
{
CERN-DST
{
BaseSE = CERN-Disk
}
CERN-RAW
{
BaseSE = CERN-Tape
}
RAL-DST
{
BaseSE = RAL-Disk
}
RAL-RAW
{
BaseSE = RAL-Tape
}
CNAF-DST
{
BaseSE = CNAF-Disk
}
CNAF_MC-DST
{
BaseSE = CNAF-Disk
}
IN2P3-DST
{
BaseSE = IN2P3-Disk
}
}
}
Operations{
Defaults
{
DataManagement
{
MultiHopMatrixOfShame
{
# Used for any source which does not have a more specific rule
Default
{
# Default -> Default basically means "anything else than all the other defined routes"
Default = GlobalDefault
# Hop between "anything else" and IN3P3-DST
IN2P3-DST = DefaultToIN2P3-DST
# Hop between "anything else" and any SE inheriting from CNAF-Disk
CNAF-Disk = DefaultToCNAF-Disk
}
# Any transfer starting from CERN-RAW
CERN-RAW
{
# CERN-RAW -> anywhere else
Default = DefaultFromCERN-RAW
# Do not use multihop between CERN-RAW and SE inheriting from CERN-Disk
CERN-Disk = disabled
# CERN-RAW -> any SE inheriting from CNAF-Disk
CNAF-Disk = CERN-RAW-CNAF-Disk
# CERN-RAW->CNAF-DST (takes precedence over CERN-RAW -> CNAF-Disk)
CNAF-DST = CERN-RAW-CNAF-DST
# CERN-RAW -> IN2P3-DST
IN2P3-DST = disabled
}
# Any transfer starting from any SE inheriting from RAL-Tape
RAL-Tape
{
# RAL-Tape -> anywhere else: do not use multihop
Default = disabled
# any SE inheriting from RAL-Tape -> any SE inheriting from CNAF-Disk
CNAF-Disk = RAL-Tape-CNAF-Disk
# any SE inheriting from RAL-Tape -> CNAF-DST (takes precedence over the previous rule)
CNAF-DST = RAL-Tape-CNAF-DST
}
# Any transfer starting from IN2P3-DST
IN2P3-DST
{
# IN2P2-DST -> CNAF-DST
CNAF-DST = IN2P3-DST-CNAF-DST
}
}
}
}
}
"""
with open(testCfgFileName, "w") as f:
f.write(cfgContent)
# Load the configuration
ConfigurationClient(fileToLoadList=[
testCfgFileName
]) # we replace the configuration by our own one.
yield
try:
os.remove(testCfgFileName)
except OSError:
pass
# SUPER UGLY: one must recreate the CFG objects of gConfigurationData
# not to conflict with other tests that might be using a local dirac.cfg
gConfigurationData.localCFG = CFG()
gConfigurationData.remoteCFG = CFG()
gConfigurationData.mergedCFG = CFG()
gConfigurationData.generateNewVersion()