def __init__(self, **overrideParams):
self.override = False
self.overrideConf = overrideParams
if overrideParams != {}:
self.override = True
# //
# // Try an get the TFC for the site
#//
self.tfc = None
self.numberOfRetries = 3
self.retryPauseTime = 600
# //
# // If override isnt None, we dont need SiteCfg, if it is
#// then we need siteCfg otherwise we are dead.
if self.override == False:
self.siteCfg = loadSiteLocalConfig()
if self.override:
self.initialiseOverride()
else:
self.initialiseSiteConf()
def __init__(self, **overrideParams):
self.override = False
self.overrideConf = overrideParams
if overrideParams != {}:
self.override = True
self.substituteGUID = True
self.fallbacks = []
# //
# // Try an get the TFC for the site
#//
self.tfc = None
self.numberOfRetries = 3
self.retryPauseTime = 600
# //
# // If override isnt None, we dont need SiteCfg, if it is
#// then we need siteCfg otherwise we are dead.
if self.override == False:
self.siteCfg = loadSiteLocalConfig()
#self.siteCfg.localStageOut['command'] = "rfcp-CERN"
if self.override:
self.initialiseOverride()
else:
self.initialiseSiteConf()
self.failed = {}
self.completedFiles = {}
def loadSiteConfig(self):
"""
_loadSiteConfig_
Load the Site config into this state object
"""
try:
self._SiteConfig = loadSiteLocalConfig()
self.siteConfigLoaded = True
except StandardError, ex:
msg = "Unable to load SiteLocalConfig:\n"
msg += str(ex)
print msg
self._SiteConfig = None
def loadSiteConfig(self):
"""
_loadSiteConfig_
Load the Site config into this state object
"""
try:
self._SiteConfig = loadSiteLocalConfig()
self.siteConfigLoaded = True
except StandardError, ex:
msg = "Unable to load SiteLocalConfig:\n"
msg += str(ex)
print msg
self._SiteConfig = None
def LocalCopy(self, list_retry, results):
"""
Tries the stage out to the CloseSE
"""
if self.debug:
print 'in LocalCopy() :'
print '\t list_retry %s utils '%list_retry
print '\t len(list_retry) %s '%len(list_retry)
list_files = list_retry
self.params['inputFileList']=list_files
### copy backup
from ProdCommon.FwkJobRep.SiteLocalConfig import loadSiteLocalConfig
siteCfg = loadSiteLocalConfig()
catalog = siteCfg.localStageOut.get("catalog", None)
tfc = siteCfg.trivialFileCatalog()
seName = siteCfg.localStageOutSEName()
if seName is None:
seName = ""
option = siteCfg.localStageOutOption()
if option is None:
option = ""
implName = siteCfg.localStageOutCommand()
if implName is None:
implName = ""
if (implName == 'srm'):
protocol = 'srmv2'
elif (implName == 'srmv2-lcg'):
protocol = 'srm-lcg'
option = option + ' -b -D srmv2 '
elif (implName == 'rfcp-CERN'):
protocol = 'rfio'
elif (implName == 'rfcp'):
protocol = 'rfio'
elif (implName == 'cp'):
protocol = 'local'
# the plugin for staging out to LStore is called vandy in storage.xml
elif (implName == 'vandy'):
protocol = 'vandy'
else: protocol = implName
self.params['protocol']=protocol
self.params['option']=option
if self.debug:
print '\t siteCFG %s '%siteCfg
print '\t catalog %s '%catalog
print '\t tfc %s '%tfc
print '\t fallback seName %s '%seName
print "\t fallback protocol %s \n"%protocol
print "\t fallback option %s \n"%option
print "\t self.params['inputFileList'] %s \n"%self.params['inputFileList']
if (str(self.params['for_lfn']).find("/store/") == 0):
temp = str(self.params['for_lfn']).replace("/store/","/store/temp/",1)
self.params['for_lfn']= temp
if ( self.params['for_lfn'][-1] != '/' ) : self.params['for_lfn'] = self.params['for_lfn'] + '/'
file_backup=[]
file_backup_surlgrid=[]
for input in self.params['inputFileList']:
file = self.params['for_lfn'] + os.path.basename(input)
surl = tfc.matchLFN(tfc.preferredProtocol, file)
###### FEDE TEST_FOR_SURL_GRID
surl_for_grid = tfc.matchLFN('srmv2', file)
if (surl_for_grid == None):
surl_for_grid = tfc.matchLFN('srmv2-lcg', file)
if (surl_for_grid == None):
surl_for_grid = tfc.matchLFN('srm', file)
if surl_for_grid:
file_backup_surlgrid.append(surl_for_grid)
######
file_backup.append(surl)
if self.debug:
print '\t for_lfn %s '%self.params['for_lfn']
print '\t file %s '%file
print '\t surl %s '%surl
destination=os.path.dirname(file_backup[0])
if ( destination[-1] != '/' ) : destination = destination + '/'
self.params['destination']=destination
self.params['se_name']=seName
######
if (len(file_backup_surlgrid)>0) :
surl_for_grid=os.path.dirname(file_backup_surlgrid[0])
if ( surl_for_grid[-1] != '/' ) : surl_for_grid = surl_for_grid + '/'
else:
surl_for_grid=''
print "surl_for_grid = ", surl_for_grid
self.params['surl_for_grid']=surl_for_grid
#####
if self.debug:
print '\tIn LocalCopy trying the stage out with: '
print "\tself.params['destination'] %s \n"%self.params['destination']
print "\tself.params['protocol'] %s \n"%self.params['protocol']
print "\tself.params['option'] %s \n"%self.params['option']
localCopy_results = self.copy( self.params['inputFileList'], self.params['protocol'], self.params['option'], backup='yes' )
if localCopy_results.keys() == [''] or localCopy_results.keys() == '' :
results.update(localCopy_results)
else:
localCopy_results, list_ok, list_retry, list_fallback = self.checkCopy(localCopy_results, len(list_files), self.params['protocol'])
results.update(localCopy_results)
if self.debug:
print "\t localCopy_results = %s \n"%localCopy_results
return results
def setProtocol( self, middleware ):
"""
define the allowed potocols based on $middlware
which depend on scheduler
"""
# default To be used with "middleware"
if self.debug:
print 'setProtocol() :\n'
print '\tmiddleware = %s utils \n'%middleware
lcgOpt={'srmv1':'-b -D srmv1 -t 2400 --verbose',
'srmv2':'-b -D srmv2 -t 2400 --verbose'}
if self.checkLcgUtils() >= 17:
lcgOpt={'srmv1':'-b -D srmv1 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose',
'srmv2':'-b -D srmv2 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose'}
srmOpt={'srmv1':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ',
'srmv2':' -report=./srmcp.report -retry_timeout=480000 -retry_num=3 -storagetype=permanent '}
rfioOpt=''
#### FEDE FOR XROOTD #########
xrootdOpt=''
#############################
supported_protocol = None
if middleware.lower() in ['osg','lcg','condor','sge','pbsv2']:
supported_protocol = [('srm-lcg',lcgOpt[self.params['srm_version']])]#,\
# (self.params['srm_version'],srmOpt[self.params['srm_version']])]
#elif middleware.lower() in ['lsf','caf']:
elif middleware.lower() in ['lsf']:
supported_protocol = [('rfio',rfioOpt)]
elif middleware.lower() in ['pbs']:
supported_protocol = [('rfio',rfioOpt),('local','')]
elif middleware.lower() in ['arc']:
supported_protocol = [('srmv2','-debug'),('srmv1','-debug')]
#### FEDE FOR XROOTD ##########
elif middleware.lower() in ['caf']:
if self.params['protocol']:
supported_protocol = [(self.params['protocol'], '')]
else:
supported_protocol = [('rfio',rfioOpt)]
#######################################
else:
## here we can add support for any kind of protocol,
## maybe some local schedulers need something dedicated
pass
# find out where we run and where we stageout
siteCfg = loadSiteLocalConfig()
localSeName = siteCfg.localStageOutSEName()
remoteSeName = self.params['se_name']
seIsLocal = localSeName == remoteSeName
print "******************************************************"
print "localSeName = ",localSeName
print "remoteSeName = ", remoteSeName
print "******************************************************"
# force lstore protocol for Vanderbilt, waiting for better
# long term solution
if seIsLocal and localSeName.endswith('vanderbilt.edu') :
print "*** I am at, and writing-to, Vanderbilt. Try lstore first ***"
supported_protocol.insert(0, ('lstore','') )
return supported_protocol
def LocalCopy(self, list_retry, results):
"""
Tries the stage out to the CloseSE
"""
if self.debug:
print 'in LocalCopy() :\n'
print '\t list_retry %s utils \n'%list_retry
print '\t len(list_retry) %s \n'%len(list_retry)
list_files = list_retry
self.params['inputFileList']=list_files
### copy backup
from ProdCommon.FwkJobRep.SiteLocalConfig import loadSiteLocalConfig
siteCfg = loadSiteLocalConfig()
catalog = siteCfg.localStageOut.get("catalog", None)
tfc = siteCfg.trivialFileCatalog()
seName = siteCfg.localStageOutSEName()
if seName is None:
seName = ""
option = siteCfg.localStageOutOption()
if option is None:
option = ""
implName = siteCfg.localStageOutCommand()
if implName is None:
implName = ""
if (implName == 'srm'):
protocol = 'srmv2'
elif (implName == 'srmv2-lcg'):
protocol = 'srm-lcg'
option = option + ' -b -D srmv2 '
elif (implName == 'rfcp-CERN'):
protocol = 'rfio'
elif (implName == 'rfcp'):
protocol = 'rfio'
elif (implName == 'cp'):
protocol = 'local'
else: protocol = implName
self.params['protocol']=protocol
self.params['option']=option
if self.debug:
print '\t siteCFG %s \n'%siteCfg
print '\t catalog %s \n'%catalog
print '\t tfc %s '%tfc
print '\t fallback seName %s \n'%seName
print "\t fallback protocol %s \n"%protocol
print "\t fallback option %s \n"%option
print "\t self.params['inputFileList'] %s \n"%self.params['inputFileList']
if (str(self.params['for_lfn']).find("/store/") == 0):
temp = str(self.params['for_lfn']).replace("/store/","/store/temp/",1)
self.params['for_lfn']= temp
if ( self.params['for_lfn'][-1] != '/' ) : self.params['for_lfn'] = self.params['for_lfn'] + '/'
file_backup=[]
file_backup_surlgrid=[]
for input in self.params['inputFileList']:
file = self.params['for_lfn'] + os.path.basename(input)
surl = tfc.matchLFN(tfc.preferredProtocol, file)
###### FEDE TEST_FOR_SURL_GRID
surl_for_grid = tfc.matchLFN('srmv2', file)
if (surl_for_grid == None):
surl_for_grid = tfc.matchLFN('srmv2-lcg', file)
if (surl_for_grid == None):
surl_for_grid = tfc.matchLFN('srm', file)
if surl_for_grid:
file_backup_surlgrid.append(surl_for_grid)
######
file_backup.append(surl)
if self.debug:
print '\t for_lfn %s \n'%self.params['for_lfn']
print '\t file %s \n'%file
print '\t surl %s \n'%surl
destination=os.path.dirname(file_backup[0])
if ( destination[-1] != '/' ) : destination = destination + '/'
self.params['destination']=destination
self.params['se_name']=seName
######
if (len(file_backup_surlgrid)>0) :
surl_for_grid=os.path.dirname(file_backup_surlgrid[0])
if ( surl_for_grid[-1] != '/' ) : surl_for_grid = surl_for_grid + '/'
else:
surl_for_grid=''
print "surl_for_grid = ", surl_for_grid
self.params['surl_for_grid']=surl_for_grid
#####
if self.debug:
print '\tIn LocalCopy trying the stage out with: \n'
print "\tself.params['destination'] %s \n"%self.params['destination']
print "\tself.params['protocol'] %s \n"%self.params['protocol']
print "\tself.params['option'] %s \n"%self.params['option']
localCopy_results = self.copy( self.params['inputFileList'], self.params['protocol'], self.params['option'], backup='yes' )
if localCopy_results.keys() == [''] or localCopy_results.keys() == '' :
results.update(localCopy_results)
else:
localCopy_results, list_ok, list_retry, list_fallback = self.checkCopy(localCopy_results, len(list_files), self.params['protocol'])
results.update(localCopy_results)
if self.debug:
print "\t localCopy_results = %s \n"%localCopy_results
return results
def setProtocol( self, middleware ):
"""
define the allowed potocols based on $middlware
which depend on scheduler
"""
# default To be used with "middleware"
if self.debug:
print 'setProtocol() :\n'
print '\tmiddleware = %s utils \n'%middleware
lcgOpt={'srmv1':'-b -D srmv1 -t 2400 --verbose',
'srmv2':'-b -D srmv2 -t 2400 --verbose'}
if self.checkLcgUtils() >= 17:
lcgOpt={'srmv1':'-b -D srmv1 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose',
'srmv2':'-b -D srmv2 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose'}
srmOpt={'srmv1':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ',
'srmv2':' -report=./srmcp.report -retry_timeout=480000 -retry_num=3 -storagetype=permanent '}
rfioOpt=''
#### FEDE FOR XROOTD #########
xrootdOpt=''
#############################
supported_protocol = None
if middleware.lower() in ['osg','lcg','condor','sge','pbsv2', 'slurm']:
supported_protocol = [('srm-lcg',lcgOpt[self.params['srm_version']])]#,\
# (self.params['srm_version'],srmOpt[self.params['srm_version']])]
#elif middleware.lower() in ['lsf','caf']:
elif middleware.lower() in ['lsf']:
supported_protocol = [('rfio',rfioOpt)]
elif middleware.lower() in ['pbs']:
supported_protocol = [('rfio',rfioOpt),('local','')]
elif middleware.lower() in ['arc']:
supported_protocol = [('srmv2','-debug'),('srmv1','-debug')]
#### FEDE FOR XROOTD ##########
elif middleware.lower() in ['caf']:
if self.params['protocol']:
supported_protocol = [(self.params['protocol'], '')]
else:
supported_protocol = [('rfio',rfioOpt)]
#######################################
else:
## here we can add support for any kind of protocol,
## maybe some local schedulers need something dedicated
pass
# find out where we run and where we stageout
siteCfg = loadSiteLocalConfig()
localSeName = siteCfg.localStageOutSEName()
remoteSeName = self.params['se_name']
seIsLocal = localSeName == remoteSeName
print "******************************************************"
print "localSeName = ",localSeName
print "remoteSeName = ", remoteSeName
print "******************************************************"
# force lstore protocol for Vanderbilt, waiting for better
# long term solution
if seIsLocal and localSeName.endswith('vanderbilt.edu') :
print "*** I am at, and writing-to, Vanderbilt. Try lstore first ***"
supported_protocol.insert(0, ('lstore','') )
return supported_protocol
def LocalCopy(self, list_retry, results):
"""
Tries the stage out to the CloseSE
"""
if self.debug:
print "in LocalCopy() :\n"
print "\t list_retry %s utils \n" % list_retry
print "\t len(list_retry) %s \n" % len(list_retry)
list_files = list_retry
self.params["inputFileList"] = list_files
### copy backup
from ProdCommon.FwkJobRep.SiteLocalConfig import loadSiteLocalConfig
siteCfg = loadSiteLocalConfig()
catalog = siteCfg.localStageOut.get("catalog", None)
tfc = siteCfg.trivialFileCatalog()
seName = siteCfg.localStageOutSEName()
if seName is None:
seName = ""
option = siteCfg.localStageOutOption()
if option is None:
option = ""
implName = siteCfg.localStageOutCommand()
if implName is None:
implName = ""
if implName == "srm":
protocol = "srmv2"
elif implName == "srmv2-lcg":
protocol = "srm-lcg"
option = option + " -b -D srmv2 "
elif implName == "rfcp-CERN":
protocol = "rfio"
elif implName == "rfcp":
protocol = "rfio"
elif implName == "cp":
protocol = "local"
else:
protocol = implName
self.params["protocol"] = protocol
self.params["option"] = option
if self.debug:
print "\t siteCFG %s \n" % siteCfg
print "\t catalog %s \n" % catalog
print "\t tfc %s " % tfc
print "\t fallback seName %s \n" % seName
print "\t fallback protocol %s \n" % protocol
print "\t fallback option %s \n" % option
print "\t self.params['inputFileList'] %s \n" % self.params["inputFileList"]
if str(self.params["for_lfn"]).find("/store/") == 0:
temp = str(self.params["for_lfn"]).replace("/store/", "/store/temp/", 1)
self.params["for_lfn"] = temp
if self.params["for_lfn"][-1] != "/":
self.params["for_lfn"] = self.params["for_lfn"] + "/"
file_backup = []
file_backup_surlgrid = []
for input in self.params["inputFileList"]:
file = self.params["for_lfn"] + os.path.basename(input)
surl = tfc.matchLFN(tfc.preferredProtocol, file)
###### FEDE TEST_FOR_SURL_GRID
surl_for_grid = tfc.matchLFN("srmv2", file)
if surl_for_grid == None:
surl_for_grid = tfc.matchLFN("srmv2-lcg", file)
if surl_for_grid == None:
surl_for_grid = tfc.matchLFN("srm", file)
if surl_for_grid:
file_backup_surlgrid.append(surl_for_grid)
######
file_backup.append(surl)
if self.debug:
print "\t for_lfn %s \n" % self.params["for_lfn"]
print "\t file %s \n" % file
print "\t surl %s \n" % surl
destination = os.path.dirname(file_backup[0])
if destination[-1] != "/":
destination = destination + "/"
self.params["destination"] = destination
self.params["se_name"] = seName
######
if len(file_backup_surlgrid) > 0:
surl_for_grid = os.path.dirname(file_backup_surlgrid[0])
if surl_for_grid[-1] != "/":
surl_for_grid = surl_for_grid + "/"
else:
surl_for_grid = ""
print "surl_for_grid = ", surl_for_grid
self.params["surl_for_grid"] = surl_for_grid
#####
if self.debug:
print "\tIn LocalCopy trying the stage out with: \n"
print "\tself.params['destination'] %s \n" % self.params["destination"]
print "\tself.params['protocol'] %s \n" % self.params["protocol"]
print "\tself.params['option'] %s \n" % self.params["option"]
localCopy_results = self.copy(
self.params["inputFileList"], self.params["protocol"], self.params["option"], backup="yes"
)
if localCopy_results.keys() == [""] or localCopy_results.keys() == "":
results.update(localCopy_results)
else:
localCopy_results, list_ok, list_retry, list_fallback = self.checkCopy(
localCopy_results, len(list_files), self.params["protocol"]
)
results.update(localCopy_results)
if self.debug:
print "\t localCopy_results = %s \n" % localCopy_results
return results
def setProtocol(self, middleware):
"""
define the allowed potocols based on $middlware
which depend on scheduler
"""
# default To be used with "middleware"
if self.debug:
print "setProtocol() :\n"
print "\tmiddleware = %s utils \n" % middleware
lcgOpt = {"srmv1": "-b -D srmv1 -t 2400 --verbose", "srmv2": "-b -D srmv2 -t 2400 --verbose"}
if self.checkLcgUtils() >= 17:
lcgOpt = {
"srmv1": "-b -D srmv1 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose",
"srmv2": "-b -D srmv2 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose",
}
srmOpt = {
"srmv1": " -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ",
"srmv2": " -report=./srmcp.report -retry_timeout=480000 -retry_num=3 -storagetype=permanent ",
}
rfioOpt = ""
#### FEDE FOR XROOTD #########
xrootdOpt = ""
#############################
supported_protocol = None
if middleware.lower() in ["osg", "lcg", "condor", "sge"]:
supported_protocol = [("srm-lcg", lcgOpt[self.params["srm_version"]])] # ,\
# (self.params['srm_version'],srmOpt[self.params['srm_version']])]
# elif middleware.lower() in ['lsf','caf']:
elif middleware.lower() in ["lsf"]:
supported_protocol = [("rfio", rfioOpt)]
elif middleware.lower() in ["pbs"]:
supported_protocol = [("rfio", rfioOpt), ("local", "")]
elif middleware.lower() in ["arc"]:
supported_protocol = [("srmv2", "-debug"), ("srmv1", "-debug")]
#### FEDE FOR XROOTD ##########
elif middleware.lower() in ["caf"]:
if self.params["protocol"]:
supported_protocol = [(self.params["protocol"], "")]
else:
supported_protocol = [("rfio", rfioOpt)]
#######################################
else:
## here we can add support for any kind of protocol,
## maybe some local schedulers need something dedicated
pass
# find out where we run and where we stageout
siteCfg = loadSiteLocalConfig()
localSeName = siteCfg.localStageOutSEName()
remoteSeName = self.params["se_name"]
seIsLocal = localSeName == remoteSeName
print "******************************************************"
print "localSeName = ", localSeName
print "remoteSeName = ", remoteSeName
print "******************************************************"
# force lstore protocol for Vanderbilt, waiting for better
# long term solution
if seIsLocal and localSeName.endswith("vanderbilt.edu"):
print "*** I am at, and writing-to, Vanderbilt. Try lstore first ***"
supported_protocol.insert(0, ("lstore", ""))
return supported_protocol