def __init__(self, **overrideParams): self.override = False self.overrideConf = overrideParams if overrideParams != {}: self.override = True # // # // Try an get the TFC for the site #// self.tfc = None self.numberOfRetries = 3 self.retryPauseTime = 600 # // # // If override isnt None, we dont need SiteCfg, if it is #// then we need siteCfg otherwise we are dead. if self.override == False: self.siteCfg = loadSiteLocalConfig() if self.override: self.initialiseOverride() else: self.initialiseSiteConf()
def __init__(self, **overrideParams): self.override = False self.overrideConf = overrideParams if overrideParams != {}: self.override = True self.substituteGUID = True self.fallbacks = [] # // # // Try an get the TFC for the site #// self.tfc = None self.numberOfRetries = 3 self.retryPauseTime = 600 # // # // If override isnt None, we dont need SiteCfg, if it is #// then we need siteCfg otherwise we are dead. if self.override == False: self.siteCfg = loadSiteLocalConfig() #self.siteCfg.localStageOut['command'] = "rfcp-CERN" if self.override: self.initialiseOverride() else: self.initialiseSiteConf() self.failed = {} self.completedFiles = {}
def loadSiteConfig(self): """ _loadSiteConfig_ Load the Site config into this state object """ try: self._SiteConfig = loadSiteLocalConfig() self.siteConfigLoaded = True except StandardError, ex: msg = "Unable to load SiteLocalConfig:\n" msg += str(ex) print msg self._SiteConfig = None
def LocalCopy(self, list_retry, results): """ Tries the stage out to the CloseSE """ if self.debug: print 'in LocalCopy() :' print '\t list_retry %s utils '%list_retry print '\t len(list_retry) %s '%len(list_retry) list_files = list_retry self.params['inputFileList']=list_files ### copy backup from ProdCommon.FwkJobRep.SiteLocalConfig import loadSiteLocalConfig siteCfg = loadSiteLocalConfig() catalog = siteCfg.localStageOut.get("catalog", None) tfc = siteCfg.trivialFileCatalog() seName = siteCfg.localStageOutSEName() if seName is None: seName = "" option = siteCfg.localStageOutOption() if option is None: option = "" implName = siteCfg.localStageOutCommand() if implName is None: implName = "" if (implName == 'srm'): protocol = 'srmv2' elif (implName == 'srmv2-lcg'): protocol = 'srm-lcg' option = option + ' -b -D srmv2 ' elif (implName == 'rfcp-CERN'): protocol = 'rfio' elif (implName == 'rfcp'): protocol = 'rfio' elif (implName == 'cp'): protocol = 'local' # the plugin for staging out to LStore is called vandy in storage.xml elif (implName == 'vandy'): protocol = 'vandy' else: protocol = implName self.params['protocol']=protocol self.params['option']=option if self.debug: print '\t siteCFG %s '%siteCfg print '\t catalog %s '%catalog print '\t tfc %s '%tfc print '\t fallback seName %s '%seName print "\t fallback protocol %s \n"%protocol print "\t fallback option %s \n"%option print "\t self.params['inputFileList'] %s \n"%self.params['inputFileList'] if (str(self.params['for_lfn']).find("/store/") == 0): temp = str(self.params['for_lfn']).replace("/store/","/store/temp/",1) self.params['for_lfn']= temp if ( self.params['for_lfn'][-1] != '/' ) : self.params['for_lfn'] = self.params['for_lfn'] + '/' file_backup=[] file_backup_surlgrid=[] for input in self.params['inputFileList']: file = self.params['for_lfn'] + os.path.basename(input) surl = tfc.matchLFN(tfc.preferredProtocol, file) ###### FEDE TEST_FOR_SURL_GRID surl_for_grid = tfc.matchLFN('srmv2', file) if (surl_for_grid == None): surl_for_grid = tfc.matchLFN('srmv2-lcg', file) if (surl_for_grid == None): surl_for_grid = tfc.matchLFN('srm', file) if surl_for_grid: file_backup_surlgrid.append(surl_for_grid) ###### file_backup.append(surl) if self.debug: print '\t for_lfn %s '%self.params['for_lfn'] print '\t file %s '%file print '\t surl %s '%surl destination=os.path.dirname(file_backup[0]) if ( destination[-1] != '/' ) : destination = destination + '/' self.params['destination']=destination self.params['se_name']=seName ###### if (len(file_backup_surlgrid)>0) : surl_for_grid=os.path.dirname(file_backup_surlgrid[0]) if ( surl_for_grid[-1] != '/' ) : surl_for_grid = surl_for_grid + '/' else: surl_for_grid='' print "surl_for_grid = ", surl_for_grid self.params['surl_for_grid']=surl_for_grid ##### if self.debug: print '\tIn LocalCopy trying the stage out with: ' print "\tself.params['destination'] %s \n"%self.params['destination'] print "\tself.params['protocol'] %s \n"%self.params['protocol'] print "\tself.params['option'] %s \n"%self.params['option'] localCopy_results = self.copy( self.params['inputFileList'], self.params['protocol'], self.params['option'], backup='yes' ) if localCopy_results.keys() == [''] or localCopy_results.keys() == '' : results.update(localCopy_results) else: localCopy_results, list_ok, list_retry, list_fallback = self.checkCopy(localCopy_results, len(list_files), self.params['protocol']) results.update(localCopy_results) if self.debug: print "\t localCopy_results = %s \n"%localCopy_results return results
def setProtocol( self, middleware ): """ define the allowed potocols based on $middlware which depend on scheduler """ # default To be used with "middleware" if self.debug: print 'setProtocol() :\n' print '\tmiddleware = %s utils \n'%middleware lcgOpt={'srmv1':'-b -D srmv1 -t 2400 --verbose', 'srmv2':'-b -D srmv2 -t 2400 --verbose'} if self.checkLcgUtils() >= 17: lcgOpt={'srmv1':'-b -D srmv1 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose', 'srmv2':'-b -D srmv2 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose'} srmOpt={'srmv1':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ', 'srmv2':' -report=./srmcp.report -retry_timeout=480000 -retry_num=3 -storagetype=permanent '} rfioOpt='' #### FEDE FOR XROOTD ######### xrootdOpt='' ############################# supported_protocol = None if middleware.lower() in ['osg','lcg','condor','sge','pbsv2']: supported_protocol = [('srm-lcg',lcgOpt[self.params['srm_version']])]#,\ # (self.params['srm_version'],srmOpt[self.params['srm_version']])] #elif middleware.lower() in ['lsf','caf']: elif middleware.lower() in ['lsf']: supported_protocol = [('rfio',rfioOpt)] elif middleware.lower() in ['pbs']: supported_protocol = [('rfio',rfioOpt),('local','')] elif middleware.lower() in ['arc']: supported_protocol = [('srmv2','-debug'),('srmv1','-debug')] #### FEDE FOR XROOTD ########## elif middleware.lower() in ['caf']: if self.params['protocol']: supported_protocol = [(self.params['protocol'], '')] else: supported_protocol = [('rfio',rfioOpt)] ####################################### else: ## here we can add support for any kind of protocol, ## maybe some local schedulers need something dedicated pass # find out where we run and where we stageout siteCfg = loadSiteLocalConfig() localSeName = siteCfg.localStageOutSEName() remoteSeName = self.params['se_name'] seIsLocal = localSeName == remoteSeName print "******************************************************" print "localSeName = ",localSeName print "remoteSeName = ", remoteSeName print "******************************************************" # force lstore protocol for Vanderbilt, waiting for better # long term solution if seIsLocal and localSeName.endswith('vanderbilt.edu') : print "*** I am at, and writing-to, Vanderbilt. Try lstore first ***" supported_protocol.insert(0, ('lstore','') ) return supported_protocol
def LocalCopy(self, list_retry, results): """ Tries the stage out to the CloseSE """ if self.debug: print 'in LocalCopy() :\n' print '\t list_retry %s utils \n'%list_retry print '\t len(list_retry) %s \n'%len(list_retry) list_files = list_retry self.params['inputFileList']=list_files ### copy backup from ProdCommon.FwkJobRep.SiteLocalConfig import loadSiteLocalConfig siteCfg = loadSiteLocalConfig() catalog = siteCfg.localStageOut.get("catalog", None) tfc = siteCfg.trivialFileCatalog() seName = siteCfg.localStageOutSEName() if seName is None: seName = "" option = siteCfg.localStageOutOption() if option is None: option = "" implName = siteCfg.localStageOutCommand() if implName is None: implName = "" if (implName == 'srm'): protocol = 'srmv2' elif (implName == 'srmv2-lcg'): protocol = 'srm-lcg' option = option + ' -b -D srmv2 ' elif (implName == 'rfcp-CERN'): protocol = 'rfio' elif (implName == 'rfcp'): protocol = 'rfio' elif (implName == 'cp'): protocol = 'local' else: protocol = implName self.params['protocol']=protocol self.params['option']=option if self.debug: print '\t siteCFG %s \n'%siteCfg print '\t catalog %s \n'%catalog print '\t tfc %s '%tfc print '\t fallback seName %s \n'%seName print "\t fallback protocol %s \n"%protocol print "\t fallback option %s \n"%option print "\t self.params['inputFileList'] %s \n"%self.params['inputFileList'] if (str(self.params['for_lfn']).find("/store/") == 0): temp = str(self.params['for_lfn']).replace("/store/","/store/temp/",1) self.params['for_lfn']= temp if ( self.params['for_lfn'][-1] != '/' ) : self.params['for_lfn'] = self.params['for_lfn'] + '/' file_backup=[] file_backup_surlgrid=[] for input in self.params['inputFileList']: file = self.params['for_lfn'] + os.path.basename(input) surl = tfc.matchLFN(tfc.preferredProtocol, file) ###### FEDE TEST_FOR_SURL_GRID surl_for_grid = tfc.matchLFN('srmv2', file) if (surl_for_grid == None): surl_for_grid = tfc.matchLFN('srmv2-lcg', file) if (surl_for_grid == None): surl_for_grid = tfc.matchLFN('srm', file) if surl_for_grid: file_backup_surlgrid.append(surl_for_grid) ###### file_backup.append(surl) if self.debug: print '\t for_lfn %s \n'%self.params['for_lfn'] print '\t file %s \n'%file print '\t surl %s \n'%surl destination=os.path.dirname(file_backup[0]) if ( destination[-1] != '/' ) : destination = destination + '/' self.params['destination']=destination self.params['se_name']=seName ###### if (len(file_backup_surlgrid)>0) : surl_for_grid=os.path.dirname(file_backup_surlgrid[0]) if ( surl_for_grid[-1] != '/' ) : surl_for_grid = surl_for_grid + '/' else: surl_for_grid='' print "surl_for_grid = ", surl_for_grid self.params['surl_for_grid']=surl_for_grid ##### if self.debug: print '\tIn LocalCopy trying the stage out with: \n' print "\tself.params['destination'] %s \n"%self.params['destination'] print "\tself.params['protocol'] %s \n"%self.params['protocol'] print "\tself.params['option'] %s \n"%self.params['option'] localCopy_results = self.copy( self.params['inputFileList'], self.params['protocol'], self.params['option'], backup='yes' ) if localCopy_results.keys() == [''] or localCopy_results.keys() == '' : results.update(localCopy_results) else: localCopy_results, list_ok, list_retry, list_fallback = self.checkCopy(localCopy_results, len(list_files), self.params['protocol']) results.update(localCopy_results) if self.debug: print "\t localCopy_results = %s \n"%localCopy_results return results
def setProtocol( self, middleware ): """ define the allowed potocols based on $middlware which depend on scheduler """ # default To be used with "middleware" if self.debug: print 'setProtocol() :\n' print '\tmiddleware = %s utils \n'%middleware lcgOpt={'srmv1':'-b -D srmv1 -t 2400 --verbose', 'srmv2':'-b -D srmv2 -t 2400 --verbose'} if self.checkLcgUtils() >= 17: lcgOpt={'srmv1':'-b -D srmv1 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose', 'srmv2':'-b -D srmv2 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose'} srmOpt={'srmv1':' -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ', 'srmv2':' -report=./srmcp.report -retry_timeout=480000 -retry_num=3 -storagetype=permanent '} rfioOpt='' #### FEDE FOR XROOTD ######### xrootdOpt='' ############################# supported_protocol = None if middleware.lower() in ['osg','lcg','condor','sge','pbsv2', 'slurm']: supported_protocol = [('srm-lcg',lcgOpt[self.params['srm_version']])]#,\ # (self.params['srm_version'],srmOpt[self.params['srm_version']])] #elif middleware.lower() in ['lsf','caf']: elif middleware.lower() in ['lsf']: supported_protocol = [('rfio',rfioOpt)] elif middleware.lower() in ['pbs']: supported_protocol = [('rfio',rfioOpt),('local','')] elif middleware.lower() in ['arc']: supported_protocol = [('srmv2','-debug'),('srmv1','-debug')] #### FEDE FOR XROOTD ########## elif middleware.lower() in ['caf']: if self.params['protocol']: supported_protocol = [(self.params['protocol'], '')] else: supported_protocol = [('rfio',rfioOpt)] ####################################### else: ## here we can add support for any kind of protocol, ## maybe some local schedulers need something dedicated pass # find out where we run and where we stageout siteCfg = loadSiteLocalConfig() localSeName = siteCfg.localStageOutSEName() remoteSeName = self.params['se_name'] seIsLocal = localSeName == remoteSeName print "******************************************************" print "localSeName = ",localSeName print "remoteSeName = ", remoteSeName print "******************************************************" # force lstore protocol for Vanderbilt, waiting for better # long term solution if seIsLocal and localSeName.endswith('vanderbilt.edu') : print "*** I am at, and writing-to, Vanderbilt. Try lstore first ***" supported_protocol.insert(0, ('lstore','') ) return supported_protocol
def LocalCopy(self, list_retry, results): """ Tries the stage out to the CloseSE """ if self.debug: print "in LocalCopy() :\n" print "\t list_retry %s utils \n" % list_retry print "\t len(list_retry) %s \n" % len(list_retry) list_files = list_retry self.params["inputFileList"] = list_files ### copy backup from ProdCommon.FwkJobRep.SiteLocalConfig import loadSiteLocalConfig siteCfg = loadSiteLocalConfig() catalog = siteCfg.localStageOut.get("catalog", None) tfc = siteCfg.trivialFileCatalog() seName = siteCfg.localStageOutSEName() if seName is None: seName = "" option = siteCfg.localStageOutOption() if option is None: option = "" implName = siteCfg.localStageOutCommand() if implName is None: implName = "" if implName == "srm": protocol = "srmv2" elif implName == "srmv2-lcg": protocol = "srm-lcg" option = option + " -b -D srmv2 " elif implName == "rfcp-CERN": protocol = "rfio" elif implName == "rfcp": protocol = "rfio" elif implName == "cp": protocol = "local" else: protocol = implName self.params["protocol"] = protocol self.params["option"] = option if self.debug: print "\t siteCFG %s \n" % siteCfg print "\t catalog %s \n" % catalog print "\t tfc %s " % tfc print "\t fallback seName %s \n" % seName print "\t fallback protocol %s \n" % protocol print "\t fallback option %s \n" % option print "\t self.params['inputFileList'] %s \n" % self.params["inputFileList"] if str(self.params["for_lfn"]).find("/store/") == 0: temp = str(self.params["for_lfn"]).replace("/store/", "/store/temp/", 1) self.params["for_lfn"] = temp if self.params["for_lfn"][-1] != "/": self.params["for_lfn"] = self.params["for_lfn"] + "/" file_backup = [] file_backup_surlgrid = [] for input in self.params["inputFileList"]: file = self.params["for_lfn"] + os.path.basename(input) surl = tfc.matchLFN(tfc.preferredProtocol, file) ###### FEDE TEST_FOR_SURL_GRID surl_for_grid = tfc.matchLFN("srmv2", file) if surl_for_grid == None: surl_for_grid = tfc.matchLFN("srmv2-lcg", file) if surl_for_grid == None: surl_for_grid = tfc.matchLFN("srm", file) if surl_for_grid: file_backup_surlgrid.append(surl_for_grid) ###### file_backup.append(surl) if self.debug: print "\t for_lfn %s \n" % self.params["for_lfn"] print "\t file %s \n" % file print "\t surl %s \n" % surl destination = os.path.dirname(file_backup[0]) if destination[-1] != "/": destination = destination + "/" self.params["destination"] = destination self.params["se_name"] = seName ###### if len(file_backup_surlgrid) > 0: surl_for_grid = os.path.dirname(file_backup_surlgrid[0]) if surl_for_grid[-1] != "/": surl_for_grid = surl_for_grid + "/" else: surl_for_grid = "" print "surl_for_grid = ", surl_for_grid self.params["surl_for_grid"] = surl_for_grid ##### if self.debug: print "\tIn LocalCopy trying the stage out with: \n" print "\tself.params['destination'] %s \n" % self.params["destination"] print "\tself.params['protocol'] %s \n" % self.params["protocol"] print "\tself.params['option'] %s \n" % self.params["option"] localCopy_results = self.copy( self.params["inputFileList"], self.params["protocol"], self.params["option"], backup="yes" ) if localCopy_results.keys() == [""] or localCopy_results.keys() == "": results.update(localCopy_results) else: localCopy_results, list_ok, list_retry, list_fallback = self.checkCopy( localCopy_results, len(list_files), self.params["protocol"] ) results.update(localCopy_results) if self.debug: print "\t localCopy_results = %s \n" % localCopy_results return results
def setProtocol(self, middleware): """ define the allowed potocols based on $middlware which depend on scheduler """ # default To be used with "middleware" if self.debug: print "setProtocol() :\n" print "\tmiddleware = %s utils \n" % middleware lcgOpt = {"srmv1": "-b -D srmv1 -t 2400 --verbose", "srmv2": "-b -D srmv2 -t 2400 --verbose"} if self.checkLcgUtils() >= 17: lcgOpt = { "srmv1": "-b -D srmv1 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose", "srmv2": "-b -D srmv2 --srm-timeout 2400 --sendreceive-timeout 2400 --connect-timeout 300 --verbose", } srmOpt = { "srmv1": " -report ./srmcp.report -retry_timeout 480000 -retry_num 3 -streams_num=1 ", "srmv2": " -report=./srmcp.report -retry_timeout=480000 -retry_num=3 -storagetype=permanent ", } rfioOpt = "" #### FEDE FOR XROOTD ######### xrootdOpt = "" ############################# supported_protocol = None if middleware.lower() in ["osg", "lcg", "condor", "sge"]: supported_protocol = [("srm-lcg", lcgOpt[self.params["srm_version"]])] # ,\ # (self.params['srm_version'],srmOpt[self.params['srm_version']])] # elif middleware.lower() in ['lsf','caf']: elif middleware.lower() in ["lsf"]: supported_protocol = [("rfio", rfioOpt)] elif middleware.lower() in ["pbs"]: supported_protocol = [("rfio", rfioOpt), ("local", "")] elif middleware.lower() in ["arc"]: supported_protocol = [("srmv2", "-debug"), ("srmv1", "-debug")] #### FEDE FOR XROOTD ########## elif middleware.lower() in ["caf"]: if self.params["protocol"]: supported_protocol = [(self.params["protocol"], "")] else: supported_protocol = [("rfio", rfioOpt)] ####################################### else: ## here we can add support for any kind of protocol, ## maybe some local schedulers need something dedicated pass # find out where we run and where we stageout siteCfg = loadSiteLocalConfig() localSeName = siteCfg.localStageOutSEName() remoteSeName = self.params["se_name"] seIsLocal = localSeName == remoteSeName print "******************************************************" print "localSeName = ", localSeName print "remoteSeName = ", remoteSeName print "******************************************************" # force lstore protocol for Vanderbilt, waiting for better # long term solution if seIsLocal and localSeName.endswith("vanderbilt.edu"): print "*** I am at, and writing-to, Vanderbilt. Try lstore first ***" supported_protocol.insert(0, ("lstore", "")) return supported_protocol