def storageInterface( self, endpoint, protocol ):
"""
Create the storage interface.
"""
if self.debug : print 'storageInterface():\n'
try:
interface = SElement( FullPath(endpoint), protocol )
except ProtocolUnknown, ex:
msg = "ERROR : Unable to create interface with %s protocol"%protocol
msg += str(ex)
raise Exception(msg)
class PostMortemServer(PostMortem):
def __init__(self, cfg_params, nj_list):
PostMortem.__init__(self, cfg_params, nj_list)
# init client server params...
CliServerParams(self)
self.copyTout = setLcgTimeout()
if common.scheduler.name().upper() in ['LSF', 'CAF']:
self.copyTout = ' '
if self.storage_path[0] != '/':
self.storage_path = '/' + self.storage_path
return
def collectLogging(self):
# get updated status from server
try:
from StatusServer import StatusServer
stat = StatusServer(self.cfg_params)
warning_msg = stat.resynchClientSide()
if warning_msg is not None:
common.logger.info(warning_msg)
except:
pass
#create once storage interaction object
seEl = None
loc = None
try:
seEl = SElement(self.storage_name, self.storage_proto,
self.storage_port)
except Exception, ex:
common.logger.debug(str(ex))
msg = "ERROR: Unable to create SE source interface \n"
raise CrabException(msg)
try:
loc = SElement("localhost", "local")
except Exception, ex:
common.logger.debug(str(ex))
msg = "ERROR: Unable to create destination interface \n"
raise CrabException(msg)
def retrieveFiles(self, filesToRetrieve):
"""
Real get output from server storage
"""
self.taskuuid = str(common._db.queryTask('name'))
common.logger.debug("Task name: " + self.taskuuid)
# create the list with the actual filenames
remotedir = os.path.join(self.storage_path, self.taskuuid)
# list of file to retrieve
osbTemplate = remotedir + '/out_files_%s.tgz'
osbFiles = [osbTemplate % str(jid) for jid in filesToRetrieve]
if self.cfg_params['CRAB.scheduler'].lower() in ["condor_g"]:
osbTemplate = remotedir + '/CMSSW_%s.stdout'
osbFiles.extend(
[osbTemplate % str(jid) for jid in filesToRetrieve])
osbTemplate = remotedir + '/CMSSW_%s.stderr'
osbFiles.extend(
[osbTemplate % str(jid) for jid in filesToRetrieve])
common.logger.debug("List of OSB files: " + str(osbFiles))
copyHere = self.outDir
destTemplate = copyHere + '/out_files_%s.tgz'
destFiles = [destTemplate % str(jid) for jid in filesToRetrieve]
if self.cfg_params['CRAB.scheduler'].lower() in ["condor_g"]:
destTemplate = copyHere + '/CMSSW_%s.stdout'
destFiles.extend(
[destTemplate % str(jid) for jid in filesToRetrieve])
destTemplate = copyHere + '/CMSSW_%s.stderr'
destFiles.extend(
[destTemplate % str(jid) for jid in filesToRetrieve])
common.logger.info("Starting retrieving output from server " +
str(self.storage_name) + "...")
try:
seEl = SElement(self.storage_name, self.storage_proto,
self.storage_port)
except Exception, ex:
common.logger.debug(str(ex))
msg = "ERROR : Unable to create SE source interface \n"
raise CrabException(msg)
def moveISB_SEAPI(self, listOffiles=[]):
## get task info from BL ##
common.logger.debug("Task name: " + self.taskuuid)
if len(listOffiles) > 0:
isblist = listOffiles
else:
isblist = str(common._db.queryTask('globalSandbox')).split(',')
common.logger.debug("List of ISB files: " + str(isblist))
# init SE interface
common.logger.info("Starting sending the project to the storage " +
str(self.storage_name) + "...")
try:
seEl = SElement(self.storage_name, self.storage_proto,
self.storage_port)
except Exception, ex:
common.logger.debug(str(ex))
msg = "ERROR : Unable to create SE destination interface \n"
msg += "Project " + self.taskuuid + " not Submitted \n"
raise CrabException(msg)
def listRemoteDir(self, endpoint):
"""
"""
#protocol = self.protocolDict[common.scheduler.name().upper()]
#### FEDE for rfio and xrootd at caf #######################################################
if (common.scheduler.name().upper() == 'CAF'
and str.split(str(endpoint), ':')[0] == 'root'):
protocol = 'xrootd'
else:
protocol = self.protocolDict[common.scheduler.name().upper()]
msg = 'protocol = ' + protocol + '\n'
msg += 'endpoint = ' + endpoint + '\n'
common.logger.debug(msg)
try:
Storage = SElement(FullPath(string.strip(endpoint)), protocol)
except Exception, ex:
common.logger.debug(traceback.format_exc())
raise Exception(str(ex))
def collectLogging(self):
# get updated status from server
try:
from StatusServer import StatusServer
stat = StatusServer(self.cfg_params)
warning_msg = stat.resynchClientSide()
if warning_msg is not None:
common.logger.info(warning_msg)
except:
pass
#create once storage interaction object
seEl = None
loc = None
try:
seEl = SElement(self.storage_name, self.storage_proto,
self.storage_port)
except Exception, ex:
common.logger.debug(str(ex))
msg = "ERROR: Unable to create SE source interface \n"
raise CrabException(msg)
class SubmitterServer(Submitter):
def __init__(self, cfg_params, parsed_range, val):
self.srvCfg = {}
self.cfg_params = cfg_params
self.submitRange = []
self.credentialType = 'Proxy'
self.copyTout = setLcgTimeout()
self.extended = int(cfg_params.get('CMSSW.extend', 0))
#wmbs
self.type = int(cfg_params.get('WMBS.automation', 0))
self.taskType = 'fullySpecified'
if self.type == 1: self.taskType = 'partiallySpecified'
if common.scheduler.name().upper() in ['LSF', 'CAF']:
self.credentialType = 'Token'
self.copyTout = ' '
if common.scheduler.name().upper() == 'REMOTEGLIDEIN':
msg = "FATAL ERROR: remoteGlidein scheduler requires use_server=0"
raise CrabException(msg)
Submitter.__init__(self, cfg_params, parsed_range, val)
# init client server params...
CliServerParams(self)
# path fix
if self.storage_path[0] != '/':
self.storage_path = '/' + self.storage_path
self.taskuuid = str(common._db.queryTask('name'))
self.limitJobs = False
return
def run(self):
"""
The main method of the class: submit jobs in range self.nj_list
"""
common.logger.debug("SubmitterServer::run() called")
start = time.time()
#wmbs
self.BuildJobList(self.type)
self.submitRange = self.nj_list
## wmbs
check = self.checkIfCreate(self.type)
if check == 0:
self.remotedir = os.path.join(self.storage_path, self.taskuuid)
self.manageCredential()
# check if it is the first submission
isFirstSubmission = common._db.checkIfNeverSubmittedBefore()
# standard submission to the server
self.performSubmission(isFirstSubmission)
stop = time.time()
common.logger.debug("Submission Time: " + str(stop - start))
#wmbs
if self.type == 0:
msg = 'Total of %d jobs submitted' % len(self.submitRange)
else:
msg = 'Request submitted to the server.'
common.logger.info(msg)
if int(self.type) == 1:
common._db.updateTask_({'jobType': 'Submitted'})
return
def moveISB_SEAPI(self, listOffiles=[]):
## get task info from BL ##
common.logger.debug("Task name: " + self.taskuuid)
if len(listOffiles) > 0:
isblist = listOffiles
else:
isblist = str(common._db.queryTask('globalSandbox')).split(',')
common.logger.debug("List of ISB files: " + str(isblist))
# init SE interface
common.logger.info("Starting sending the project to the storage " +
str(self.storage_name) + "...")
try:
seEl = SElement(self.storage_name, self.storage_proto,
self.storage_port)
except Exception, ex:
common.logger.debug(str(ex))
msg = "ERROR : Unable to create SE destination interface \n"
msg += "Project " + self.taskuuid + " not Submitted \n"
raise CrabException(msg)
try:
loc = SElement("localhost", "local")
except Exception, ex:
common.logger.debug(str(ex))
msg = "ERROR : Unable to create SE source interface \n"
msg += "Project " + self.taskuuid + " not Submitted \n"
raise CrabException(msg)
class GetOutputServer(GetOutput, StatusServer):
def __init__(self, *args):
GetOutput.__init__(self, *args)
# init client server params...
CliServerParams(self)
if self.storage_path[0] != '/':
self.storage_path = '/' + self.storage_path
self.copyTout = setLcgTimeout()
if common.scheduler.name().upper() in ['LSF', 'CAF']:
self.copyTout = ' '
return
def getOutput(self):
# get updated status from server #inherited from StatusServer
warning_msg = self.resynchClientSide()
if warning_msg is not None:
common.logger.info(warning_msg)
# understand whether the required output are available
self.checkBeforeGet()
# retrive files
filesAndJodId = {}
filesAndJodId.update(self.retrieveFiles(self.list_id))
common.logger.debug("Files to be organized and notified " +
str(filesAndJodId))
# load updated task
task = common._db.getTask()
self.organizeOutput(task, self.list_id)
self.notifyRetrievalToServer(filesAndJodId)
return
def retrieveFiles(self, filesToRetrieve):
"""
Real get output from server storage
"""
self.taskuuid = str(common._db.queryTask('name'))
common.logger.debug("Task name: " + self.taskuuid)
# create the list with the actual filenames
remotedir = os.path.join(self.storage_path, self.taskuuid)
# list of file to retrieve
osbTemplate = remotedir + '/out_files_%s.tgz'
osbFiles = [osbTemplate % str(jid) for jid in filesToRetrieve]
if self.cfg_params['CRAB.scheduler'].lower() in ["condor_g"]:
osbTemplate = remotedir + '/CMSSW_%s.stdout'
osbFiles.extend(
[osbTemplate % str(jid) for jid in filesToRetrieve])
osbTemplate = remotedir + '/CMSSW_%s.stderr'
osbFiles.extend(
[osbTemplate % str(jid) for jid in filesToRetrieve])
common.logger.debug("List of OSB files: " + str(osbFiles))
copyHere = self.outDir
destTemplate = copyHere + '/out_files_%s.tgz'
destFiles = [destTemplate % str(jid) for jid in filesToRetrieve]
if self.cfg_params['CRAB.scheduler'].lower() in ["condor_g"]:
destTemplate = copyHere + '/CMSSW_%s.stdout'
destFiles.extend(
[destTemplate % str(jid) for jid in filesToRetrieve])
destTemplate = copyHere + '/CMSSW_%s.stderr'
destFiles.extend(
[destTemplate % str(jid) for jid in filesToRetrieve])
common.logger.info("Starting retrieving output from server " +
str(self.storage_name) + "...")
try:
seEl = SElement(self.storage_name, self.storage_proto,
self.storage_port)
except Exception, ex:
common.logger.debug(str(ex))
msg = "ERROR : Unable to create SE source interface \n"
raise CrabException(msg)
try:
loc = SElement("localhost", "local")
except Exception, ex:
common.logger.debug(str(ex))
msg = "ERROR : Unable to create destination interface \n"
raise CrabException(msg)