def _parse_status_string(self, status_string):
try:
dom = xml.dom.minidom.parseString(status_string)
except Exception:
raise BackendError("Couldn't parse qstat XML output!")
for job_node in dom.getElementsByTagName('job_list'):
job_info = {}
try:
for node in job_node.childNodes:
if node.nodeType != xml.dom.minidom.Node.ELEMENT_NODE:
continue
if node.hasChildNodes():
job_info[str(node.nodeName)] = str(
node.childNodes[0].nodeValue)
for job_info_key in job_info:
if str(job_info_key).lower() in self._job_status_key:
job_info[CheckInfo.WMSID] = job_info.pop(job_info_key)
job_info[CheckInfo.RAW_STATUS] = job_info.pop('state')
if 'queue_name' in job_info:
queue, node = job_info['queue_name'].split('@')
job_info[CheckInfo.QUEUE] = queue
job_info[CheckInfo.WN] = node
except Exception:
raise BackendError('Error reading job info:\n%s' %
job_node.toxml())
yield job_info
def _tidyUpWorkingDirectory(self, forceCleanup=False):
# active remote submission should clean up when no jobs remain
if self.remoteType == PoolType.SSH or self.remoteType == PoolType.GSISSH:
self.debugOut(
"Revising remote working directory for cleanup. Forced CleanUp: %s"
% forceCleanup)
activity = utils.ActivityLog('revising remote work directory')
# check whether there are any remote working directories remaining
checkProcess = self.Pool.LoggedExecute(
'find %s -maxdepth 1 -type d | wc -l' % self.getWorkdirPath())
try:
if forceCleanup or (int(checkProcess.getOutput()) <= 1):
cleanupProcess = self.Pool.LoggedExecute(
'rm -rf %s' % self.getWorkdirPath())
if cleanupProcess.wait() != 0:
if self.explainError(cleanupProcess,
cleanupProcess.wait()):
return
cleanupProcess.logError(self.errorLog)
raise BackendError(
'Cleanup process %s returned: %s' %
(cleanupProcess.cmd, cleanupProcess.getMessage()))
except Exception:
self._log.warning(
'There might be some junk data left in: %s @ %s',
self.getWorkdirPath(), self.Pool.getDomain())
raise BackendError(
'Unable to clean up remote working directory')
activity.finish()
def _cleanup_remote_output_dn(self):
# active remote submission should clean up when no jobs remain
if self._remote_type in (PoolType.SSH, PoolType.GSISSH):
activity = Activity('clearing remote work directory')
# check whether there are any remote working directories remaining
check_proc = self._proc_factory.logged_execute(
'find %s -maxdepth 1 -type d | wc -l' %
self._get_remote_output_dn())
try:
if int(check_proc.get_output()) <= 1:
cleanup_cmd = 'rm -rf %s' % self._get_remote_output_dn()
cleanup_proc = self._proc_factory.logged_execute(
cleanup_cmd)
if cleanup_proc.wait() != 0:
if self._explain_error(cleanup_proc,
cleanup_proc.wait()):
return
cleanup_proc.log_error(self._error_log_fn)
raise BackendError(
'Cleanup process %s returned: %s' %
(cleanup_proc.cmd, cleanup_proc.get_message()))
except Exception:
self._log.warning(
'There might be some junk data left in: %s @ %s',
self._get_remote_output_dn(),
self._proc_factory.get_domain())
raise BackendError(
'Unable to clean up remote working directory')
activity.finish()
def _init_pool_interface_remote(self, config, sched, collector, host):
if self._remote_type == PoolType.SSH:
self._proc_factory = ProcessHandler.create_instance('SSHProcessHandler',
remote_host=host, sshLink=config.get_work_path('.ssh', self._name + host))
else:
self._proc_factory = ProcessHandler.create_instance('GSISSHProcessHandler',
remote_host=host, sshLink=config.get_work_path('.gsissh', self._name + host))
# ssh type instructions rely on commands being available on remote pool
self._submit_exec = 'condor_submit'
self._transfer_exec = 'false' # disabled for this type
# test availability of commands
version_proc = self._proc_factory.logged_execute('condor_version')
if version_proc.wait() != 0:
version_proc.log_error(self._error_log_fn)
raise BackendError('Failed to access remote Condor tools! ' +
'The pool you are submitting to is very likely not configured properly.')
# get initial workdir on remote pool
remote_work_dn = config.get('remote workdir', '')
if remote_work_dn:
remote_user_name = self._proc_factory.logged_execute('whoami').get_output().strip()
self._pool_work_dn = os.path.join(remote_work_dn, remote_user_name)
remote_dn_proc = self._proc_factory.logged_execute('mkdir -p %s' % self._pool_work_dn)
else:
remote_dn_proc = self._proc_factory.logged_execute('pwd')
self._pool_work_dn = remote_dn_proc.get_output().strip()
if remote_dn_proc.wait() != 0:
self._log.critical('Code: %d\nOutput Message: %s\nError Message: %s',
remote_dn_proc.wait(), remote_dn_proc.get_output(), remote_dn_proc.get_error())
raise BackendError('Failed to determine, create or verify base work directory on remote host')
def _statusReturnLineRead(self, line):
try:
statusReturnValues = line.split()
# transform output string to dictionary
jobinfo = dict(izip(self.statusReturnKeys, statusReturnValues))
# extract GC and WMS ID, check for consistency
jobID, wmsID = jobinfo['GCID@WMSID'].split('@')
if (wmsID != jobinfo['wmsid']):
raise BackendError(
"Critical! Unable to match jobs in queue! \n CondorID: %s Expected: %s \n%s"
% (jobinfo['wmsid'], wmsID, line))
jobinfo['jobid'] = int(jobID)
del jobinfo['GCID@WMSID']
# extract Host and Queue data
if "@" in jobinfo["RemoteHost"]:
jobinfo['dest'] = jobinfo["RemoteHost"].split(
"@")[1] + ': /' + jobinfo.get("Queue", "")
else:
jobinfo['dest'] = jobinfo["RemoteHost"]
del jobinfo["RemoteHost"]
if "Queue" in jobinfo:
del jobinfo["Queue"]
# convert status to appropriate format
status = self._statusMap[jobinfo['status']]
jobinfo['status'] = self._humanMap[jobinfo['status']]
return (jobinfo['jobid'], jobinfo['wmsid'], status, jobinfo)
except Exception:
raise BackendError('Error reading job info:\n%s' % line)
def _initPoolInterfaces(self, config):
# check submissal type
self.remoteType = config.getEnum('remote Type', PoolType, PoolType.LOCAL)
self.debugOut("Selected pool type: %s" % PoolType.enum2str(self.remoteType))
# get remote destination features
user,sched,collector = self._getDestination(config)
nice_user = user or "<local default>"
nice_sched = sched or "<local default>"
nice_collector = collector or "<local default>"
self.debugOut("Destination:\n")
self.debugOut("\tuser:%s @ sched:%s via collector:%s" % (nice_user, nice_sched, nice_collector))
# prepare commands appropriate for pool type
if self.remoteType == PoolType.LOCAL or self.remoteType == PoolType.SPOOL:
self.user=user
self.Pool=self.Pool=ProcessHandler.createInstance("LocalProcessHandler")
# local and remote use condor tools installed locally - get them
self.submitExec = utils.resolveInstallPath('condor_submit')
self.historyExec = utils.resolveInstallPath('condor_history') # completed/failed jobs are stored outside the queue
self.cancelExec = utils.resolveInstallPath('condor_rm')
self.transferExec = utils.resolveInstallPath('condor_transfer_data') # submission might spool to another schedd and need to fetch output
self.configValExec = utils.resolveInstallPath('condor_config_val') # service is better when being able to adjust to pool settings
if self.remoteType == PoolType.SPOOL:
# remote requires adding instructions for accessing remote pool
self.submitExec+= " %s %s" % (utils.QM(sched,"-remote %s"%sched,""),utils.QM(collector, "-pool %s"%collector, ""))
self.historyExec = "false" # disabled for this type
self.cancelExec+= " %s %s" % (utils.QM(sched,"-name %s"%sched,""),utils.QM(collector, "-pool %s"%collector, ""))
self.transferExec+= " %s %s" % (utils.QM(sched,"-name %s"%sched,""),utils.QM(collector, "-pool %s"%collector, ""))
else:
# ssh type instructions are passed to the remote host via regular ssh/gsissh
host="%s%s"%(utils.QM(user,"%s@" % user,""), sched)
if self.remoteType == PoolType.SSH:
self.Pool=ProcessHandler.createInstance("SSHProcessHandler", remoteHost = host, sshLink = config.getWorkPath(".ssh", self._name + host))
else:
self.Pool=ProcessHandler.createInstance("GSISSHProcessHandler", remoteHost = host, sshLink = config.getWorkPath(".gsissh", self._name + host))
# ssh type instructions rely on commands being available on remote pool
self.submitExec = 'condor_submit'
self.historyExec = 'condor_history'
self.cancelExec = 'condor_rm'
self.transferExec = "false" # disabled for this type
self.configValExec = 'condor_config_val'
# test availability of commands
testProcess=self.Pool.LoggedExecute("condor_version")
self.debugOut("*** Testing remote connectivity:\n%s"%testProcess.cmd)
if testProcess.wait()!=0:
testProcess.logError(self.errorLog)
raise BackendError("Failed to access remote Condor tools! The pool you are submitting to is very likely not configured properly.")
# get initial workdir on remote pool
remote_workdir = config.get("remote workdir", '')
if remote_workdir:
uName = self.Pool.LoggedExecute("whoami").getOutput().strip()
self.poolWorkDir = os.path.join(remote_workdir, uName)
pwdProcess = self.Pool.LoggedExecute("mkdir -p %s" % self.poolWorkDir )
else:
pwdProcess=self.Pool.LoggedExecute("pwd")
self.poolWorkDir=pwdProcess.getOutput().strip()
if pwdProcess.wait()!=0:
self._log.critical("Code: %d\nOutput Message: %s\nError Message: %s", pwdProcess.wait(), pwdProcess.getOutput(), pwdProcess.getError())
raise BackendError("Failed to determine, create or verify base work directory on remote host")
def __new__(cls, config, name):
ec = ExceptionCollector()
for cmd, wms in [('sgepasswd', 'OGE'), ('pbs-config', 'PBS'), ('qsub', 'OGE'), ('bsub', 'LSF'), ('job_slurm', 'SLURM')]:
try:
utils.resolveInstallPath(cmd)
except Exception:
ec.collect()
continue
try:
wmsCls = WMS.getClass(wms)
except Exception:
raise BackendError('Unable to load backend class %s' % repr(wms))
config_wms = config.changeView(viewClass = 'TaggedConfigView', setClasses = [wmsCls])
return WMS.createInstance(wms, config_wms, name)
ec.raise_any(BackendError('No valid local backend found!')) # at this point all backends have failed!
def _getValidSocketArgs(self):
if self._socketMisses >= self._socketMaxMiss:
self._socketMisses -= 1
return []
# validate that current socket does exist and is fresh enough, else pick next
try:
if (time.time() - os.path.getctime(
self._getCurrentSocket())) > self._socketMinSec:
raise OSError
except OSError:
self._incrementSocket()
while not self._validateControlMaster():
self._socketMisses += 1
if not self._needSocket:
self._log(
logging.INFO3, 'Failed to validate socket. (%d/%d)' %
(self._socketMisses, self._socketMaxMiss))
if self._socketMisses == self._socketMaxMiss:
self._socketMisses + self._socketMaxMiss
self._log(
logging.INFO2,
'Disabling failing sockets for %d operations.' %
self._socketMaxMiss)
return []
if self._socketMisses == self._socketMaxMiss:
raise BackendError("Repeated failure to create ControlMaster.")
self._socketMisses = max(self._socketMisses - 1, 0)
return self._getCurrentSocketArgs()
def execute(self,
wms_id_list): # yields list of (wms_id, job_status, job_info)
exc = ExceptionCollector()
for wms_id in wms_id_list:
try:
job_info = filter_dict(
dict(self._status_fun(wms_id)),
value_filter=lambda v: v not in ['', '0'])
job_info[CheckInfo.RAW_STATUS] = job_info.pop('status',
'').lower()
if 'destination' in job_info:
try:
dest_info = job_info['destination'].split('/', 1)
job_info[CheckInfo.SITE] = dest_info[0].strip()
job_info[CheckInfo.QUEUE] = dest_info[1].strip()
except Exception:
clear_current_exception()
yield (wms_id,
self._status_map.get(job_info[CheckInfo.RAW_STATUS],
Job.UNKNOWN), job_info)
except Exception:
exc.collect()
if abort():
break
exc.raise_any(
BackendError('Encountered errors while checking job status'))
def _submitJob(self, jobNum, module):
activity = utils.ActivityLog('submitting jobs')
try:
sandbox = tempfile.mkdtemp('',
'%s.%04d.' % (module.taskID, jobNum),
self.sandPath)
except Exception:
raise BackendError('Unable to create sandbox directory "%s"!' %
sandbox)
sbPrefix = sandbox.replace(self.sandPath, '').lstrip('/')
def translateTarget(d, s, t):
return (d, s, os.path.join(sbPrefix, t))
self.smSBIn.doTransfer(
ismap(translateTarget, self._getSandboxFilesIn(module)))
self._writeJobConfig(
os.path.join(sandbox, '_jobconfig.sh'), jobNum, module, {
'GC_SANDBOX': sandbox,
'GC_SCRATCH_SEARCH': str.join(' ', self.scratchPath)
})
reqs = self.brokerSite.brokerAdd(module.getRequirements(jobNum),
WMS.SITES)
reqs = dict(self.brokerQueue.brokerAdd(reqs, WMS.QUEUES))
if (self.memory > 0) and (reqs.get(WMS.MEMORY, 0) < self.memory):
reqs[
WMS.
MEMORY] = self.memory # local jobs need higher (more realistic) memory requirements
(stdout, stderr) = (os.path.join(sandbox, 'gc.stdout'),
os.path.join(sandbox, 'gc.stderr'))
jobName = module.getDescription(jobNum).jobName
proc = utils.LoggedProcess(
self.submitExec, '%s %s "%s" %s' %
(self.submitOpts,
self.getSubmitArguments(jobNum, jobName, reqs, sandbox, stdout,
stderr), utils.pathShare('gc-local.sh'),
self.getJobArguments(jobNum, sandbox)))
retCode = proc.wait()
wmsIdText = proc.getOutput().strip().strip('\n')
try:
wmsId = self.parseSubmitOutput(wmsIdText)
except Exception:
wmsId = None
del activity
if retCode != 0:
self._log.warning('%s failed:', self.submitExec)
elif wmsId is None:
self._log.warning('%s did not yield job id:\n%s', self.submitExec,
wmsIdText)
if wmsId:
wmsId = self._createId(wmsId)
open(os.path.join(sandbox, wmsId), 'w')
else:
proc.logError(self.errorLog)
return (jobNum, utils.QM(wmsId, wmsId, None), {'sandbox': sandbox})
def _get_jobs_output(self, gc_id_jobnum_list):
# Get output of jobs and yield output dirs
if len(gc_id_jobnum_list) == 0:
raise StopIteration
root_dn = os.path.join(self._path_output, 'tmp')
try:
if len(gc_id_jobnum_list) == 1:
# For single jobs create single subdir
tmp_dn = os.path.join(root_dn,
md5_hex(gc_id_jobnum_list[0][0]))
else:
tmp_dn = root_dn
ensure_dir_exists(tmp_dn)
except Exception:
raise BackendError(
'Temporary path "%s" could not be created.' % tmp_dn,
BackendError)
map_gc_id2jobnum = dict(gc_id_jobnum_list)
jobs = self._write_wms_id_list(gc_id_jobnum_list)
activity = Activity('retrieving %d job outputs' %
len(gc_id_jobnum_list))
proc = LocalProcess(self._output_exec, '--noint', '--logfile',
'/dev/stderr', '-i', jobs, '--dir', tmp_dn)
# yield output dirs
todo = map_gc_id2jobnum.values()
current_jobnum = None
for line in imap(str.strip, proc.stdout.iter(timeout=60)):
if line.startswith(tmp_dn):
todo.remove(current_jobnum)
output_dn = line.strip()
unpack_wildcard_tar(self._log, output_dn)
yield (current_jobnum, output_dn)
current_jobnum = None
else:
current_jobnum = map_gc_id2jobnum.get(self._create_gc_id(line),
current_jobnum)
exit_code = proc.status(timeout=0, terminate=True)
activity.finish()
if exit_code != 0:
if 'Keyboard interrupt raised by user' in proc.stderr.read(
timeout=0):
remove_files([jobs, root_dn])
raise StopIteration
else:
self._log.log_process(proc,
files={'jobs': SafeFile(jobs).read()})
self._log.error('Trying to recover from error ...')
for dn in os.listdir(root_dn):
yield (None, os.path.join(root_dn, dn))
# return unretrievable jobs
for jobnum in todo:
yield (jobnum, None)
remove_files([jobs, tmp_dn])
def cancelJobs(self, ids):
if not len(ids):
raise StopIteration
activity = utils.ActivityLog('cancelling jobs')
proc = utils.LoggedProcess(self.cancelExec, self.getCancelArguments(self._getRawIDs(ids)))
if proc.wait() != 0:
for line in proc.getError().splitlines():
if not self.unknownID() in line:
utils.eprint(line.strip())
del activity
activity = utils.ActivityLog('waiting for jobs to finish')
time.sleep(5)
for wmsId, jobNum in ids:
path = self._getSandbox(wmsId)
if path is None:
utils.eprint('Sandbox for job %d with wmsId "%s" could not be found' % (jobNum, wmsId))
continue
try:
shutil.rmtree(path)
except Exception:
raise BackendError('Sandbox for job %d with wmsId "%s" could not be deleted' % (jobNum, wmsId))
yield (jobNum, wmsId)
del activity
def _secureLinkSocket(self, sshLink, enforce = True):
if os.path.exists(sshLink):
if stat.S_ISSOCK(os.stat(sshLink).st_mode):
try:
os.chmod(sshLink, stat.S_IRWXU)
except Exception:
if self.socketEnforce:
raise BackendError("Could not secure SSHLink:\n %s" % sshLink)
else:
return False
else:
if self.socketEnforce:
raise BackendError("Non-socket object already exists for SSHLink:\n %s" % sshLink)
else:
return False
return True
def _initPoolInterfaces(self, config):
"""
Establish adapters and connection details for pool and schedd
"""
poolConfig = {}
for poolConfigFileName in config.getList('poolConfig', [],
onChange=None):
try:
self._log(logging.DEBUG1,
"Reading pool config '%s'" % poolConfigFileName)
confFile = open(poolConfigFileName, 'r')
poolConfig.update(json.load(confFile)) # TODO: json
confFile.close()
except Exception:
raise BackendError('Failed to parse pool configuration file!')
self._jobFeatureMap = poolConfig.get('jobFeatureMap', {})
self._queueQueryMap = poolConfig.get('queueQueryMap', {})
self._niceName = poolConfig.get('NiceName', '<POOLNAME>')
cfgScheddURI = config.get('ScheddURI', '', onChange=changeOnlyUnset)
if cfgScheddURI:
self._schedd = HTCScheddFactory(cfgScheddURI, parentPool=self)
else:
self._schedd = self._getDynamicSchedd(poolConfig)
config.set('ScheddURI', self._schedd.getURI())
self._log(
logging.INFO1, 'Using Schedd %s (none explicitly defined)' %
(self._schedd.getURI()))
self._log(logging.INFO1,
'Connected to Schedd %s' % (self._schedd.getURI()))
def getJobsOutput(self, htcIDs):
retrievedJobs = []
for index, htcID in enumerate(htcIDs):
self._log(
logging.DEBUG3, "Retrieving job files (%d/%d): %s" %
(index, len(htcIDs), jobData[0]))
getProcess = self._adapter.LoggedGet(
self.getStagingDir(htcID),
self.parentPool.getSandboxPath(htcID.jobNum))
if getProcess.wait(timeout=self._adapterMaxWait):
getProcess.logError(self.parentPool.errorLog, brief=True)
self._log(logging.INFO1,
"Retrieval failed for job %d." % (jobData[0]))
else:
retrievedJobs.append(htcID)
try:
self.cleanStagingDir(htcID=htcID)
except Exception:
self._log(logging.DEFAULT, 'Unable to clean staging dir')
# clean up task dir if no job(dir)s remain
try:
statProcess = self._adapter.LoggedExecute(
'find %s -maxdepth 1 -type d | wc -l' %
self.getStagingDir(taskID=htcIDs[0].gctaskID))
if statProcess.wait(timeout=self._adapterMaxWait):
statProcess.logError(self.parentPool.errorLog, brief=True)
raise BackendError(
'Failed to check remote dir for cleanup : %s @ %s' %
(self.getStagingDir(taskID=htcIDs[0].gctaskID)))
elif (int(checkProcess.getOutput()) == 1):
self.cleanStagingDir(taskID=htcIDs[0].gctaskID)
except Exception:
self._log(logging.DEFAULT, 'unable to clean task dir')
return retrievedJobs
def createWMS(wms):
try:
wmsCls = WMS.getClass(wms)
except Exception:
raise BackendError('Unable to load backend class %s' % repr(wms))
wms_config = config.changeView(viewClass = 'TaggedConfigView', setClasses = [wmsCls])
return WMS.createInstance(wms, wms_config, name)
def _purge_directory(log, path, wms_id):
try:
shutil.rmtree(path)
except Exception:
log.critical('Unable to delete directory %r: %r', path,
os.listdir(path))
raise BackendError('Sandbox for job %r could not be deleted', wms_id)
def __new__(cls, config, name):
def _create_backend(wms):
try:
backend_cls = WMS.get_class(wms)
except Exception:
raise BackendError('Unable to load backend class %s' %
repr(wms))
wms_config = config.change_view(view_class='TaggedConfigView',
set_classes=[backend_cls])
return WMS.create_instance(wms, wms_config, name)
wms = config.get('wms', '')
if wms:
return _create_backend(wms)
exc = ExceptionCollector()
for cmd, wms in [('sacct', 'SLURM'), ('sgepasswd', 'OGE'),
('pbs-config', 'PBS'), ('qsub', 'OGE'),
('condor_q', 'Condor'), ('bsub', 'LSF'),
('job_slurm', 'JMS')]:
try:
resolve_install_path(cmd)
except Exception:
exc.collect()
continue
return _create_backend(wms)
# at this point all backends have failed!
exc.raise_any(BackendError('No valid local backend found!'))
def _write_wms_id_list(self, gc_id_jobnum_list):
try:
job_fd, job_fn = tempfile.mkstemp('.jobids')
safe_write(os.fdopen(job_fd, 'w'),
str.join('\n', self._iter_wms_ids(gc_id_jobnum_list)))
except Exception:
raise BackendError('Could not write wms ids to %s.' % job_fn)
return job_fn
def getSandboxPath(self, subdirToken=""):
sandpath = os.path.join(self._sandboxDir, str(subdirToken), '' )
if not os.path.exists(sandpath):
try:
os.makedirs(sandpath)
except Exception:
raise BackendError('Error accessing or creating sandbox directory:\n %s' % sandpath)
return sandpath
def writeWMSIds(self, ids):
try:
fd, jobs = tempfile.mkstemp('.jobids')
utils.safeWrite(os.fdopen(fd, 'w'),
str.join('\n', self._getRawIDs(ids)))
except Exception:
raise BackendError('Could not write wms ids to %s.' % jobs)
return jobs
def _get_status_direct(wms_id):
wrapper_status.getStatus(wms_id, 0)
err, api_msg = wrapper_status.get_error()
if err:
raise BackendError(api_msg)
info = wrapper_status.loadStatus()
return lzip(imap(str.lower, job_status.states_names),
info[0:job_status.ATTR_MAX])
def getStatusDirect(wmsID):
wrStatus.getStatus(wmsID, 0)
err, apiMsg = wrStatus.get_error()
if err:
raise BackendError(apiMsg)
info = wrStatus.loadStatus()
return lzip(imap(str.lower, jobStatus.states_names),
info[0:jobStatus.ATTR_MAX])
def submitJobs(self, jobNumList, module):
if not self.bulkSubmissionBegin(): # Trying to delegate proxy failed
if self._forceDelegate: # User switched on forcing delegation => exception
raise BackendError('Unable to delegate proxy!')
utils.eprint('Unable to delegate proxy! Continue with automatic delegation...')
self._submitParams.update({ '-a': ' ' })
self._useDelegate = False
for submitInfo in GridWMS.submitJobs(self, jobNumList, module):
yield submitInfo
def _create_backend(wms):
try:
backend_cls = WMS.get_class(wms)
except Exception:
raise BackendError('Unable to load backend class %s' %
repr(wms))
wms_config = config.change_view(view_class='TaggedConfigView',
set_classes=[backend_cls])
return WMS.create_instance(wms, wms_config, name)
def _secureLinkDirectory(self, sshLink, enforce = True):
sshLinkDir = os.path.dirname(sshLink)
if not os.path.isdir(sshLinkDir):
try:
os.makedirs(sshLinkDir)
except Exception:
if self.socketEnforce:
raise BackendError("Could not create or access directory for SSHLink:\n %s" % sshLinkDir)
else:
return False
if sshLinkDir!=os.path.dirname(os.path.expanduser("~/.ssh/")):
try:
os.chmod(sshLinkDir, stat.S_IRWXU)
except Exception:
if self.socketEnforce:
raise BackendError("Could not secure directory for SSHLink:\n %s" % sshLinkDir)
else:
return False
return True
def _write_jdl(self, jobnum_list, task):
# construct a temporary JDL for this batch of jobs
jdl_fd, jdl_fn = tempfile.mkstemp(suffix='.jdl')
try:
data = self._get_jdl_str_list(jobnum_list, task)
safe_write(os.fdopen(jdl_fd, 'w'), data)
except Exception:
remove_files([jdl_fn])
raise BackendError('Could not write jdl data to %s.' % jdl_fn)
return jdl_fn
def getSandboxPath(self, jobNum=''):
sandpath = os.path.join(self.sandPath, str(jobNum), '')
if not os.path.exists(sandpath):
try:
os.makedirs(sandpath)
except Exception:
raise BackendError(
'Error accessing or creating sandbox directory:\n %s' %
sandpath)
return sandpath
def _parse(self, proc):
for line in ifilter(identity, proc.stdout.iter(self._timeout)):
if 'error' in line.lower():
raise BackendError('Unable to parse status line %s' % repr(line))
tmp = line.split()
try:
wmsID = str(int(tmp[0]))
except Exception:
continue
yield {CheckInfo.WMSID: wmsID, CheckInfo.RAW_STATUS: tmp[2], CheckInfo.QUEUE: tmp[1]}
def _ssh_link_secure(ssh_link_fn, init_dn):
ssh_link_dn = ensure_dir_exists(os.path.dirname(ssh_link_fn),
'SSH link direcory', BackendError)
if ssh_link_dn != os.path.dirname(os.path.expanduser('~/.ssh/')):
try:
os.chmod(ssh_link_dn, stat.S_IRWXU)
except Exception:
raise BackendError('Could not secure directory for SSHLink %s' %
ssh_link_dn)
if init_dn:
return
if os.path.exists(ssh_link_fn):
if not stat.S_ISSOCK(os.stat(ssh_link_fn).st_mode):
raise BackendError(
'Non-socket object already exists for SSHLink %s' %
ssh_link_fn)
try:
os.chmod(ssh_link_fn, stat.S_IRWXU)
except Exception:
raise BackendError('Could not secure SSHLink %s' % ssh_link_fn)
def __init__(self, msg, proc): (cmd, status, stdout, stderr) = (proc.cmd, proc.wait(), proc.get_output(), proc.get_error()) BackendError.__init__(msg + '\n\tCommand: %s Return code: %s\nstdout: %s\nstderr: %s' % (cmd, status, stdout, stderr))
