def _parse_status_string(self, status_string): try: dom = xml.dom.minidom.parseString(status_string) except Exception: raise BackendError("Couldn't parse qstat XML output!") for job_node in dom.getElementsByTagName('job_list'): job_info = {} try: for node in job_node.childNodes: if node.nodeType != xml.dom.minidom.Node.ELEMENT_NODE: continue if node.hasChildNodes(): job_info[str(node.nodeName)] = str( node.childNodes[0].nodeValue) for job_info_key in job_info: if str(job_info_key).lower() in self._job_status_key: job_info[CheckInfo.WMSID] = job_info.pop(job_info_key) job_info[CheckInfo.RAW_STATUS] = job_info.pop('state') if 'queue_name' in job_info: queue, node = job_info['queue_name'].split('@') job_info[CheckInfo.QUEUE] = queue job_info[CheckInfo.WN] = node except Exception: raise BackendError('Error reading job info:\n%s' % job_node.toxml()) yield job_info
def _tidyUpWorkingDirectory(self, forceCleanup=False): # active remote submission should clean up when no jobs remain if self.remoteType == PoolType.SSH or self.remoteType == PoolType.GSISSH: self.debugOut( "Revising remote working directory for cleanup. Forced CleanUp: %s" % forceCleanup) activity = utils.ActivityLog('revising remote work directory') # check whether there are any remote working directories remaining checkProcess = self.Pool.LoggedExecute( 'find %s -maxdepth 1 -type d | wc -l' % self.getWorkdirPath()) try: if forceCleanup or (int(checkProcess.getOutput()) <= 1): cleanupProcess = self.Pool.LoggedExecute( 'rm -rf %s' % self.getWorkdirPath()) if cleanupProcess.wait() != 0: if self.explainError(cleanupProcess, cleanupProcess.wait()): return cleanupProcess.logError(self.errorLog) raise BackendError( 'Cleanup process %s returned: %s' % (cleanupProcess.cmd, cleanupProcess.getMessage())) except Exception: self._log.warning( 'There might be some junk data left in: %s @ %s', self.getWorkdirPath(), self.Pool.getDomain()) raise BackendError( 'Unable to clean up remote working directory') activity.finish()
def _cleanup_remote_output_dn(self): # active remote submission should clean up when no jobs remain if self._remote_type in (PoolType.SSH, PoolType.GSISSH): activity = Activity('clearing remote work directory') # check whether there are any remote working directories remaining check_proc = self._proc_factory.logged_execute( 'find %s -maxdepth 1 -type d | wc -l' % self._get_remote_output_dn()) try: if int(check_proc.get_output()) <= 1: cleanup_cmd = 'rm -rf %s' % self._get_remote_output_dn() cleanup_proc = self._proc_factory.logged_execute( cleanup_cmd) if cleanup_proc.wait() != 0: if self._explain_error(cleanup_proc, cleanup_proc.wait()): return cleanup_proc.log_error(self._error_log_fn) raise BackendError( 'Cleanup process %s returned: %s' % (cleanup_proc.cmd, cleanup_proc.get_message())) except Exception: self._log.warning( 'There might be some junk data left in: %s @ %s', self._get_remote_output_dn(), self._proc_factory.get_domain()) raise BackendError( 'Unable to clean up remote working directory') activity.finish()
def _init_pool_interface_remote(self, config, sched, collector, host): if self._remote_type == PoolType.SSH: self._proc_factory = ProcessHandler.create_instance('SSHProcessHandler', remote_host=host, sshLink=config.get_work_path('.ssh', self._name + host)) else: self._proc_factory = ProcessHandler.create_instance('GSISSHProcessHandler', remote_host=host, sshLink=config.get_work_path('.gsissh', self._name + host)) # ssh type instructions rely on commands being available on remote pool self._submit_exec = 'condor_submit' self._transfer_exec = 'false' # disabled for this type # test availability of commands version_proc = self._proc_factory.logged_execute('condor_version') if version_proc.wait() != 0: version_proc.log_error(self._error_log_fn) raise BackendError('Failed to access remote Condor tools! ' + 'The pool you are submitting to is very likely not configured properly.') # get initial workdir on remote pool remote_work_dn = config.get('remote workdir', '') if remote_work_dn: remote_user_name = self._proc_factory.logged_execute('whoami').get_output().strip() self._pool_work_dn = os.path.join(remote_work_dn, remote_user_name) remote_dn_proc = self._proc_factory.logged_execute('mkdir -p %s' % self._pool_work_dn) else: remote_dn_proc = self._proc_factory.logged_execute('pwd') self._pool_work_dn = remote_dn_proc.get_output().strip() if remote_dn_proc.wait() != 0: self._log.critical('Code: %d\nOutput Message: %s\nError Message: %s', remote_dn_proc.wait(), remote_dn_proc.get_output(), remote_dn_proc.get_error()) raise BackendError('Failed to determine, create or verify base work directory on remote host')
def _statusReturnLineRead(self, line): try: statusReturnValues = line.split() # transform output string to dictionary jobinfo = dict(izip(self.statusReturnKeys, statusReturnValues)) # extract GC and WMS ID, check for consistency jobID, wmsID = jobinfo['GCID@WMSID'].split('@') if (wmsID != jobinfo['wmsid']): raise BackendError( "Critical! Unable to match jobs in queue! \n CondorID: %s Expected: %s \n%s" % (jobinfo['wmsid'], wmsID, line)) jobinfo['jobid'] = int(jobID) del jobinfo['GCID@WMSID'] # extract Host and Queue data if "@" in jobinfo["RemoteHost"]: jobinfo['dest'] = jobinfo["RemoteHost"].split( "@")[1] + ': /' + jobinfo.get("Queue", "") else: jobinfo['dest'] = jobinfo["RemoteHost"] del jobinfo["RemoteHost"] if "Queue" in jobinfo: del jobinfo["Queue"] # convert status to appropriate format status = self._statusMap[jobinfo['status']] jobinfo['status'] = self._humanMap[jobinfo['status']] return (jobinfo['jobid'], jobinfo['wmsid'], status, jobinfo) except Exception: raise BackendError('Error reading job info:\n%s' % line)
def _initPoolInterfaces(self, config): # check submissal type self.remoteType = config.getEnum('remote Type', PoolType, PoolType.LOCAL) self.debugOut("Selected pool type: %s" % PoolType.enum2str(self.remoteType)) # get remote destination features user,sched,collector = self._getDestination(config) nice_user = user or "<local default>" nice_sched = sched or "<local default>" nice_collector = collector or "<local default>" self.debugOut("Destination:\n") self.debugOut("\tuser:%s @ sched:%s via collector:%s" % (nice_user, nice_sched, nice_collector)) # prepare commands appropriate for pool type if self.remoteType == PoolType.LOCAL or self.remoteType == PoolType.SPOOL: self.user=user self.Pool=self.Pool=ProcessHandler.createInstance("LocalProcessHandler") # local and remote use condor tools installed locally - get them self.submitExec = utils.resolveInstallPath('condor_submit') self.historyExec = utils.resolveInstallPath('condor_history') # completed/failed jobs are stored outside the queue self.cancelExec = utils.resolveInstallPath('condor_rm') self.transferExec = utils.resolveInstallPath('condor_transfer_data') # submission might spool to another schedd and need to fetch output self.configValExec = utils.resolveInstallPath('condor_config_val') # service is better when being able to adjust to pool settings if self.remoteType == PoolType.SPOOL: # remote requires adding instructions for accessing remote pool self.submitExec+= " %s %s" % (utils.QM(sched,"-remote %s"%sched,""),utils.QM(collector, "-pool %s"%collector, "")) self.historyExec = "false" # disabled for this type self.cancelExec+= " %s %s" % (utils.QM(sched,"-name %s"%sched,""),utils.QM(collector, "-pool %s"%collector, "")) self.transferExec+= " %s %s" % (utils.QM(sched,"-name %s"%sched,""),utils.QM(collector, "-pool %s"%collector, "")) else: # ssh type instructions are passed to the remote host via regular ssh/gsissh host="%s%s"%(utils.QM(user,"%s@" % user,""), sched) if self.remoteType == PoolType.SSH: self.Pool=ProcessHandler.createInstance("SSHProcessHandler", remoteHost = host, sshLink = config.getWorkPath(".ssh", self._name + host)) else: self.Pool=ProcessHandler.createInstance("GSISSHProcessHandler", remoteHost = host, sshLink = config.getWorkPath(".gsissh", self._name + host)) # ssh type instructions rely on commands being available on remote pool self.submitExec = 'condor_submit' self.historyExec = 'condor_history' self.cancelExec = 'condor_rm' self.transferExec = "false" # disabled for this type self.configValExec = 'condor_config_val' # test availability of commands testProcess=self.Pool.LoggedExecute("condor_version") self.debugOut("*** Testing remote connectivity:\n%s"%testProcess.cmd) if testProcess.wait()!=0: testProcess.logError(self.errorLog) raise BackendError("Failed to access remote Condor tools! The pool you are submitting to is very likely not configured properly.") # get initial workdir on remote pool remote_workdir = config.get("remote workdir", '') if remote_workdir: uName = self.Pool.LoggedExecute("whoami").getOutput().strip() self.poolWorkDir = os.path.join(remote_workdir, uName) pwdProcess = self.Pool.LoggedExecute("mkdir -p %s" % self.poolWorkDir ) else: pwdProcess=self.Pool.LoggedExecute("pwd") self.poolWorkDir=pwdProcess.getOutput().strip() if pwdProcess.wait()!=0: self._log.critical("Code: %d\nOutput Message: %s\nError Message: %s", pwdProcess.wait(), pwdProcess.getOutput(), pwdProcess.getError()) raise BackendError("Failed to determine, create or verify base work directory on remote host")
def __new__(cls, config, name): ec = ExceptionCollector() for cmd, wms in [('sgepasswd', 'OGE'), ('pbs-config', 'PBS'), ('qsub', 'OGE'), ('bsub', 'LSF'), ('job_slurm', 'SLURM')]: try: utils.resolveInstallPath(cmd) except Exception: ec.collect() continue try: wmsCls = WMS.getClass(wms) except Exception: raise BackendError('Unable to load backend class %s' % repr(wms)) config_wms = config.changeView(viewClass = 'TaggedConfigView', setClasses = [wmsCls]) return WMS.createInstance(wms, config_wms, name) ec.raise_any(BackendError('No valid local backend found!')) # at this point all backends have failed!
def _getValidSocketArgs(self): if self._socketMisses >= self._socketMaxMiss: self._socketMisses -= 1 return [] # validate that current socket does exist and is fresh enough, else pick next try: if (time.time() - os.path.getctime( self._getCurrentSocket())) > self._socketMinSec: raise OSError except OSError: self._incrementSocket() while not self._validateControlMaster(): self._socketMisses += 1 if not self._needSocket: self._log( logging.INFO3, 'Failed to validate socket. (%d/%d)' % (self._socketMisses, self._socketMaxMiss)) if self._socketMisses == self._socketMaxMiss: self._socketMisses + self._socketMaxMiss self._log( logging.INFO2, 'Disabling failing sockets for %d operations.' % self._socketMaxMiss) return [] if self._socketMisses == self._socketMaxMiss: raise BackendError("Repeated failure to create ControlMaster.") self._socketMisses = max(self._socketMisses - 1, 0) return self._getCurrentSocketArgs()
def execute(self, wms_id_list): # yields list of (wms_id, job_status, job_info) exc = ExceptionCollector() for wms_id in wms_id_list: try: job_info = filter_dict( dict(self._status_fun(wms_id)), value_filter=lambda v: v not in ['', '0']) job_info[CheckInfo.RAW_STATUS] = job_info.pop('status', '').lower() if 'destination' in job_info: try: dest_info = job_info['destination'].split('/', 1) job_info[CheckInfo.SITE] = dest_info[0].strip() job_info[CheckInfo.QUEUE] = dest_info[1].strip() except Exception: clear_current_exception() yield (wms_id, self._status_map.get(job_info[CheckInfo.RAW_STATUS], Job.UNKNOWN), job_info) except Exception: exc.collect() if abort(): break exc.raise_any( BackendError('Encountered errors while checking job status'))
def _submitJob(self, jobNum, module): activity = utils.ActivityLog('submitting jobs') try: sandbox = tempfile.mkdtemp('', '%s.%04d.' % (module.taskID, jobNum), self.sandPath) except Exception: raise BackendError('Unable to create sandbox directory "%s"!' % sandbox) sbPrefix = sandbox.replace(self.sandPath, '').lstrip('/') def translateTarget(d, s, t): return (d, s, os.path.join(sbPrefix, t)) self.smSBIn.doTransfer( ismap(translateTarget, self._getSandboxFilesIn(module))) self._writeJobConfig( os.path.join(sandbox, '_jobconfig.sh'), jobNum, module, { 'GC_SANDBOX': sandbox, 'GC_SCRATCH_SEARCH': str.join(' ', self.scratchPath) }) reqs = self.brokerSite.brokerAdd(module.getRequirements(jobNum), WMS.SITES) reqs = dict(self.brokerQueue.brokerAdd(reqs, WMS.QUEUES)) if (self.memory > 0) and (reqs.get(WMS.MEMORY, 0) < self.memory): reqs[ WMS. MEMORY] = self.memory # local jobs need higher (more realistic) memory requirements (stdout, stderr) = (os.path.join(sandbox, 'gc.stdout'), os.path.join(sandbox, 'gc.stderr')) jobName = module.getDescription(jobNum).jobName proc = utils.LoggedProcess( self.submitExec, '%s %s "%s" %s' % (self.submitOpts, self.getSubmitArguments(jobNum, jobName, reqs, sandbox, stdout, stderr), utils.pathShare('gc-local.sh'), self.getJobArguments(jobNum, sandbox))) retCode = proc.wait() wmsIdText = proc.getOutput().strip().strip('\n') try: wmsId = self.parseSubmitOutput(wmsIdText) except Exception: wmsId = None del activity if retCode != 0: self._log.warning('%s failed:', self.submitExec) elif wmsId is None: self._log.warning('%s did not yield job id:\n%s', self.submitExec, wmsIdText) if wmsId: wmsId = self._createId(wmsId) open(os.path.join(sandbox, wmsId), 'w') else: proc.logError(self.errorLog) return (jobNum, utils.QM(wmsId, wmsId, None), {'sandbox': sandbox})
def _get_jobs_output(self, gc_id_jobnum_list): # Get output of jobs and yield output dirs if len(gc_id_jobnum_list) == 0: raise StopIteration root_dn = os.path.join(self._path_output, 'tmp') try: if len(gc_id_jobnum_list) == 1: # For single jobs create single subdir tmp_dn = os.path.join(root_dn, md5_hex(gc_id_jobnum_list[0][0])) else: tmp_dn = root_dn ensure_dir_exists(tmp_dn) except Exception: raise BackendError( 'Temporary path "%s" could not be created.' % tmp_dn, BackendError) map_gc_id2jobnum = dict(gc_id_jobnum_list) jobs = self._write_wms_id_list(gc_id_jobnum_list) activity = Activity('retrieving %d job outputs' % len(gc_id_jobnum_list)) proc = LocalProcess(self._output_exec, '--noint', '--logfile', '/dev/stderr', '-i', jobs, '--dir', tmp_dn) # yield output dirs todo = map_gc_id2jobnum.values() current_jobnum = None for line in imap(str.strip, proc.stdout.iter(timeout=60)): if line.startswith(tmp_dn): todo.remove(current_jobnum) output_dn = line.strip() unpack_wildcard_tar(self._log, output_dn) yield (current_jobnum, output_dn) current_jobnum = None else: current_jobnum = map_gc_id2jobnum.get(self._create_gc_id(line), current_jobnum) exit_code = proc.status(timeout=0, terminate=True) activity.finish() if exit_code != 0: if 'Keyboard interrupt raised by user' in proc.stderr.read( timeout=0): remove_files([jobs, root_dn]) raise StopIteration else: self._log.log_process(proc, files={'jobs': SafeFile(jobs).read()}) self._log.error('Trying to recover from error ...') for dn in os.listdir(root_dn): yield (None, os.path.join(root_dn, dn)) # return unretrievable jobs for jobnum in todo: yield (jobnum, None) remove_files([jobs, tmp_dn])
def cancelJobs(self, ids): if not len(ids): raise StopIteration activity = utils.ActivityLog('cancelling jobs') proc = utils.LoggedProcess(self.cancelExec, self.getCancelArguments(self._getRawIDs(ids))) if proc.wait() != 0: for line in proc.getError().splitlines(): if not self.unknownID() in line: utils.eprint(line.strip()) del activity activity = utils.ActivityLog('waiting for jobs to finish') time.sleep(5) for wmsId, jobNum in ids: path = self._getSandbox(wmsId) if path is None: utils.eprint('Sandbox for job %d with wmsId "%s" could not be found' % (jobNum, wmsId)) continue try: shutil.rmtree(path) except Exception: raise BackendError('Sandbox for job %d with wmsId "%s" could not be deleted' % (jobNum, wmsId)) yield (jobNum, wmsId) del activity
def _secureLinkSocket(self, sshLink, enforce = True): if os.path.exists(sshLink): if stat.S_ISSOCK(os.stat(sshLink).st_mode): try: os.chmod(sshLink, stat.S_IRWXU) except Exception: if self.socketEnforce: raise BackendError("Could not secure SSHLink:\n %s" % sshLink) else: return False else: if self.socketEnforce: raise BackendError("Non-socket object already exists for SSHLink:\n %s" % sshLink) else: return False return True
def _initPoolInterfaces(self, config): """ Establish adapters and connection details for pool and schedd """ poolConfig = {} for poolConfigFileName in config.getList('poolConfig', [], onChange=None): try: self._log(logging.DEBUG1, "Reading pool config '%s'" % poolConfigFileName) confFile = open(poolConfigFileName, 'r') poolConfig.update(json.load(confFile)) # TODO: json confFile.close() except Exception: raise BackendError('Failed to parse pool configuration file!') self._jobFeatureMap = poolConfig.get('jobFeatureMap', {}) self._queueQueryMap = poolConfig.get('queueQueryMap', {}) self._niceName = poolConfig.get('NiceName', '<POOLNAME>') cfgScheddURI = config.get('ScheddURI', '', onChange=changeOnlyUnset) if cfgScheddURI: self._schedd = HTCScheddFactory(cfgScheddURI, parentPool=self) else: self._schedd = self._getDynamicSchedd(poolConfig) config.set('ScheddURI', self._schedd.getURI()) self._log( logging.INFO1, 'Using Schedd %s (none explicitly defined)' % (self._schedd.getURI())) self._log(logging.INFO1, 'Connected to Schedd %s' % (self._schedd.getURI()))
def getJobsOutput(self, htcIDs): retrievedJobs = [] for index, htcID in enumerate(htcIDs): self._log( logging.DEBUG3, "Retrieving job files (%d/%d): %s" % (index, len(htcIDs), jobData[0])) getProcess = self._adapter.LoggedGet( self.getStagingDir(htcID), self.parentPool.getSandboxPath(htcID.jobNum)) if getProcess.wait(timeout=self._adapterMaxWait): getProcess.logError(self.parentPool.errorLog, brief=True) self._log(logging.INFO1, "Retrieval failed for job %d." % (jobData[0])) else: retrievedJobs.append(htcID) try: self.cleanStagingDir(htcID=htcID) except Exception: self._log(logging.DEFAULT, 'Unable to clean staging dir') # clean up task dir if no job(dir)s remain try: statProcess = self._adapter.LoggedExecute( 'find %s -maxdepth 1 -type d | wc -l' % self.getStagingDir(taskID=htcIDs[0].gctaskID)) if statProcess.wait(timeout=self._adapterMaxWait): statProcess.logError(self.parentPool.errorLog, brief=True) raise BackendError( 'Failed to check remote dir for cleanup : %s @ %s' % (self.getStagingDir(taskID=htcIDs[0].gctaskID))) elif (int(checkProcess.getOutput()) == 1): self.cleanStagingDir(taskID=htcIDs[0].gctaskID) except Exception: self._log(logging.DEFAULT, 'unable to clean task dir') return retrievedJobs
def createWMS(wms): try: wmsCls = WMS.getClass(wms) except Exception: raise BackendError('Unable to load backend class %s' % repr(wms)) wms_config = config.changeView(viewClass = 'TaggedConfigView', setClasses = [wmsCls]) return WMS.createInstance(wms, wms_config, name)
def _purge_directory(log, path, wms_id): try: shutil.rmtree(path) except Exception: log.critical('Unable to delete directory %r: %r', path, os.listdir(path)) raise BackendError('Sandbox for job %r could not be deleted', wms_id)
def __new__(cls, config, name): def _create_backend(wms): try: backend_cls = WMS.get_class(wms) except Exception: raise BackendError('Unable to load backend class %s' % repr(wms)) wms_config = config.change_view(view_class='TaggedConfigView', set_classes=[backend_cls]) return WMS.create_instance(wms, wms_config, name) wms = config.get('wms', '') if wms: return _create_backend(wms) exc = ExceptionCollector() for cmd, wms in [('sacct', 'SLURM'), ('sgepasswd', 'OGE'), ('pbs-config', 'PBS'), ('qsub', 'OGE'), ('condor_q', 'Condor'), ('bsub', 'LSF'), ('job_slurm', 'JMS')]: try: resolve_install_path(cmd) except Exception: exc.collect() continue return _create_backend(wms) # at this point all backends have failed! exc.raise_any(BackendError('No valid local backend found!'))
def _write_wms_id_list(self, gc_id_jobnum_list): try: job_fd, job_fn = tempfile.mkstemp('.jobids') safe_write(os.fdopen(job_fd, 'w'), str.join('\n', self._iter_wms_ids(gc_id_jobnum_list))) except Exception: raise BackendError('Could not write wms ids to %s.' % job_fn) return job_fn
def getSandboxPath(self, subdirToken=""): sandpath = os.path.join(self._sandboxDir, str(subdirToken), '' ) if not os.path.exists(sandpath): try: os.makedirs(sandpath) except Exception: raise BackendError('Error accessing or creating sandbox directory:\n %s' % sandpath) return sandpath
def writeWMSIds(self, ids): try: fd, jobs = tempfile.mkstemp('.jobids') utils.safeWrite(os.fdopen(fd, 'w'), str.join('\n', self._getRawIDs(ids))) except Exception: raise BackendError('Could not write wms ids to %s.' % jobs) return jobs
def _get_status_direct(wms_id): wrapper_status.getStatus(wms_id, 0) err, api_msg = wrapper_status.get_error() if err: raise BackendError(api_msg) info = wrapper_status.loadStatus() return lzip(imap(str.lower, job_status.states_names), info[0:job_status.ATTR_MAX])
def getStatusDirect(wmsID): wrStatus.getStatus(wmsID, 0) err, apiMsg = wrStatus.get_error() if err: raise BackendError(apiMsg) info = wrStatus.loadStatus() return lzip(imap(str.lower, jobStatus.states_names), info[0:jobStatus.ATTR_MAX])
def submitJobs(self, jobNumList, module): if not self.bulkSubmissionBegin(): # Trying to delegate proxy failed if self._forceDelegate: # User switched on forcing delegation => exception raise BackendError('Unable to delegate proxy!') utils.eprint('Unable to delegate proxy! Continue with automatic delegation...') self._submitParams.update({ '-a': ' ' }) self._useDelegate = False for submitInfo in GridWMS.submitJobs(self, jobNumList, module): yield submitInfo
def _create_backend(wms): try: backend_cls = WMS.get_class(wms) except Exception: raise BackendError('Unable to load backend class %s' % repr(wms)) wms_config = config.change_view(view_class='TaggedConfigView', set_classes=[backend_cls]) return WMS.create_instance(wms, wms_config, name)
def _secureLinkDirectory(self, sshLink, enforce = True): sshLinkDir = os.path.dirname(sshLink) if not os.path.isdir(sshLinkDir): try: os.makedirs(sshLinkDir) except Exception: if self.socketEnforce: raise BackendError("Could not create or access directory for SSHLink:\n %s" % sshLinkDir) else: return False if sshLinkDir!=os.path.dirname(os.path.expanduser("~/.ssh/")): try: os.chmod(sshLinkDir, stat.S_IRWXU) except Exception: if self.socketEnforce: raise BackendError("Could not secure directory for SSHLink:\n %s" % sshLinkDir) else: return False return True
def _write_jdl(self, jobnum_list, task): # construct a temporary JDL for this batch of jobs jdl_fd, jdl_fn = tempfile.mkstemp(suffix='.jdl') try: data = self._get_jdl_str_list(jobnum_list, task) safe_write(os.fdopen(jdl_fd, 'w'), data) except Exception: remove_files([jdl_fn]) raise BackendError('Could not write jdl data to %s.' % jdl_fn) return jdl_fn
def getSandboxPath(self, jobNum=''): sandpath = os.path.join(self.sandPath, str(jobNum), '') if not os.path.exists(sandpath): try: os.makedirs(sandpath) except Exception: raise BackendError( 'Error accessing or creating sandbox directory:\n %s' % sandpath) return sandpath
def _parse(self, proc): for line in ifilter(identity, proc.stdout.iter(self._timeout)): if 'error' in line.lower(): raise BackendError('Unable to parse status line %s' % repr(line)) tmp = line.split() try: wmsID = str(int(tmp[0])) except Exception: continue yield {CheckInfo.WMSID: wmsID, CheckInfo.RAW_STATUS: tmp[2], CheckInfo.QUEUE: tmp[1]}
def _ssh_link_secure(ssh_link_fn, init_dn): ssh_link_dn = ensure_dir_exists(os.path.dirname(ssh_link_fn), 'SSH link direcory', BackendError) if ssh_link_dn != os.path.dirname(os.path.expanduser('~/.ssh/')): try: os.chmod(ssh_link_dn, stat.S_IRWXU) except Exception: raise BackendError('Could not secure directory for SSHLink %s' % ssh_link_dn) if init_dn: return if os.path.exists(ssh_link_fn): if not stat.S_ISSOCK(os.stat(ssh_link_fn).st_mode): raise BackendError( 'Non-socket object already exists for SSHLink %s' % ssh_link_fn) try: os.chmod(ssh_link_fn, stat.S_IRWXU) except Exception: raise BackendError('Could not secure SSHLink %s' % ssh_link_fn)
def __init__(self, msg, proc): (cmd, status, stdout, stderr) = (proc.cmd, proc.wait(), proc.get_output(), proc.get_error()) BackendError.__init__(msg + '\n\tCommand: %s Return code: %s\nstdout: %s\nstderr: %s' % (cmd, status, stdout, stderr))