def get_graph_image(graph_dot): proc = LocalProcess('twopi', '-Tpng') proc.stdin.write(graph_dot) proc.stdin.close() if proc.status(timeout = 20) is None: return 'Unable to render graph!' return proc.stdout.read_log() or 'Empty render result!'
def _parseTickets(self, cached = True): # Return cached results if requested if cached and self._cache: return self._cache # Call klist and parse results proc = LocalProcess(self._klistExec) self._cache = {} try: for line in proc.stdout.iter(timeout = 10): if line.count('@') and (line.count(':') > 1): issued_expires, principal = rsplit(line, ' ', 1) issued_expires = issued_expires.replace('/', ' ').split() assert(len(issued_expires) % 2 == 0) issued_str = str.join(' ', issued_expires[:int(len(issued_expires) / 2)]) expires_str = str.join(' ', issued_expires[int(len(issued_expires) / 2):]) parseDate = lambda value, format: time.mktime(time.strptime(value, format)) if expires_str.count(' ') == 3: if len(expires_str.split()[2]) == 2: expires = parseDate(expires_str, '%m %d %y %H:%M:%S') else: expires = parseDate(expires_str, '%m %d %Y %H:%M:%S') elif expires_str.count(' ') == 2: # year information is missing currentYear = int(time.strftime('%Y')) expires = parseDate(expires_str + ' %d' % currentYear, '%b %d %H:%M:%S %Y') issued = parseDate(issued_str + ' %d' % currentYear, '%b %d %H:%M:%S %Y') if expires < issued: # wraparound at new year expires = parseDate(expires_str + ' %d' % (currentYear + 1), '%b %d %H:%M:%S %Y') self._cache.setdefault('tickets', {})[principal] = expires elif line.count(':') == 1: key, value = lmap(str.strip, line.split(':', 1)) self._cache[key.lower()] = value except Exception: raise AccessTokenError('Unable to parse kerberos ticket information!') proc.status_raise(timeout = 0) return self._cache
def _script_thread(self, script, task, jobnum=None, job_obj=None, add_dict=None): # Get both task and job config / state dicts try: tmp = {} if job_obj is not None: for key, value in job_obj.get_dict().items(): tmp[key.upper()] = value tmp['GC_WORKDIR'] = self._path_work if jobnum is not None: tmp.update(task.get_job_dict(jobnum)) tmp.update(add_dict or {}) env = dict(os.environ) for key, value in tmp.items(): if not key.startswith('GC_'): key = 'GC_' + key env[key] = str(value) script = task.substitute_variables('monitoring script', script, jobnum, tmp) if not self._silent: proc = LocalProcess(*shlex.split(script), **{'env_dict': env}) proc_output = proc.get_output(timeout=self._script_timeout) if proc_output.strip(): self._log.info(proc_output.strip()) else: os.system(script) except Exception: self._log.exception('Error while running user script') clear_current_exception()
def _scriptThread(self, script, jobNum = None, jobObj = None, allDict = None): try: tmp = {} if jobObj is not None: for key, value in jobObj.get_dict().items(): tmp[key.upper()] = value tmp['WORKDIR'] = self._workPath tmp.update(self._task.getTaskConfig()) if jobNum is not None: tmp.update(self._task.getJobConfig(jobNum)) tmp.update(allDict or {}) env = dict(os.environ) for key, value in tmp.items(): if not key.startswith('GC_'): key = 'GC_' + key env[key] = str(value) script = self._task.substVars('monitoring script', script, jobNum, tmp) if not self._silent: proc = LocalProcess(*shlex.split(script), **{'environment': env}) proc_output = proc.get_output(timeout = self._runningMax) if proc_output.strip(): self._log.info(proc_output.strip()) else: os.system(script) except Exception: self._log.exception('Error while running user script')
def _submitJob(self, jobNum, module): fd, jdl = tempfile.mkstemp('.jdl') try: jdlData = self.makeJDL(jobNum, module) utils.safeWrite(os.fdopen(fd, 'w'), jdlData) except Exception: utils.removeFiles([jdl]) raise BackendError('Could not write jdl data to %s.' % jdl) try: submitArgs = [] for key_value in utils.filterDict(self._submitParams, vF = lambda v: v).items(): submitArgs.extend(key_value) submitArgs.append(jdl) activity = Activity('submitting job %d' % jobNum) proc = LocalProcess(self._submitExec, '--nomsg', '--noint', '--logfile', '/dev/stderr', *submitArgs) gcID = None for line in ifilter(lambda x: x.startswith('http'), imap(str.strip, proc.stdout.iter(timeout = 60))): gcID = line retCode = proc.status(timeout = 0, terminate = True) activity.finish() if (retCode != 0) or (gcID is None): if self.explainError(proc, retCode): pass else: self._log.log_process(proc, files = {'jdl': SafeFile(jdl).read()}) finally: utils.removeFiles([jdl]) return (jobNum, utils.QM(gcID, self._createId(gcID), None), {'jdl': str.join('', jdlData)})
def _get_cms_cert(config): config = config.change_view(set_sections=['cms', 'access', 'proxy']) try: access = AccessToken.create_instance('VomsAccessToken', config, 'cms-proxy') except Exception: if os.environ.get('X509_USER_PROXY'): return os.environ['X509_USER_PROXY'] raise CMSAuthenticationException('Unable to find grid environment') can_submit = ignore_exception(Exception, False, access.can_submit, 5 * 60, True) if not can_submit: logging.getLogger('access.cms').warning('The grid proxy has expired or is invalid!') role = config.get_list('new proxy roles', '', on_change=None) timeout = config.get_time('new proxy timeout', 10, on_change=None) lifetime = config.get_time('new proxy lifetime', 192 * 60, on_change=None) # password in variable name removes it from debug log password = getpass.getpass('Please enter proxy password: '******'voms-proxy-init') proc = LocalProcess(proxy_init_exec, '--voms', str.join(':', ['cms'] + role), '--valid', '%d:%d' % (lifetime / 60, lifetime % 60), logging=False) if password: proc.stdin.write(password + '\n') proc.stdin.close() proc.get_output(timeout=timeout) except Exception: raise CMSAuthenticationException('Unable to create new grid proxy') access = AccessToken.create_instance('VomsAccessToken', config, 'cms-proxy') # new instance can_submit = ignore_exception(Exception, False, access.can_submit, 5 * 60, True) if not can_submit: raise CMSAuthenticationException('Newly created grid proxy is also invalid') return access.get_auth_fn_list()[0]
def cancelJobs(self, allIds): if len(allIds) == 0: raise StopIteration waitFlag = False for ids in imap(lambda x: allIds[x:x+5], irange(0, len(allIds), 5)): # Delete jobs in groups of 5 - with 5 seconds between groups if waitFlag and not utils.wait(5): break waitFlag = True jobNumMap = dict(ids) jobs = self.writeWMSIds(ids) activity = utils.ActivityLog('cancelling jobs') proc = LocalProcess(self._cancelExec, '--noint', '--logfile', '/dev/stderr', '-i', jobs) retCode = proc.status(timeout = 60, terminate = True) del activity # select cancelled jobs for deletedWMSId in ifilter(lambda x: x.startswith('- '), proc.stdout.iter()): deletedWMSId = self._createId(deletedWMSId.strip('- \n')) yield (jobNumMap.get(deletedWMSId), deletedWMSId) if retCode != 0: if self.explainError(proc, retCode): pass else: self._log.log_process(proc, files = {'jobs': utils.safeRead(jobs)}) utils.removeFiles([jobs])
def _get_version(self, value): old_wd = os.getcwd() os.chdir(clean_path(value)) git_proc = LocalProcess('git', 'rev-parse', '--short', 'HEAD') version = git_proc.get_output(timeout=10, raise_errors=False) os.chdir(old_wd) return version.strip() or 'undefined'
def _scriptThread(self, script, jobNum = None, jobObj = None, allDict = None): try: tmp = {} if jobNum is not None: tmp.update(self._task.getSubmitInfo(jobNum)) if jobObj is not None: tmp.update(jobObj.getAll()) tmp['WORKDIR'] = self._workPath tmp.update(self._task.getTaskConfig()) if jobNum is not None: tmp.update(self._task.getJobConfig(jobNum)) tmp.update(self._task.getSubmitInfo(jobNum)) tmp.update(allDict or {}) for key, value in tmp.items(): if not key.startswith('GC_'): key = 'GC_' + key os.environ[key] = str(value) script = self._task.substVars(script, jobNum, tmp) if not self._silent: proc = LocalProcess(script) self._log.info(proc.get_output(timeout = self._runningMax)) else: os.system(script) except Exception: self._log.exception('Error while running user script!')
def _get_version(self, value): svn_proc = LocalProcess('svnversion', clean_path(value)) version = svn_proc.get_output(timeout=10, raise_errors=False).strip().lower() # different SVN versions yield different output for unversioned directories: if ('exported' in version) or ('unversioned' in version): version = None return version or 'undefined'
def discover(self): proc = LocalProcess(self._exec) for line in proc.stdout.iter(timeout=self._timeout): if not line.startswith(' ') and len(line) > 1: node = line.strip() if ('state = ' in line) and ('down' not in line) and ('offline' not in line): yield {'name': node} proc.status_raise(timeout=0)
def ping_host(host): proc = LocalProcess('ping', '-Uqnc', 1, '-W', 1, host) try: tmp = proc.get_output(timeout=1).splitlines() assert (tmp[-1].endswith('ms')) return float(tmp[-1].split('/')[-2]) / 1000. except Exception: return None
def _list_endpoint_all(self): result = [] proc = LocalProcess(self._lcg_infosites_exec, 'wms') for line in proc.stdout.iter(timeout=10): result.append(line.strip()) proc.status_raise(timeout=0) random.shuffle(result) return result
def _list_endpoint_all(self): result = [] proc = LocalProcess(self._lcg_infosites_exec, 'wms') for line in proc.stdout.iter(timeout=10): result.append(line.strip()) proc.status_raise(timeout=0) random.shuffle(result) return result
def ping_host(host): proc = LocalProcess('ping', '-Uqnc', 1, '-W', 1, host) try: tmp = proc.get_output(timeout = 1).splitlines() assert(tmp[-1].endswith('ms')) return float(tmp[-1].split('/')[-2]) / 1000. except Exception: return None
def listWMS_all(self): result = [] proc = LocalProcess(self._exeLCGInfoSites, 'wms') for line in proc.stdout.iter(timeout = 10): result.append(line.strip()) proc.status_raise(timeout = 0) random.shuffle(result) return result
def _getJobsOutput(self, ids): if len(ids) == 0: raise StopIteration basePath = os.path.join(self._outputPath, 'tmp') try: if len(ids) == 1: # For single jobs create single subdir tmpPath = os.path.join(basePath, md5(ids[0][0]).hexdigest()) else: tmpPath = basePath utils.ensureDirExists(tmpPath) except Exception: raise BackendError('Temporary path "%s" could not be created.' % tmpPath, BackendError) jobNumMap = dict(ids) jobs = self.writeWMSIds(ids) activity = Activity('retrieving %d job outputs' % len(ids)) proc = LocalProcess(self._outputExec, '--noint', '--logfile', '/dev/stderr', '-i', jobs, '--dir', tmpPath) # yield output dirs todo = jobNumMap.values() currentJobNum = None for line in imap(str.strip, proc.stdout.iter(timeout = 60)): if line.startswith(tmpPath): todo.remove(currentJobNum) outputDir = line.strip() if os.path.exists(outputDir): if 'GC_WC.tar.gz' in os.listdir(outputDir): wildcardTar = os.path.join(outputDir, 'GC_WC.tar.gz') try: tarfile.TarFile.open(wildcardTar, 'r:gz').extractall(outputDir) os.unlink(wildcardTar) except Exception: self._log.error('Can\'t unpack output files contained in %s', wildcardTar) yield (currentJobNum, line.strip()) currentJobNum = None else: currentJobNum = jobNumMap.get(self._createId(line), currentJobNum) retCode = proc.status(timeout = 0, terminate = True) activity.finish() if retCode != 0: if 'Keyboard interrupt raised by user' in proc.stderr.read(timeout = 0): utils.removeFiles([jobs, basePath]) raise StopIteration else: self._log.log_process(proc, files = {'jobs': SafeFile(jobs).read()}) self._log.error('Trying to recover from error ...') for dirName in os.listdir(basePath): yield (None, os.path.join(basePath, dirName)) # return unretrievable jobs for jobNum in todo: yield (jobNum, None) utils.removeFiles([jobs, basePath])
def discover(self): proc = LocalProcess(self._exec) for line in proc.stdout.iter(timeout=10): if not line.startswith(' ') and len(line) > 1: node = line.strip() if ('state = ' in line) and ('down' not in line) and ('offline' not in line): yield {'name': node} proc.status_raise(timeout=0)
def _get_git_version(path): from grid_control.utils.process_base import LocalProcess from grid_control.utils import clean_path path = clean_path(path) old_wd = os.getcwd() os.chdir(path) git_proc = LocalProcess('git', 'rev-parse', '--short', 'HEAD') version = git_proc.get_output(timeout=10, raise_errors=False) os.chdir(old_wd) return version.strip() or None
def getNodes(self): result = [] proc = LocalProcess(self._nodesExec) for line in proc.stdout.iter(): if not line.startswith(' ') and len(line) > 1: node = line.strip() if ('state = ' in line) and ('down' not in line) and ('offline' not in line): result.append(node) proc.status_raise(timeout = 0) if len(result) > 0: return result
def getNodes(self): result = [] proc = LocalProcess(self._nodesExec) for line in proc.stdout.iter(): if not line.startswith(' ') and len(line) > 1: node = line.strip() if ('state = ' in line) and ('down' not in line) and ('offline' not in line): result.append(node) proc.status_raise(timeout = 0) if len(result) > 0: return result
def _get_jobs_output(self, gc_id_jobnum_list): # Get output of jobs and yield output dirs if len(gc_id_jobnum_list) == 0: raise StopIteration root_dn = os.path.join(self._path_output, 'tmp') try: if len(gc_id_jobnum_list) == 1: # For single jobs create single subdir tmp_dn = os.path.join(root_dn, md5_hex(gc_id_jobnum_list[0][0])) else: tmp_dn = root_dn ensure_dir_exists(tmp_dn) except Exception: raise BackendError('Temporary path "%s" could not be created.' % tmp_dn, BackendError) map_gc_id2jobnum = dict(gc_id_jobnum_list) jobs = self._write_wms_id_list(gc_id_jobnum_list) activity = Activity('retrieving %d job outputs' % len(gc_id_jobnum_list)) proc = LocalProcess(self._output_exec, '--noint', '--logfile', '/dev/stderr', '-i', jobs, '--dir', tmp_dn) # yield output dirs todo = map_gc_id2jobnum.values() current_jobnum = None for line in imap(str.strip, proc.stdout.iter(timeout=60)): if line.startswith(tmp_dn): todo.remove(current_jobnum) output_dn = line.strip() unpack_wildcard_tar(self._log, output_dn) yield (current_jobnum, output_dn) current_jobnum = None else: current_jobnum = map_gc_id2jobnum.get(self._create_gc_id(line), current_jobnum) exit_code = proc.status(timeout=0, terminate=True) activity.finish() if exit_code != 0: if 'Keyboard interrupt raised by user' in proc.stderr.read(timeout=0): remove_files([jobs, root_dn]) raise StopIteration else: self._log.log_process(proc, files={'jobs': SafeFile(jobs).read()}) self._log.error('Trying to recover from error ...') for dn in os.listdir(root_dn): yield (None, os.path.join(root_dn, dn)) # return unretrievable jobs for jobnum in todo: yield (jobnum, None) remove_files([jobs, tmp_dn])
def _purge_done_jobs(self, wms_id_list_done): purge_log_fn = tempfile.mktemp('.log') purge_proc = LocalProcess(resolve_install_path('glite-ce-job-purge'), '--noint', '--logfile', purge_log_fn, str.join(' ', wms_id_list_done)) exit_code = purge_proc.status(timeout=60) if exit_code != 0: if self._explain_error(purge_proc, exit_code): pass else: self._log.log_process(purge_proc) remove_files([purge_log_fn])
def _parseProxy(self, cached = True): # Return cached results if requested if cached and self._cache: return self._cache # Call voms-proxy-info and parse results proc = LocalProcess(self._infoExec, *self._getProxyArgs()) (retCode, stdout, stderr) = proc.finish(timeout = 10) if (retCode != 0) and not self._ignoreWarning: msg = ('%s output:\n%s\n%s\n' % (self._infoExec, stdout, stderr)).replace('\n\n', '\n') msg += 'If job submission is still possible, you can set [access] ignore warnings = True\n' raise AccessTokenError(msg + '%s failed with return code %d' % (self._infoExec, retCode)) self._cache = DictFormat(':').parse(stdout) return self._cache
def getVersion(): try: proc_ver = LocalProcess('svnversion', '-c', pathPKG()) version = proc_ver.get_output(timeout = 10).strip() if version != '': assert(lfilter(str.isdigit, version)) proc_branch = LocalProcess('svn info', pathPKG()) if 'stable' in proc_branch.get_output(timeout = 10): return '%s - stable' % version return '%s - testing' % version except Exception: clear_current_exception() return __import__('grid_control').__version__ + ' or later'
def _parse_tickets(self, cached=True): # Return cached results if requested if cached and self._cache: return self._cache # Call klist and parse results proc = LocalProcess(self._klist_exec) self._cache = {} try: for line in proc.stdout.iter(timeout=10): if line.count('@') and (line.count(':') > 1): issued_expires, principal = rsplit(line, ' ', 1) issued_expires = issued_expires.replace('/', ' ').split() assert len(issued_expires) % 2 == 0 issued_str = str.join( ' ', issued_expires[:int(len(issued_expires) / 2)]) expires_str = str.join( ' ', issued_expires[int(len(issued_expires) / 2):]) if expires_str.count(' ') == 3: if len(expires_str.split()[2]) == 2: expires = _parse_date(expires_str, '%m %d %y %H:%M:%S') elif len(expires_str.split()[2]) == 4: expires = _parse_date(expires_str, '%m %d %Y %H:%M:%S') # here else: # On NAF, we get an entirely different format now: Sep 2 12:31:34 2021 expires = _parse_date(expires_str, '%b %d %H:%M:%S %Y') elif expires_str.count( ' ') == 2: # year information is missing cur_year = int(time.strftime('%Y')) expires = _parse_date(expires_str + ' %d' % cur_year, '%b %d %H:%M:%S %Y') issued = _parse_date(issued_str + ' %d' % cur_year, '%b %d %H:%M:%S %Y') if expires < issued: # wraparound at new year expires = _parse_date( expires_str + ' %d' % (cur_year + 1), '%b %d %H:%M:%S %Y') self._cache.setdefault('tickets', {})[principal] = expires elif line.count(':') == 1: (key, value) = lmap(str.strip, line.split(':', 1)) self._cache[key.lower()] = value except Exception: raise AccessTokenError( 'Unable to parse kerberos ticket information!') proc.status_raise(timeout=0) return self._cache
def _parseProxy(self, cached=True): # Return cached results if requested if cached and self._cache: return self._cache # Call voms-proxy-info and parse results proc = LocalProcess(self._infoExec, *self._getProxyArgs()) (retCode, stdout, stderr) = proc.finish(timeout=10) if (retCode != 0) and not self._ignoreWarning: msg = ('%s output:\n%s\n%s\n' % (self._infoExec, stdout, stderr)).replace('\n\n', '\n') msg += 'If job submission is still possible, you can set [access] ignore warnings = True\n' raise AccessTokenError(msg + '%s failed with return code %d' % (self._infoExec, retCode)) self._cache = DictFormat(':').parse(stdout) return self._cache
def _parseProxy(self, cached = True): # Return cached results if requested if cached and self._cache: return self._cache # Call voms-proxy-info and parse results args = ['--all'] if self._proxyPath: args.extend(['--file', self._proxyPath]) proc = LocalProcess(self._infoExec, *args) (retCode, stdout, stderr) = proc.finish(timeout = 10) if (retCode != 0) and not self._ignoreWarning: msg = ('voms-proxy-info output:\n%s\n%s\n' % (stdout, stderr)).replace('\n\n', '\n') msg += 'If job submission is still possible, you can set [access] ignore warnings = True\n' raise AccessTokenError(msg + 'voms-proxy-info failed with return code %d' % retCode) self._cache = utils.DictFormat(':').parse(stdout) return self._cache
def _se_runcmd(cmd, *urls, **kwargs): def _clean_se_path(url): return url.replace('dir://', 'file://') url_iter = imap(_clean_se_path, imap(_norm_se_path, urls)) return LocalProcess(get_path_share('gc-storage-tool'), cmd, *url_iter, **kwargs)
def _parse_proxy(self, cached=True): # Return cached results if requested if cached and self._cache: return self._cache # Call voms-proxy-info and parse results proc = LocalProcess(self._proxy_info_exec, *self._get_proxy_info_arguments()) (exit_code, stdout, stderr) = proc.finish(timeout=10) if (exit_code != 0) and not self._ignore_warning: msg = ('%s output:\n%s\n%s\n' % (self._proxy_info_exec, stdout, stderr)).replace('\n\n', '\n') msg += 'If job submission is still possible, you can set [access] ignore warnings = True\n' msg += '%s failed with return code %d' % (self._proxy_info_exec, exit_code) raise AccessTokenError(msg) self._cache = DictFormat(':').parse(stdout) if not self._cache: msg = 'Unable to parse access token information:\n\t%s\n\t%s\n' raise AccessTokenError(msg % (stdout.strip(), stderr.strip())) return self._cache
def _get_cms_cert(config): config = config.change_view(set_sections=['cms', 'access', 'proxy']) try: access = AccessToken.create_instance('VomsAccessToken', config, 'cms-proxy') except Exception: if os.environ.get('X509_USER_PROXY'): return os.environ['X509_USER_PROXY'] raise CMSAuthenticationException('Unable to find grid environment') can_submit = ignore_exception(Exception, False, access.can_submit, 5 * 60, True) if not can_submit: logging.getLogger('access.cms').warning( 'The grid proxy has expired or is invalid!') role = config.get_list('new proxy roles', '', on_change=None) timeout = config.get_time('new proxy timeout', 10, on_change=None) lifetime = config.get_time('new proxy lifetime', 192 * 60, on_change=None) signal_handler = signal.signal(signal.SIGINT, signal.SIG_DFL) # reset Ctrl+C handler # password in variable name removes it from debug log password = getpass.getpass('Please enter proxy password: '******'voms-proxy-init') proc = LocalProcess(proxy_init_exec, '--voms', str.join(':', ['cms'] + role), '--valid', '%d:%d' % (lifetime / 60, lifetime % 60), logging=False) if password: proc.stdin.write(password + '\n') proc.stdin.close() proc.get_output(timeout=timeout) except Exception: raise CMSAuthenticationException('Unable to create new grid proxy') access = AccessToken.create_instance('VomsAccessToken', config, 'cms-proxy') # new instance can_submit = ignore_exception(Exception, False, access.can_submit, 5 * 60, True) if not can_submit: raise CMSAuthenticationException( 'Newly created grid proxy is also invalid') return access.get_auth_fn_list()[0]
def discover(self): active = False keys = [WMS.MEMORY, WMS.CPUTIME, WMS.WALLTIME] parser = dict(izip(keys, [int, parse_time, parse_time])) proc = LocalProcess(self._exec, '-q') for line in proc.stdout.iter(timeout=10): if line.startswith('-'): active = True elif line.startswith(' '): active = False elif active: fields = lmap(str.strip, line.split()[:4]) queue_dict = {'name': fields[0]} for key, value in ifilter(lambda k_v: not k_v[1].startswith('-'), izip(keys, fields[1:])): queue_dict[key] = parser[key](value) yield queue_dict proc.status_raise(timeout=0)
def getQueues(self): (queues, active) = ({}, False) keys = [WMS.MEMORY, WMS.CPUTIME, WMS.WALLTIME] parser = dict(izip(keys, [int, parseTime, parseTime])) proc = LocalProcess(self.statusExec, '-q') for line in proc.stdout.iter(timeout = 10): if line.startswith('-'): active = True elif line.startswith(' '): active = False elif active: fields = lmap(str.strip, line.split()[:4]) queueInfo = {} for key, value in ifilter(lambda k_v: not k_v[1].startswith('-'), izip(keys, fields[1:])): queueInfo[key] = parser[key](value) queues[fields[0]] = queueInfo proc.status_raise(timeout = 0) return queues
def getQueues(self): (queues, active) = ({}, False) keys = [WMS.MEMORY, WMS.CPUTIME, WMS.WALLTIME] parser = dict(izip(keys, [int, parseTime, parseTime])) proc = LocalProcess(self.statusExec, '-q') for line in proc.stdout.iter(timeout = 10): if line.startswith('-'): active = True elif line.startswith(' '): active = False elif active: fields = lmap(str.strip, line.split()[:4]) queueInfo = {} for key, value in ifilter(lambda k_v: not k_v[1].startswith('-'), izip(keys, fields[1:])): queueInfo[key] = parser[key](value) queues[fields[0]] = queueInfo proc.status_raise(timeout = 0) return queues
def _parseProxy(self, cached=True): # Return cached results if requested if cached and self._cache: return self._cache # Call voms-proxy-info and parse results args = ['--all'] if self._proxyPath: args.extend(['--file', self._proxyPath]) proc = LocalProcess(self._infoExec, *args) (retCode, stdout, stderr) = proc.finish(timeout=10) if (retCode != 0) and not self._ignoreWarning: msg = ('voms-proxy-info output:\n%s\n%s\n' % (stdout, stderr)).replace('\n\n', '\n') msg += 'If job submission is still possible, you can set [access] ignore warnings = True\n' raise AccessTokenError( msg + 'voms-proxy-info failed with return code %d' % retCode) self._cache = utils.DictFormat(':').parse(stdout) return self._cache
def matchSites(self, endpoint): activity = Activity('Discovering available WMS services - testing %s' % endpoint) checkArgs = ['-a'] if endpoint: checkArgs.extend(['-e', endpoint]) checkArgs.append(utils.pathShare('null.jdl')) proc = LocalProcess(self._exeGliteWMSJobListMatch, *checkArgs) result = [] for line in proc.stdout.iter(timeout = 3): if line.startswith(' - '): result.append(line[3:].strip()) activity.finish() if proc.status(timeout = 0) is None: self.wms_timeout[endpoint] = self.wms_timeout.get(endpoint, 0) + 1 if self.wms_timeout.get(endpoint, 0) > 10: # remove endpoints after 10 failures self.wms_all.remove(endpoint) return [] return result
def discover(self): nodes = set() proc = LocalProcess(self._configExec, '-shgrpl') for group in proc.stdout.iter(timeout=10): yield {'name': group.strip()} proc_g = LocalProcess(self._configExec, '-shgrp_resolved', group) for host_list in proc_g.stdout.iter(timeout=10): nodes.update(host_list.split()) proc_g.status_raise(timeout=0) for host in sorted(nodes): yield {'name': host.strip()} proc.status_raise(timeout=0)
def _match_sites(self, endpoint): activity = Activity('Discovering available WMS services - testing %s' % endpoint) check_arg_list = ['-a'] if endpoint: check_arg_list.extend(['-e', endpoint]) check_arg_list.append(get_path_share('null.jdl')) proc = LocalProcess(self._job_list_match_exec, *check_arg_list) result = [] for line in proc.stdout.iter(timeout=3): if line.startswith(' - '): result.append(line[3:].strip()) activity.finish() if proc.status(timeout=0) is None: self._wms_timeout_dict[endpoint] = self._wms_timeout_dict.get(endpoint, 0) + 1 if self._wms_timeout_dict.get(endpoint, 0) > 10: # remove endpoints after 10 failures self._wms_list_all.remove(endpoint) return [] return result
def getNodes(self): result = set() proc = LocalProcess(self._configExec, '-shgrpl') for group in proc.stdout.iter(timeout = 10): result.add(group.strip()) proc_g = LocalProcess(self._configExec, '-shgrp_resolved %s' % group) for host in proc_g.stdout.iter(timeout = 10): result.update(host.split()) proc_g.status_raise(timeout = 0) proc.status_raise(timeout = 0) if len(result) > 0: return list(result)
def discover(self): active = False keys = [WMS.MEMORY, WMS.CPUTIME, WMS.WALLTIME] parser = dict(izip(keys, [int, parseTime, parseTime])) proc = LocalProcess(self._exec, '-q') for line in proc.stdout.iter(timeout=10): if line.startswith('-'): active = True elif line.startswith(' '): active = False elif active: fields = lmap(str.strip, line.split()[:4]) queueInfo = {'name': fields[0]} for key, value in ifilter( lambda k_v: not k_v[1].startswith('-'), izip(keys, fields[1:])): queueInfo[key] = parser[key](value) yield queueInfo proc.status_raise(timeout=0)
def se_runcmd(cmd, varDict, *urls): runLib = utils.pathShare('gc-run.lib') args = str.join( ' ', imap(lambda x: '"%s"' % ensurePrefix(x).replace('dir://', 'file://'), urls)) varString = str.join( ' ', imap(lambda x: 'export %s="%s";' % (x, varDict[x]), varDict)) return LocalProcess( '/bin/bash', '-c', '. %s || exit 99; %s %s %s' % (runLib, varString, cmd, args))
def _submit_job(self, jobnum, task): # Submit job and yield (jobnum, WMS ID, other data) jdl_fd, jdl_fn = tempfile.mkstemp('.jdl') try: jdl_line_list = self._make_jdl(jobnum, task) safe_write(os.fdopen(jdl_fd, 'w'), jdl_line_list) except Exception: remove_files([jdl_fn]) raise BackendError('Could not write jdl data to %s.' % jdl_fn) try: submit_arg_list = [] for key_value in filter_dict(self._submit_args_dict, value_filter=identity).items(): submit_arg_list.extend(key_value) submit_arg_list.append(jdl_fn) activity = Activity('submitting job %d' % jobnum) proc = LocalProcess(self._submit_exec, '--nomsg', '--noint', '--logfile', '/dev/stderr', *submit_arg_list) wms_id = None stripped_stdout_iter = imap(str.strip, proc.stdout.iter(timeout=60)) for line in ifilter(lambda x: x.startswith('http'), stripped_stdout_iter): wms_id = line exit_code = proc.status(timeout=0, terminate=True) activity.finish() if (exit_code != 0) or (wms_id is None): if self._explain_error(proc, exit_code): pass else: self._log.log_process( proc, files={'jdl': SafeFile(jdl_fn).read()}) finally: remove_files([jdl_fn]) job_data = {'jdl': str.join('', jdl_line_list)} return (jobnum, self._create_gc_id(wms_id), job_data)
def checkJobs(self, ids): if len(ids) == 0: raise StopIteration jobNumMap = dict(ids) jobs = self.writeWMSIds(ids) activity = utils.ActivityLog('checking job status') proc = LocalProcess(self._statusExec, '--verbosity', 1, '--noint', '--logfile', '/dev/stderr', '-i', jobs) for data in self._parseStatus(proc.stdout.iter(timeout = 60)): data['id'] = self._createId(data['id']) yield (jobNumMap.get(data['id']), data['id'], self._statusMap[data['status']], data) retCode = proc.status(timeout = 0, terminate = True) del activity if retCode != 0: if self.explainError(proc, retCode): pass else: self._log.log_process(proc, files = {'jobs': utils.safeRead(jobs)}) utils.removeFiles([jobs])
def _parse_proxy(self, cached=True): # Return cached results if requested if cached and self._cache: return self._cache # Call voms-proxy-info and parse results proc = LocalProcess(self._proxy_info_exec, *self._get_proxy_info_arguments()) (exit_code, stdout, stderr) = proc.finish(timeout=10) if (exit_code != 0) and not self._ignore_warning: msg = ('%s output:\n%s\n%s\n' % (self._proxy_info_exec, stdout, stderr)).replace( '\n\n', '\n') msg += 'If job submission is still possible, you can set [access] ignore warnings = True\n' msg += '%s failed with return code %d' % (self._proxy_info_exec, exit_code) raise AccessTokenError(msg) self._cache = DictFormat(':').parse(stdout) if not self._cache: msg = 'Unable to parse access token information:\n\t%s\n\t%s\n' raise AccessTokenError(msg % (stdout.strip(), stderr.strip())) return self._cache
def bulkSubmissionBegin(self): self._submitParams.update({ '-d': None }) if self._discovery_module: self._submitParams.update({ '-e': self._discovery_module.getWMS() }) if self._useDelegate is False: self._submitParams.update({ '-a': ' ' }) return True dID = 'GCD' + md5_hex(str(time.time()))[:10] activity = utils.ActivityLog('creating delegate proxy for job submission') deletegateArgs = [] if self._configVO: deletegateArgs.extend(['--config', self._configVO]) proc = LocalProcess(self._delegateExec, '-d', dID, '--noint', '--logfile', '/dev/stderr', *deletegateArgs) output = proc.get_output(timeout = 10, raise_errors = False) if ('glite-wms-job-delegate-proxy Success' in output) and (dID in output): self._submitParams.update({ '-d': dID }) del activity if proc.status(timeout = 0, terminate = True) != 0: self._log.log_process(proc) return (self._submitParams.get('-d', None) is not None)
def _get_submit_proc(self, jobnum, sandbox, job_name, reqs): (stdout, stderr) = (os.path.join(sandbox, 'gc.stdout'), os.path.join(sandbox, 'gc.stderr')) submit_args = list(self._submit_opt_list) submit_args.extend( shlex.split( self._get_submit_arguments(jobnum, job_name, reqs, sandbox, stdout, stderr))) submit_args.append(get_path_share('gc-local.sh')) submit_args.extend( shlex.split(self._get_job_arguments(jobnum, sandbox))) return LocalProcess(self._submit_exec, *submit_args)
def _begin_bulk_submission(self): self._submit_args_dict.update({'-D': None}) if self._use_delegate is False: self._submit_args_dict.update({'-a': ' '}) return True delegate_id = 'GCD' + md5_hex(str(time.time()))[:10] activity = Activity('creating delegate proxy for job submission') delegate_arg_list = ['-e', self._ce[:self._ce.rfind("/")]] if self._config_fn: delegate_arg_list.extend(['--config', self._config_fn]) proc = LocalProcess(self._delegate_exec, '-d', delegate_id, '--logfile', '/dev/stderr', *delegate_arg_list) output = proc.get_output(timeout=10, raise_errors=False) if ('succesfully delegated to endpoint' in output) and (delegate_id in output): self._submit_args_dict.update({'-D': delegate_id}) activity.finish() if proc.status(timeout=0, terminate=True) != 0: self._log.log_process(proc) return self._submit_args_dict.get('-D') is not None
def _match_sites(self, endpoint): activity = Activity('Discovering available WMS services - testing %s' % endpoint) check_arg_list = ['-a'] if endpoint: check_arg_list.extend(['-e', endpoint]) check_arg_list.append(get_path_share('null.jdl')) proc = LocalProcess(self._job_list_match_exec, *check_arg_list) result = [] for line in proc.stdout.iter(timeout=3): if line.startswith(' - '): result.append(line[3:].strip()) activity.finish() if proc.status(timeout=0) is None: self._wms_timeout_dict[endpoint] = self._wms_timeout_dict.get( endpoint, 0) + 1 if self._wms_timeout_dict.get( endpoint, 0) > 10: # remove endpoints after 10 failures self._wms_list_all.remove(endpoint) return [] return result
def _begin_bulk_submission(self): self._submit_args_dict.update({'-d': None}) if self._discovery_plugin: self._submit_args_dict.update({'-e': self._discovery_plugin.get_endpoint()}) if self._use_delegate is False: self._submit_args_dict.update({'-a': ' '}) return True delegate_id = 'GCD' + md5_hex(str(time.time()))[:10] activity = Activity('creating delegate proxy for job submission') delegate_arg_list = [] if self._config_fn: delegate_arg_list.extend(['--config', self._config_fn]) proc = LocalProcess(self._delegate_exec, '-d', delegate_id, '--noint', '--logfile', '/dev/stderr', *delegate_arg_list) output = proc.get_output(timeout=10, raise_errors=False) if ('glite-wms-job-delegate-proxy Success' in output) and (delegate_id in output): self._submit_args_dict.update({'-d': delegate_id}) activity.finish() if proc.status(timeout=0, terminate=True) != 0: self._log.log_process(proc) return self._submit_args_dict.get('-d') is not None
def _recover_jobs(self): proc = LocalProcess('zip', '-FF', self._db_fn, '--out', '%s.tmp' % self._db_fn) proc.stdin.write('y\n') proc.status(timeout=None) os.rename(self._db_fn, self._db_fn + '.broken') os.rename(self._db_fn + '.tmp', self._db_fn) tar = zipfile.ZipFile(self._db_fn, 'r', zipfile.ZIP_DEFLATED) remove_files([self._db_fn + '.broken']) broken_fn_list = [] for tar_info_fn in tar.namelist(): try: tuple(imap(lambda s: int(s[1:]), tar_info_fn.split('_', 1))) # check name fp = tar.open(tar_info_fn) try: fp.read() finally: fp.close() except Exception: clear_current_exception() broken_fn_list.append(tar_info_fn) for broken in broken_fn_list: os.system('zip %s -d %s' % (self._db_fn, broken))
def _submitJob(self, jobNum, module): fd, jdl = tempfile.mkstemp('.jdl') try: jdlData = self.makeJDL(jobNum, module) utils.safeWrite(os.fdopen(fd, 'w'), jdlData) except Exception: utils.removeFiles([jdl]) raise BackendError('Could not write jdl data to %s.' % jdl) try: submitArgs = [] for key_value in utils.filterDict(self._submitParams, vF=lambda v: v).items(): submitArgs.extend(key_value) submitArgs.append(jdl) activity = Activity('submitting job %d' % jobNum) proc = LocalProcess(self._submitExec, '--nomsg', '--noint', '--logfile', '/dev/stderr', *submitArgs) gcID = None for line in ifilter(lambda x: x.startswith('http'), imap(str.strip, proc.stdout.iter(timeout=60))): gcID = line retCode = proc.status(timeout=0, terminate=True) activity.finish() if (retCode != 0) or (gcID is None): if self.explainError(proc, retCode): pass else: self._log.log_process(proc, files={'jdl': SafeFile(jdl).read()}) finally: utils.removeFiles([jdl]) return (jobNum, utils.QM(gcID, self._createId(gcID), None), { 'jdl': str.join('', jdlData) })
def discover(self): tags = ['h_vmem', 'h_cpu', 's_rt'] reqs = dict(izip(tags, [WMS.MEMORY, WMS.CPUTIME, WMS.WALLTIME])) parser = dict(izip(tags, [int, parseTime, parseTime])) proc = LocalProcess(self._configExec, '-sql') for queue in imap(str.strip, proc.stdout.iter(timeout=10)): proc_q = LocalProcess(self._configExec, '-sq', queue) queueInfo = {'name': queue} for line in proc_q.stdout.iter(timeout=10): attr, value = lmap(str.strip, line.split(' ', 1)) if (attr in tags) and (value != 'INFINITY'): queueInfo[reqs[attr]] = parser[attr](value) proc_q.status_raise(timeout=0) yield queueInfo proc.status_raise(timeout=0)
def _submit_job(self, jobnum, task): # Submit job and yield (jobnum, WMS ID, other data) jdl_fd, jdl_fn = tempfile.mkstemp('.jdl') try: jdl_line_list = self._make_jdl(jobnum, task) safe_write(os.fdopen(jdl_fd, 'w'), jdl_line_list) except Exception: remove_files([jdl_fn]) raise BackendError('Could not write jdl data to %s.' % jdl_fn) try: submit_arg_list = [] for key_value in filter_dict(self._submit_args_dict, value_filter=identity).items(): submit_arg_list.extend(key_value) submit_arg_list.append(jdl_fn) activity = Activity('submitting job %d' % jobnum) proc = LocalProcess(self._submit_exec, '--nomsg', '--noint', '--logfile', '/dev/stderr', *submit_arg_list) wms_id = None stripped_stdout_iter = imap(str.strip, proc.stdout.iter(timeout=60)) for line in ifilter(lambda x: x.startswith('http'), stripped_stdout_iter): wms_id = line exit_code = proc.status(timeout=0, terminate=True) activity.finish() if (exit_code != 0) or (wms_id is None): if self._explain_error(proc, exit_code): pass else: self._log.log_process(proc, files={'jdl': SafeFile(jdl_fn).read()}) finally: remove_files([jdl_fn]) job_data = {'jdl': str.join('', jdl_line_list)} return (jobnum, self._create_gc_id(wms_id), job_data)
def checkJobs(self, ids): if len(ids) == 0: raise StopIteration jobNumMap = dict(ids) jobs = self.writeWMSIds(ids) activity = utils.ActivityLog('checking job status') proc = LocalProcess(self._statusExec, '--verbosity', 1, '--noint', '--logfile', '/dev/stderr', '-i', jobs) for data in self._parseStatus(proc.stdout.iter(timeout=60)): data['id'] = self._createId(data['id']) yield (jobNumMap.get(data['id']), data['id'], self._statusMap[data['status']], data) retCode = proc.status(timeout=0, terminate=True) del activity if retCode != 0: if self.explainError(proc, retCode): pass else: self._log.log_process(proc, files={'jobs': utils.safeRead(jobs)}) utils.removeFiles([jobs])
def getQueues(self): queues = {} tags = ['h_vmem', 'h_cpu', 's_rt'] reqs = dict(izip(tags, [WMS.MEMORY, WMS.CPUTIME, WMS.WALLTIME])) parser = dict(izip(tags, [int, parseTime, parseTime])) proc = LocalProcess(self._configExec, '-sql') for queue in imap(str.strip, proc.stdout.iter(timeout=10)): queues[queue] = dict() proc_q = LocalProcess(self._configExec, '-sq %s' % queue) for line in proc_q.stdout.iter(timeout=10): attr, value = lmap(str.strip, line.split(' ', 1)) if (attr in tags) and (value != 'INFINITY'): queues[queue][reqs[attr]] = parser[attr](value) proc_q.status_raise(timeout=0) proc.status_raise(timeout=0) return queues
def getNodes(self): result = set() proc = LocalProcess(self._configExec, '-shgrpl') for group in proc.stdout.iter(timeout = 10): result.add(group.strip()) proc_g = LocalProcess(self._configExec, '-shgrp_resolved %s' % group) for host in proc_g.stdout.iter(timeout = 10): result.update(host.split()) proc_g.status_raise(timeout = 0) proc.status_raise(timeout = 0) if len(result) > 0: return list(result)
def getVersion(): try: proc_ver = LocalProcess('svnversion', '-c', pathPKG()) version = proc_ver.get_output(timeout=10).strip() if version != '': assert (any(imap(str.isdigit, version))) proc_branch = LocalProcess('svn info', pathPKG()) if 'stable' in proc_branch.get_output(timeout=10): return '%s - stable' % version return '%s - testing' % version except Exception: pass return __import__('grid_control').__version__ + ' or later'
def image(self): proc = LocalProcess('neato', '-Tpng') proc.stdin.write(self._get_workflow_graph()) proc.stdin.close() cherrypy.response.headers['Content-Type'] = 'image/png' return proc.get_output(timeout=20)