def _get_cms_cert(config): config = config.change_view(set_sections=['cms', 'access', 'proxy']) try: access = AccessToken.create_instance('VomsAccessToken', config, 'cms-proxy') except Exception: if os.environ.get('X509_USER_PROXY'): return os.environ['X509_USER_PROXY'] raise CMSAuthenticationException('Unable to find grid environment') can_submit = ignore_exception(Exception, False, access.can_submit, 5 * 60, True) if not can_submit: logging.getLogger('access.cms').warning('The grid proxy has expired or is invalid!') role = config.get_list('new proxy roles', '', on_change=None) timeout = config.get_time('new proxy timeout', 10, on_change=None) lifetime = config.get_time('new proxy lifetime', 192 * 60, on_change=None) # password in variable name removes it from debug log password = getpass.getpass('Please enter proxy password: '******'voms-proxy-init') proc = LocalProcess(proxy_init_exec, '--voms', str.join(':', ['cms'] + role), '--valid', '%d:%d' % (lifetime / 60, lifetime % 60), logging=False) if password: proc.stdin.write(password + '\n') proc.stdin.close() proc.get_output(timeout=timeout) except Exception: raise CMSAuthenticationException('Unable to create new grid proxy') access = AccessToken.create_instance('VomsAccessToken', config, 'cms-proxy') # new instance can_submit = ignore_exception(Exception, False, access.can_submit, 5 * 60, True) if not can_submit: raise CMSAuthenticationException('Newly created grid proxy is also invalid') return access.get_auth_fn_list()[0]
def getVersion(): try: proc_ver = LocalProcess('svnversion', '-c', pathPKG()) version = proc_ver.get_output(timeout = 10).strip() if version != '': assert(lfilter(str.isdigit, version)) proc_branch = LocalProcess('svn info', pathPKG()) if 'stable' in proc_branch.get_output(timeout = 10): return '%s - stable' % version return '%s - testing' % version except Exception: clear_current_exception() return __import__('grid_control').__version__ + ' or later'
def getVersion(): try: proc_ver = LocalProcess('svnversion', '-c', pathPKG()) version = proc_ver.get_output(timeout=10).strip() if version != '': assert (any(imap(str.isdigit, version))) proc_branch = LocalProcess('svn info', pathPKG()) if 'stable' in proc_branch.get_output(timeout=10): return '%s - stable' % version return '%s - testing' % version except Exception: pass return __import__('grid_control').__version__ + ' or later'
def _scriptThread(self, script, jobNum=None, jobObj=None, allDict=None): try: tmp = {} if jobObj is not None: for key, value in jobObj.get_dict().items(): tmp[key.upper()] = value tmp['WORKDIR'] = self._workPath tmp.update(self._task.getTaskConfig()) if jobNum is not None: tmp.update(self._task.getJobConfig(jobNum)) tmp.update(allDict or {}) env = dict(os.environ) for key, value in tmp.items(): if not key.startswith('GC_'): key = 'GC_' + key env[key] = str(value) script = self._task.substVars('monitoring script', script, jobNum, tmp) if not self._silent: proc = LocalProcess(*shlex.split(script), **{'environment': env}) proc_output = proc.get_output(timeout=self._runningMax) if proc_output.strip(): self._log.info(proc_output.strip()) else: os.system(script) except Exception: self._log.exception('Error while running user script')
def _set_proxy_lifetime(self): activity = Activity('Get proxy lifetime...') proc = LocalProcess(resolve_install_path('voms-proxy-info')) output = proc.get_output(timeout=10, raise_errors=False) end_of_proxy = 0 proxy_key = None for l in output.split('\n'): if 'subject' in l: proxy_key = l.encode("hex")[-15:] if 'timeleft' in l: h, m, s = int(l.split(':')[-3]), int(l.split(':')[-2]), int( l.split(':')[-1]) end_of_proxy = time.time() + h * 60 * 60 + m * 60 + s break if end_of_proxy == 0: self._log.warning('couldnt evaluate end of proxy. Output was:') self._log.warning(output) time.sleep(300) self._set_proxy_lifetime() else: self._end_of_proxy_lifetime = end_of_proxy if proxy_key is not None: self._delegated_proxy_filename = os.path.join( os.path.expanduser("~"), ".gcDelegatedProxy" + proxy_key) left_time_str = datetime.fromtimestamp( self._end_of_proxy_lifetime).strftime("%A, %B %d, %Y %I:%M:%S") self._log.info('End of current proxy lifetime: %s' % left_time_str) activity.finish() return 0
def _get_version(self, value): old_wd = os.getcwd() os.chdir(clean_path(value)) git_proc = LocalProcess('git', 'rev-parse', '--short', 'HEAD') version = git_proc.get_output(timeout=10, raise_errors=False) os.chdir(old_wd) return version.strip() or 'undefined'
def _script_thread(self, script, task, jobnum=None, job_obj=None, add_dict=None): # Get both task and job config / state dicts try: tmp = {} if job_obj is not None: for key, value in job_obj.get_dict().items(): tmp[key.upper()] = value tmp['GC_WORKDIR'] = self._path_work if jobnum is not None: tmp.update(task.get_job_dict(jobnum)) tmp.update(add_dict or {}) env = dict(os.environ) for key, value in tmp.items(): if not key.startswith('GC_'): key = 'GC_' + key env[key] = str(value) script = task.substitute_variables('monitoring script', script, jobnum, tmp) if not self._silent: proc = LocalProcess(*shlex.split(script), **{'env_dict': env}) proc_output = proc.get_output(timeout=self._script_timeout) if proc_output.strip(): self._log.info(proc_output.strip()) else: os.system(script) except Exception: self._log.exception('Error while running user script') clear_current_exception()
def _scriptThread(self, script, jobNum = None, jobObj = None, allDict = None): try: tmp = {} if jobNum is not None: tmp.update(self._task.getSubmitInfo(jobNum)) if jobObj is not None: tmp.update(jobObj.getAll()) tmp['WORKDIR'] = self._workPath tmp.update(self._task.getTaskConfig()) if jobNum is not None: tmp.update(self._task.getJobConfig(jobNum)) tmp.update(self._task.getSubmitInfo(jobNum)) tmp.update(allDict or {}) for key, value in tmp.items(): if not key.startswith('GC_'): key = 'GC_' + key os.environ[key] = str(value) script = self._task.substVars(script, jobNum, tmp) if not self._silent: proc = LocalProcess(script) self._log.info(proc.get_output(timeout = self._runningMax)) else: os.system(script) except Exception: self._log.exception('Error while running user script!')
def _scriptThread(self, script, jobNum = None, jobObj = None, allDict = None): try: tmp = {} if jobObj is not None: for key, value in jobObj.get_dict().items(): tmp[key.upper()] = value tmp['WORKDIR'] = self._workPath tmp.update(self._task.getTaskConfig()) if jobNum is not None: tmp.update(self._task.getJobConfig(jobNum)) tmp.update(allDict or {}) env = dict(os.environ) for key, value in tmp.items(): if not key.startswith('GC_'): key = 'GC_' + key env[key] = str(value) script = self._task.substVars('monitoring script', script, jobNum, tmp) if not self._silent: proc = LocalProcess(*shlex.split(script), **{'environment': env}) proc_output = proc.get_output(timeout = self._runningMax) if proc_output.strip(): self._log.info(proc_output.strip()) else: os.system(script) except Exception: self._log.exception('Error while running user script')
def _get_version(self, value): old_wd = os.getcwd() os.chdir(clean_path(value)) git_proc = LocalProcess('git', 'rev-parse', '--short', 'HEAD') version = git_proc.get_output(timeout=10, raise_errors=False) os.chdir(old_wd) return version.strip() or 'undefined'
def _get_version(self, value): svn_proc = LocalProcess('svnversion', clean_path(value)) version = svn_proc.get_output(timeout=10, raise_errors=False).strip().lower() # different SVN versions yield different output for unversioned directories: if ('exported' in version) or ('unversioned' in version): version = None return version or 'undefined'
def _scriptThread(self, script, jobNum=None, jobObj=None, allDict=None): try: tmp = {} if jobNum is not None: tmp.update(self._task.getSubmitInfo(jobNum)) if jobObj is not None: tmp.update(jobObj.getAll()) tmp['WORKDIR'] = self._workPath tmp.update(self._task.getTaskConfig()) if jobNum is not None: tmp.update(self._task.getJobConfig(jobNum)) tmp.update(self._task.getSubmitInfo(jobNum)) tmp.update(allDict or {}) for key, value in tmp.items(): if not key.startswith('GC_'): key = 'GC_' + key os.environ[key] = str(value) script = self._task.substVars(script, jobNum, tmp) if not self._silent: proc = LocalProcess(script) self._log.info(proc.get_output(timeout=self._runningMax)) else: os.system(script) except Exception: self._log.exception('Error while running user script!')
def ping_host(host): try: proc = LocalProcess('ping', '-Uqnc', 1, '-W', 1, host) tmp = proc.get_output(timeout=1).splitlines() assert (tmp[-1].endswith('ms')) return float(tmp[-1].split('/')[-2]) / 1000. except Exception: return None
def _get_version(self, value): svn_proc = LocalProcess('svnversion', clean_path(value)) version = svn_proc.get_output(timeout=10, raise_errors=False).strip().lower() # different SVN versions yield different output for unversioned directories: if ('exported' in version) or ('unversioned' in version): version = None return version or 'undefined'
def ping_host(host): proc = LocalProcess('ping', '-Uqnc', 1, '-W', 1, host) try: tmp = proc.get_output(timeout = 1).splitlines() assert(tmp[-1].endswith('ms')) return float(tmp[-1].split('/')[-2]) / 1000. except Exception: return None
def _get_git_version(path): from grid_control.utils.process_base import LocalProcess from grid_control.utils import clean_path path = clean_path(path) old_wd = os.getcwd() os.chdir(path) git_proc = LocalProcess('git', 'rev-parse', '--short', 'HEAD') version = git_proc.get_output(timeout=10, raise_errors=False) os.chdir(old_wd) return version.strip() or None
def _begin_bulk_submission(self): self._submit_args_dict.update({'-D': None}) if self._use_delegate is False: self._submit_args_dict.update({'-a': ' '}) return True delegate_id = 'GCD' + md5_hex(str(time.time()))[:10] activity = Activity('creating delegate proxy for job submission') delegate_arg_list = ['-e', self._ce[:self._ce.rfind("/")]] if self._config_fn: delegate_arg_list.extend(['--config', self._config_fn]) proc = LocalProcess(self._delegate_exec, '-d', delegate_id, '--logfile', '/dev/stderr', *delegate_arg_list) output = proc.get_output(timeout=10, raise_errors=False) if ('succesfully delegated to endpoint' in output) and (delegate_id in output): self._submit_args_dict.update({'-D': delegate_id}) activity.finish() if proc.status(timeout=0, terminate=True) != 0: self._log.log_process(proc) return self._submit_args_dict.get('-D') is not None
def bulkSubmissionBegin(self): self._submitParams.update({ '-d': None }) if self._discovery_module: self._submitParams.update({ '-e': self._discovery_module.getWMS() }) if self._useDelegate is False: self._submitParams.update({ '-a': ' ' }) return True dID = 'GCD' + md5_hex(str(time.time()))[:10] activity = utils.ActivityLog('creating delegate proxy for job submission') deletegateArgs = [] if self._configVO: deletegateArgs.extend(['--config', self._configVO]) proc = LocalProcess(self._delegateExec, '-d', dID, '--noint', '--logfile', '/dev/stderr', *deletegateArgs) output = proc.get_output(timeout = 10, raise_errors = False) if ('glite-wms-job-delegate-proxy Success' in output) and (dID in output): self._submitParams.update({ '-d': dID }) del activity if proc.status(timeout = 0, terminate = True) != 0: self._log.log_process(proc) return (self._submitParams.get('-d', None) is not None)
def _begin_bulk_submission(self): self._submit_args_dict.update({'-d': None}) if self._discovery_plugin: self._submit_args_dict.update({'-e': self._discovery_plugin.get_endpoint()}) if self._use_delegate is False: self._submit_args_dict.update({'-a': ' '}) return True delegate_id = 'GCD' + md5_hex(str(time.time()))[:10] activity = Activity('creating delegate proxy for job submission') delegate_arg_list = [] if self._config_fn: delegate_arg_list.extend(['--config', self._config_fn]) proc = LocalProcess(self._delegate_exec, '-d', delegate_id, '--noint', '--logfile', '/dev/stderr', *delegate_arg_list) output = proc.get_output(timeout=10, raise_errors=False) if ('glite-wms-job-delegate-proxy Success' in output) and (delegate_id in output): self._submit_args_dict.update({'-d': delegate_id}) activity.finish() if proc.status(timeout=0, terminate=True) != 0: self._log.log_process(proc) return self._submit_args_dict.get('-d') is not None
def image(self): proc = LocalProcess('neato', '-Tpng') proc.stdin.write(self._get_workflow_graph()) proc.stdin.close() cherrypy.response.headers['Content-Type'] = 'image/png' return proc.get_output(timeout=20)
def image(self): proc = LocalProcess('neato', '-Tpng') proc.stdin.write(self._get_workflow_graph()) proc.stdin.close() cherrypy.response.headers['Content-Type'] = 'image/png' return proc.get_output(timeout = 20)
def _begin_bulk_submission(self): self._set_proxy_lifetime() if self._end_of_proxy_lifetime is None: raise Exception("_end_of_proxy_lifetime is not set") if self._delegated_proxy_filename is None: raise Exception("_delegated_proxy_filename is not set") if self._end_of_proxy_lifetime <= time.time(): self._log.info( "renew proxy is necessary: %s <= %s" % (str(self._end_of_proxy_lifetime), str(time.time()))) x = threading.Thread(target=CreamWMS.delfile, args=(self._lock_filename, 0, self._log)) x.start() y = threading.Thread(target=CreamWMS.delfile, args=(self._delegated_proxy_filename, 0, self._log)) y.start() raise Exception("renew proxy is necessary") elif '-D' in self._submit_args_dict.keys( ) and self._submit_args_dict['-D'] is not None: try: left_time_str = timedelta(seconds=self._end_of_proxy_lifetime - time.time()) except: left_time_str = str(self._end_of_proxy_lifetime - time.time()) + ' sec.' self._log.info( "Proxy delegation IS NOT ISSUED since expected to be OK. left: %s " % left_time_str) else: if os.path.isfile(self._delegated_proxy_filename): file = open(self._delegated_proxy_filename, "r") delegate_id = file.read() file.close() # file is empty -> another process edditing it? # if delegate_id is None or delegate_id == '': return False if delegate_id is not None and delegate_id != "": self._submit_args_dict.update({'-D': delegate_id}) self._log.info('Proxy delegation read from a file: %s ' % (delegate_id)) elif not os.path.isfile( self._delegated_proxy_lock ): #not os.path.isfile(self._delegated_proxy_filename): file_lock = open(self._delegated_proxy_lock, "w+") file = open(self._delegated_proxy_filename, "w+") activity = Activity('Delegating proxy for job submission') self._submit_args_dict.update({'-D': None}) #if self._use_delegate is False: # self._submit_args_dict.update({'-a': ' '}) # return True t = time.time() thehex = md5_hex(str(t)) self._log.info('Proxy delegation full hex: %s at time %s' % (thehex, str(t))) delegate_id = 'GCD' + thehex[:15] delegate_arg_list = ['-e', self._ce[:self._ce.rfind("/")]] if self._config_fn: delegate_arg_list.extend(['--config', self._config_fn]) proc = LocalProcess(self._delegate_exec, '-d', delegate_id, '--logfile', '/dev/stderr', *delegate_arg_list) output = proc.get_output(timeout=10, raise_errors=False) if ('succesfully delegated to endpoint' in output) and (delegate_id in output): self._submit_args_dict.update({'-D': delegate_id}) activity.finish() if proc.status(timeout=0, terminate=True) != 0: self._log.log_process(proc) file.write(delegate_id) file.close() file_lock.close() y = threading.Thread(target=CreamWMS.delfile, args=(self._delegated_proxy_lock, 0, self._log)) y.start() return self._submit_args_dict.get('-D') is not None