def _get_sandbox_file_list(self, task, sm_list): # Prepare all input files dep_list = set(ichain(imap(lambda x: x.get_dependency_list(), [task] + sm_list))) dep_fn_list = lmap(lambda dep: resolve_path('env.%s.sh' % dep, lmap(lambda pkg: get_path_share('', pkg=pkg), os.listdir(get_path_pkg()))), dep_list) task_config_dict = dict_union(self._remote_event_handler.get_mon_env_dict(), *imap(lambda x: x.get_task_dict(), [task] + sm_list)) task_config_dict.update({'GC_DEPFILES': str.join(' ', dep_list), 'GC_USERNAME': self._token.get_user_name(), 'GC_WMS_NAME': self._name}) task_config_str_list = DictFormat(escape_strings=True).format( task_config_dict, format='export %s%s%s\n') vn_alias_dict = dict(izip(self._remote_event_handler.get_mon_env_dict().keys(), self._remote_event_handler.get_mon_env_dict().keys())) vn_alias_dict.update(task.get_var_alias_map()) vn_alias_str_list = DictFormat(delimeter=' ').format(vn_alias_dict, format='%s%s%s\n') # Resolve wildcards in task input files def _get_task_fn_list(): for fpi in task.get_sb_in_fpi_list(): matched = glob.glob(fpi.path_abs) if matched != []: for match in matched: yield match else: yield fpi.path_abs return lchain([self._remote_event_handler.get_file_list(), dep_fn_list, _get_task_fn_list(), [ VirtualFile('_config.sh', sorted(task_config_str_list)), VirtualFile('_varmap.dat', sorted(vn_alias_str_list))]])
def _iter_datasource_items(self, item, metadata_dict, entries, location_list, obj_dict): job_info_path = os.path.join(item, 'job.info') try: job_info_dict = DictFormat('=').parse(open(job_info_path)) if job_info_dict.get('exitcode') == 0: obj_dict['JOBINFO'] = job_info_dict yield (item, metadata_dict, entries, location_list, obj_dict) except Exception: self._log.log(logging.INFO2, 'Unable to parse job info file %r', job_info_path) clear_current_exception()
def __init__(self, config, job_limit=-1, job_selector=None): JobDB.__init__(self, config, job_limit, job_selector) self._path_db = config.get_work_path('jobs') self._fmt = DictFormat(escape_strings=True) try: self._job_map = self._read_jobs(self._job_limit) except Exception: raise JobError('Unable to read stored job information!') if self._job_limit < 0 and len(self._job_map) > 0: self._job_limit = max(self._job_map) + 1
def _parse(self, proc): for section in accumulate(proc.stdout.iter(self._timeout), '', lambda x, buf: x == '\n'): try: lines = section.replace('\n\t', '').split('\n') job_info = DictFormat(' = ').parse(lines[1:]) job_info[CheckInfo.WMSID] = lines[0].split(':')[1].split('.')[0].strip() job_info[CheckInfo.RAW_STATUS] = job_info.pop('job_state') job_info[CheckInfo.QUEUE] = job_info.pop('queue', None) if 'exec_host' in job_info: exec_host = job_info.pop('exec_host').split('/')[0] job_info[CheckInfo.WN] = exec_host + '.' + job_info.get('server', '') except Exception: raise BackendError('Error reading job info:\n%s' % section) yield job_info
class JobInfoProcessor(OutputProcessor): def __init__(self): OutputProcessor.__init__(self) self._df = DictFormat() def process(self, dn): fn = os.path.join(dn, 'job.info') try: if not os.path.exists(fn): raise JobResultError('Job result file %r does not exist' % fn) try: info_content = open(fn, 'r').read() except Exception: raise JobResultError('Unable to read job result file %r' % fn) if not info_content: raise JobResultError('Job result file %r is empty' % fn) data = self._df.parse(info_content, keyParser = {None: str}) # impossible to fail try: jobNum = data.pop('JOBID') exitCode = data.pop('EXITCODE') return {JobResult.JOBNUM: jobNum, JobResult.EXITCODE: exitCode, JobResult.RAW: data} except Exception: raise JobResultError('Job result file %r is incomplete' % fn) except Exception: raise JobResultError('Unable to process output directory %r' % dn)
class JobInfoProcessor(OutputProcessor): alias_list = ['jobinfo'] def __init__(self): OutputProcessor.__init__(self) self._df = DictFormat() def process(self, dn): fn = os.path.join(dn, 'job.info') try: if not os.path.exists(fn): raise JobResultError('Job result file %r does not exist' % fn) try: info_content = SafeFile(fn).read_close() except Exception: raise JobResultError('Unable to read job result file %r' % fn) if not info_content: raise JobResultError('Job result file %r is empty' % fn) data = self._df.parse(info_content, key_parser={None: str}) # impossible to fail try: jobnum = data.pop('JOBID') exit_code = data.pop('EXITCODE') message = data.pop('MESSAGE', None) return {JobResult.JOBNUM: jobnum, JobResult.EXITCODE: exit_code, JobResult.MESSAGE: message, JobResult.RAW: data} except Exception: raise JobResultError('Job result file %r is incomplete' % fn) except Exception: raise JobResultError('Unable to process output directory %r' % dn)
def logError(self, target, brief=False, **kwargs): # Can also log content of additional files via kwargs now = time.time() entry = '%s.%s' % (time.strftime('%Y-%m-%d_%H:%M:%S', time.localtime(now)), ('%.5f' % (now - int(now)))[2:]) self._logger.log_time(logging.WARNING, '%s failed with code %d', self.niceCmd, self.wait()) if not brief: self._logger.log_time(logging.WARNING, '\n%s', self.getError()) try: tar = tarfile.TarFile.open(target, 'a') data = {'retCode': self.wait(), 'exec': self.cmd, 'args': self.args} files = [VirtualFile(os.path.join(entry, 'info'), DictFormat().format(data))] kwargs.update({'stdout': self.getOutput(), 'stderr': self.getError()}) for key, value in kwargs.items(): try: content = open(value, 'r').readlines() except Exception: content = [value] files.append(VirtualFile(os.path.join(entry, key), content)) for fileObj in files: info, handle = fileObj.getTarInfo() tar.addfile(info, handle) handle.close() tar.close() except Exception: raise GCError('Unable to log errors of external process "%s" to "%s"' % (self.niceCmd, target)) self._logger.info('All logfiles were moved to %s', target)
class JobInfoProcessor(OutputProcessor): def __init__(self): OutputProcessor.__init__(self) self._df = DictFormat() def process(self, dn): fn = os.path.join(dn, 'job.info') if not os.path.exists(fn): raise JobResultError('Job result file %r does not exist' % fn) try: info_content = open(fn, 'r').read() except Exception: raise JobResultError('Unable to read job result file %r' % fn) if not info_content: raise JobResultError('Job result file %r is empty' % fn) try: data = self._df.parse(info_content, keyParser={None: str}) except Exception: raise JobResultError('Unable to parse job result file %r' % fn) try: jobNum = data.pop('JOBID') exitCode = data.pop('EXITCODE') return { JobResult.JOBNUM: jobNum, JobResult.EXITCODE: exitCode, JobResult.RAW: data } except Exception: raise JobResultError('Job result file %r is incomplete' % fn)
class JobInfoProcessor(OutputProcessor): alias_list = ['jobinfo'] def __init__(self): OutputProcessor.__init__(self) self._df = DictFormat() def process(self, dn): fn = os.path.join(dn, 'job.info') try: if not os.path.exists(fn): raise JobResultError('Job result file %r does not exist' % fn) try: info_content = SafeFile(fn).read_close() except Exception: raise JobResultError('Unable to read job result file %r' % fn) if not info_content: raise JobResultError('Job result file %r is empty' % fn) data = self._df.parse(info_content, key_parser={None: str}) # impossible to fail try: jobnum = data.pop('JOBID') exit_code = data.pop('EXITCODE') message = data.pop('MESSAGE', None) return { JobResult.JOBNUM: jobnum, JobResult.EXITCODE: exit_code, JobResult.MESSAGE: message, JobResult.RAW: data } except Exception: raise JobResultError('Job result file %r is incomplete' % fn) except Exception: raise JobResultError('Unable to process output directory %r' % dn)
def _write_job_config(self, job_config_fn, jobnum, task, extras): try: job_env_dict = dict_union(task.get_job_dict(jobnum), extras) job_env_dict['GC_ARGS'] = task.get_job_arguments(jobnum).strip() content = DictFormat(escape_strings=True).format(job_env_dict, format='export %s%s%s\n') safe_write(open(job_config_fn, 'w'), content) except Exception: raise BackendError('Could not write job config data to %s.' % job_config_fn)
def __init__(self, path): activity = Activity('Reading dataset partition file') self._fmt = DictFormat() try: self._tar = tarfile.open(path, 'r:') metadata = self._fmt.parse(self._tar.extractfile('Metadata').readlines(), key_parser={None: str}) FilePartitionReader.__init__(self, path, metadata.pop('MaxJobs')) self._metadata = metadata activity.finish() except Exception: raise PartitionReaderError('No valid dataset splitting found in %s' % path) self._map_enum2parser = { None: str, DataSplitter.NEntries: int, DataSplitter.Skipped: int, DataSplitter.Invalid: parse_bool, DataSplitter.Locations: lambda x: parse_list(x, ','), DataSplitter.MetadataHeader: parse_json, DataSplitter.Metadata: lambda x: parse_json(x.strip("'")) } (self._cache_nested_fn, self._cache_nested_tar) = (None, None)
def _parseProxy(self, cached=True): # Return cached results if requested if cached and self._cache: return self._cache # Call voms-proxy-info and parse results proc = LocalProcess(self._infoExec, *self._getProxyArgs()) (retCode, stdout, stderr) = proc.finish(timeout=10) if (retCode != 0) and not self._ignoreWarning: msg = ('%s output:\n%s\n%s\n' % (self._infoExec, stdout, stderr)).replace('\n\n', '\n') msg += 'If job submission is still possible, you can set [access] ignore warnings = True\n' raise AccessTokenError(msg + '%s failed with return code %d' % (self._infoExec, retCode)) self._cache = DictFormat(':').parse(stdout) return self._cache
class TarPartitionReader(FilePartitionReader): def __init__(self, path): activity = Activity('Reading dataset partition file') self._fmt = DictFormat() try: self._tar = tarfile.open(path, 'r:') metadata = self._fmt.parse(self._tar.extractfile('Metadata').readlines(), key_parser={None: str}) FilePartitionReader.__init__(self, path, metadata.pop('MaxJobs')) self._metadata = metadata activity.finish() except Exception: raise PartitionReaderError('No valid dataset splitting found in %s' % path) self._map_enum2parser = { None: str, DataSplitter.NEntries: int, DataSplitter.Skipped: int, DataSplitter.Invalid: parse_bool, DataSplitter.Locations: lambda x: parse_list(x, ','), DataSplitter.MetadataHeader: parse_json, DataSplitter.Metadata: lambda x: parse_json(x.strip("'")) } (self._cache_nested_fn, self._cache_nested_tar) = (None, None) def _combine_partition_parts(self, partition, url_list): if DataSplitter.CommonPrefix in partition: common_prefix = partition.pop(DataSplitter.CommonPrefix) url_list = imap(lambda x: '%s/%s' % (common_prefix, x), url_list) partition[DataSplitter.FileList] = lmap(str.strip, url_list) return partition def _get_nested_tar(self, nested_fn): if self._cache_nested_fn != nested_fn: # caching gives 3-4x speedup for sequential access self._cache_nested_tar = self._open_nested_tar(nested_fn) self._cache_nested_fn = nested_fn return self._cache_nested_tar def _open_nested_tar(self, nested_fn): nested_tar_fp = self._tar.extractfile(nested_fn) nested_tar_fp = BytesBuffer(gzip.GzipFile(fileobj=nested_tar_fp).read()) return tarfile.open(mode='r', fileobj=nested_tar_fp)
def _parse_proxy(self, cached=True): # Return cached results if requested if cached and self._cache: return self._cache # Call voms-proxy-info and parse results proc = LocalProcess(self._proxy_info_exec, *self._get_proxy_info_arguments()) (exit_code, stdout, stderr) = proc.finish(timeout=10) if (exit_code != 0) and not self._ignore_warning: msg = ('%s output:\n%s\n%s\n' % (self._proxy_info_exec, stdout, stderr)).replace( '\n\n', '\n') msg += 'If job submission is still possible, you can set [access] ignore warnings = True\n' msg += '%s failed with return code %d' % (self._proxy_info_exec, exit_code) raise AccessTokenError(msg) self._cache = DictFormat(':').parse(stdout) if not self._cache: msg = 'Unable to parse access token information:\n\t%s\n\t%s\n' raise AccessTokenError(msg % (stdout.strip(), stderr.strip())) return self._cache
def _parse(self, proc): for section in accumulate(proc.stdout.iter(self._timeout), '', lambda x, buf: x == '\n'): try: lines = section.replace('\n\t', '').split('\n') job_info = DictFormat(' = ').parse(lines[1:]) job_info[CheckInfo.WMSID] = lines[0].split(':')[1].split( '.')[0].strip() job_info[CheckInfo.RAW_STATUS] = job_info.pop('job_state') job_info[CheckInfo.QUEUE] = job_info.pop('queue', None) if 'exec_host' in job_info: exec_host = job_info.pop('exec_host').split('/')[0] job_info[CheckInfo.WN] = exec_host + '.' + job_info.get( 'server', '') except Exception: raise BackendError('Error reading job info:\n%s' % section) yield job_info
class TextFileJobDB(JobDB): alias_list = ['textdb'] def __init__(self, config, job_limit=-1, job_selector=None): JobDB.__init__(self, config, job_limit, job_selector) self._path_db = config.get_work_path('jobs') self._fmt = DictFormat(escape_strings=True) try: self._job_map = self._read_jobs(self._job_limit) except Exception: raise JobError('Unable to read stored job information!') if self._job_limit < 0 and len(self._job_map) > 0: self._job_limit = max(self._job_map) + 1 def commit(self, jobnum, job_obj): with_file(SafeFile(os.path.join(self._path_db, 'job_%d.txt' % jobnum), 'w'), lambda fp: fp.writelines(self._fmt.format(self._serialize_job_obj(job_obj)))) self._job_map[jobnum] = job_obj def get_job(self, jobnum): return self._job_map.get(jobnum) def get_job_persistent(self, jobnum): return self._job_map.get(jobnum, Job()) def get_job_transient(self, jobnum): return self._job_map.get(jobnum, self._default_job_obj) def _create_job_obj(self, name, data): try: job = Job() job.state = Job.str2enum(data.pop('status'), Job.UNKNOWN) if 'id' in data: gc_id = data.pop('id') if not gc_id.startswith('WMSID'): # Legacy support data['legacy_gc_id'] = gc_id if gc_id.startswith('https'): gc_id = 'WMSID.GLITEWMS.%s' % gc_id else: wms_id, wms_name = tuple(gc_id.split('.', 1)) gc_id = 'WMSID.%s.%s' % (wms_name, wms_id) job.gc_id = gc_id for key in ['attempt', 'submitted', 'changed']: if key in data: setattr(job, key, data[key]) if 'runtime' not in data: if 'submitted' in data and (job.submitted > 0): data['runtime'] = time.time() - float(job.submitted) else: data['runtime'] = 0 for key in irange(1, job.attempt + 1): if ('history_' + str(key)).strip() in data: job.history[key] = data['history_' + str(key)] job.set_dict(data) except Exception: raise JobError('Unable to parse data in %s:\n%r' % (name, data)) return job def _read_jobs(self, job_limit): ensure_dir_exists(self._path_db, 'job database directory', JobError) candidates = [] for job_fn in fnmatch.filter(os.listdir(self._path_db), 'job_*.txt'): try: # 2xsplit is faster than regex jobnum = int(job_fn.split(".")[0].split("_")[1]) except Exception: clear_current_exception() continue candidates.append((jobnum, job_fn)) (job_map, max_job_len) = ({}, len(candidates)) activity = Activity('Reading job infos') idx = 0 for (jobnum, job_fn) in sorted(candidates): idx += 1 if jobnum >= job_limit >= 0: self._log.info('Stopped reading job infos at job #%d out of %d available job files, ' + 'since the limit of %d jobs is reached', jobnum, len(candidates), job_limit) break try: job_fn_full = os.path.join(self._path_db, job_fn) data = self._fmt.parse(SafeFile(job_fn_full).iter_close()) job_obj = self._create_job_obj(job_fn_full, data) except Exception: raise JobError('Unable to process job file %r' % job_fn_full) job_map[jobnum] = job_obj activity.update('Reading job infos %d [%d%%]' % (idx, (100.0 * idx) / max_job_len)) activity.finish() return job_map def _serialize_job_obj(self, job_obj): data = job_obj.get_dict_full() for key, value in job_obj.history.items(): data['history_' + str(key)] = value if job_obj.gc_id is not None: data['id'] = job_obj.get('legacy_gc_id') or job_obj.gc_id # store legacy gc_id return data
def __init__(self): OutputProcessor.__init__(self) self._df = DictFormat()
class JDLWriter(object): def __init__(self): self._esc_dict = {'\\': r'\\', '\"': r'\"', '\n': r'\n'} self._fmt = DictFormat(' = ') def format(self, req_list, result=None): contents = self.prepare(req_list, result) return self._fmt.format(contents, format='%s%s%s;\n') def prepare(self, req_list, result=None): result = result or dict() self._format_reqs(req_list, result) self._format_cpus(req_list, result) return result def _escape(self, value): return '"' + str.join('', imap(lambda char: self._esc_dict.get(char, char), value)) + '"' def _format_cpus(self, req_list, result): for req_type, arg in req_list: if (req_type == WMS.CPUS) and (arg > 1): result['CpuNumber'] = arg def _format_reqs(self, req_list, result): req_string_list = ['other.GlueHostNetworkAdapterOutboundIP'] for req_type, arg in req_list: if req_type == WMS.SOFTWARE: software_template_str = 'Member(%s, other.GlueHostApplicationSoftwareRunTimeEnvironment)' req_string_list.append(software_template_str % self._escape(arg)) elif req_type == WMS.WALLTIME: if arg > 0: req_string_list.append('(other.GlueCEPolicyMaxWallClockTime >= %d)' % int((arg + 59) / 60)) elif req_type == WMS.CPUTIME: if arg > 0: req_string_list.append('(other.GlueCEPolicyMaxCPUTime >= %d)' % int((arg + 59) / 60)) elif req_type == WMS.MEMORY: if arg > 0: req_string_list.append('(other.GlueHostMainMemoryRAMSize >= %d)' % arg) elif req_type == WMS.STORAGE: req_string_list.append(self._format_reqs_storage(arg)) elif req_type == WMS.SITES: req_string_list.append(self._format_reqs_sites(arg)) elif req_type in (WMS.CPUS, WMS.DISKSPACE): pass # Handled outside of "requirement" directive or GlueCE attribute not available else: raise APIError('Unknown requirement type %r or argument %r' % (WMS.enum2str(req_type), arg)) result['Requirements'] = str.join(' && ', iidfilter(req_string_list)) def _format_reqs_sites(self, sites): def _fmt_sites(site): return 'RegExp(%s, other.GlueCEUniqueID)' % self._escape(site) (blacklist, whitelist) = split_blackwhite_list(sites) sitereqs = lmap(lambda x: '!' + _fmt_sites(x), blacklist) if whitelist: sitereqs.append('(%s)' % str.join(' || ', imap(_fmt_sites, whitelist))) if sitereqs: return '( %s )' % str.join(' && ', sitereqs) def _format_reqs_storage(self, locations): if locations: location_template_str = 'Member(%s, other.GlueCESEBindGroupSEUniqueID)' location_iter = imap(lambda x: location_template_str % self._escape(x), locations) return '( %s )' % str.join(' || ', location_iter)
def __init__(self): self._esc_dict = {'\\': r'\\', '\"': r'\"', '\n': r'\n'} self._fmt = DictFormat(' = ')
def __init__(self): PartitionWriter.__init__(self) self._fmt = DictFormat() # use a single instance to save time
class TextFileJobDB(JobDB): alias_list = ['textdb'] def __init__(self, config, job_limit=-1, job_selector=None): JobDB.__init__(self, config, job_limit, job_selector) self._path_db = config.get_work_path('jobs') self._fmt = DictFormat(escape_strings=True) try: self._job_map = self._read_jobs(self._job_limit) except Exception: raise JobError('Unable to read stored job information!') if self._job_limit < 0 and len(self._job_map) > 0: self._job_limit = max(self._job_map) + 1 def commit(self, jobnum, job_obj): with_file( SafeFile(os.path.join(self._path_db, 'job_%d.txt' % jobnum), 'w'), lambda fp: fp.writelines( self._fmt.format(self._serialize_job_obj(job_obj)))) self._job_map[jobnum] = job_obj def get_job(self, jobnum): return self._job_map.get(jobnum) def get_job_persistent(self, jobnum): return self._job_map.get(jobnum, Job()) def get_job_transient(self, jobnum): return self._job_map.get(jobnum, self._default_job_obj) def _create_job_obj(self, name, data): try: job = Job() job.state = Job.str2enum(data.pop('status'), Job.UNKNOWN) if 'id' in data: gc_id = data.pop('id') if not gc_id.startswith('WMSID'): # Legacy support data['legacy_gc_id'] = gc_id if gc_id.startswith('https'): gc_id = 'WMSID.GLITEWMS.%s' % gc_id else: wms_id, wms_name = tuple(gc_id.split('.', 1)) gc_id = 'WMSID.%s.%s' % (wms_name, wms_id) job.gc_id = gc_id for key in ['attempt', 'submitted', 'changed']: if key in data: setattr(job, key, data[key]) if 'runtime' not in data: if 'submitted' in data and (job.submitted > 0): data['runtime'] = time.time() - float(job.submitted) else: data['runtime'] = 0 for key in irange(1, job.attempt + 1): if ('history_' + str(key)).strip() in data: job.history[key] = data['history_' + str(key)] job.set_dict(data) except Exception: raise JobError('Unable to parse data in %s:\n%r' % (name, data)) return job def _read_jobs(self, job_limit): ensure_dir_exists(self._path_db, 'job database directory', JobError) candidates = [] for job_fn in fnmatch.filter(os.listdir(self._path_db), 'job_*.txt'): try: # 2xsplit is faster than regex jobnum = int(job_fn.split(".")[0].split("_")[1]) except Exception: clear_current_exception() continue candidates.append((jobnum, job_fn)) (job_map, max_job_len) = ({}, len(candidates)) activity = Activity('Reading job infos') idx = 0 for (jobnum, job_fn) in sorted(candidates): idx += 1 if jobnum >= job_limit >= 0: self._log.info( 'Stopped reading job infos at job #%d out of %d available job files, ' + 'since the limit of %d jobs is reached', jobnum, len(candidates), job_limit) break try: job_fn_full = os.path.join(self._path_db, job_fn) data = self._fmt.parse(SafeFile(job_fn_full).iter_close()) job_obj = self._create_job_obj(job_fn_full, data) except Exception: raise JobError('Unable to process job file %r' % job_fn_full) job_map[jobnum] = job_obj activity.update('Reading job infos %d [%d%%]' % (idx, (100.0 * idx) / max_job_len)) activity.finish() return job_map def _serialize_job_obj(self, job_obj): data = job_obj.get_dict_full() for key, value in job_obj.history.items(): data['history_' + str(key)] = value if job_obj.gc_id is not None: data['id'] = job_obj.get( 'legacy_gc_id') or job_obj.gc_id # store legacy gc_id return data