from grid_control.output_processor import JobInfoProcessor, JobResult from grid_control.report import ImageReport from grid_control.utils.data_structures import make_enum from hpfwk import clear_current_exception from python_compat import BytesBuffer, irange, izip JobMetrics = make_enum([ # pylint:disable=invalid-name 'EVENT_COUNT', 'FILESIZE_IN_TOTAL', 'FILESIZE_OUT_TOTAL', 'TS_CMSSW_CMSRUN1_DONE', 'TS_CMSSW_CMSRUN1_START', 'TS_DEPLOYMENT_DONE', 'TS_DEPLOYMENT_START', 'TS_EXECUTION_DONE', 'TS_EXECUTION_START', 'TS_SE_IN_DONE', 'TS_SE_IN_START', 'TS_SE_OUT_DONE', 'TS_SE_OUT_START', 'TS_WRAPPER_DONE', 'TS_WRAPPER_START', ]) class PlotReport(ImageReport): alias_list = ['plot'] def __init__(self, config, name, job_db, task=None): ImageReport.__init__(self, config, name, job_db, task)
def set(self, key, value): self._dict[key.lower()] = value def set_dict(self, value): self._dict = value def update(self, state): self.state = state self.changed = time.time() self.history[self.attempt] = self.get_job_location() make_enum([ 'INIT', 'SUBMITTED', 'DISABLED', 'READY', 'WAITING', 'QUEUED', 'ABORTED', 'RUNNING', 'CANCEL', 'UNKNOWN', 'CANCELLED', 'DONE', 'FAILED', 'SUCCESS', 'IGNORED' ], Job) class JobClassHolder(object): def __init__(self, *state_list): self.state_list = tuple(state_list) def lookup_job_class_name(cls, state_list): for prop_name in dir(cls): prop = getattr(cls, prop_name) if isinstance(prop, JobClassHolder): if sorted(getattr(prop, 'state_list')) == sorted(state_list): return prop_name
from grid_control.utils.algos import dict_union from grid_control.utils.data_structures import make_enum from grid_control.utils.file_tools import SafeFile, VirtualFile from hpfwk import AbstractError, NestedException, clear_current_exception, ignore_exception from python_compat import ichain, identity, imap, izip, lchain, lmap, set, sorted class BackendError(NestedException): pass BackendJobState = make_enum([ # pylint:disable=invalid-name 'ABORTED', # job was aborted by the WMS 'CANCELLED', # job was cancelled 'DONE', # job is finished 'QUEUED', # job is at WMS and is assigned a place to run 'RUNNING', # job is running 'UNKNOWN', # job status is unknown 'WAITING', # job is at WMS but was not yet assigned some place to run ]) WallTimeMode = make_enum(['hard', 'soft', 'ignore']) # pylint:disable=invalid-name class WMS(NamedPlugin): alias_list = ['NullWMS'] config_section_list = NamedPlugin.config_section_list + ['wms', 'backend'] config_tag_name = 'wms' def __init__(self, config, name): name = (name or self.__class__.__name__).upper().replace('.', '_')
def get_job_location(self): job_location_str = (self._dict.get('site') or '') + '/' + (self._dict.get('queue') or '') return job_location_str.strip('/') or 'N/A' def set(self, key, value): self._dict[key.lower()] = value def set_dict(self, value): self._dict = value def update(self, state): self.state = state self.changed = time.time() self.history[self.attempt] = self.get_job_location() make_enum(['INIT', 'SUBMITTED', 'DISABLED', 'READY', 'WAITING', 'QUEUED', 'ABORTED', 'RUNNING', 'CANCEL', 'UNKNOWN', 'CANCELLED', 'DONE', 'FAILED', 'SUCCESS', 'IGNORED'], Job) class JobClassHolder(object): def __init__(self, *state_list): self.state_list = tuple(state_list) def lookup_job_class_name(cls, state_list): for prop_name in dir(cls): prop = getattr(cls, prop_name) if isinstance(prop, JobClassHolder): if sorted(getattr(prop, 'state_list')) == sorted(state_list): return prop_name lookup_job_class_name = classmethod(lookup_job_class_name)
from grid_control_gui.ansi import ANSI except Exception: clear_current_exception() ANSI = None # pylint:disable=invalid-name try: from grid_control_gui.report_bar import ProgressBarActivity except Exception: clear_current_exception() ProgressBarActivity = ProgressActivity # pylint:disable=invalid-name log = logging.getLogger('se_output_download') # pylint:disable=invalid-name logging.getLogger('logging.process').disabled = True get_thread_state = resolve_fun('threading:Thread.is_alive', 'threading:Thread.isAlive') # pylint:disable=invalid-name JobDownloadStatus = make_enum(['JOB_OK', 'JOB_ALREADY', 'JOB_NO_OUTPUT', # pylint:disable=invalid-name 'JOB_PROCESSING', 'JOB_FAILED', 'JOB_RETRY', 'JOB_INCOMPLETE']) FileDownloadStatus = make_enum(['FILE_OK', 'FILE_EXISTS', 'FILE_TIMEOUT', # pylint:disable=invalid-name 'FILE_SE_BLACKLIST', 'FILE_HASH_FAILED', 'FILE_TRANSFER_FAILED', 'FILE_MKDIR_FAILED']) def accepted_se(opts, fi): return any(imap(fi[FileInfo.Path].__contains__, opts.select_se)) or not opts.select_se def check_token(token): if time.time() - check_token.last_check > 10: check_token.last_check = time.time() if not token.can_submit(20 * 60, True): log.critical('\nPlease renew access token') sys.exit(os.EX_UNAVAILABLE) check_token.last_check = 0 # <global-state>
# | Unless required by applicable law or agreed to in writing, software # | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. import copy from grid_control.datasets.provider_base import DataProvider from grid_control.datasets.splitter_base import DataSplitter, PartitionError, PartitionResyncHandler from grid_control.utils import Result, TwoSidedIterator from grid_control.utils.activity import Activity from grid_control.utils.data_structures import make_enum from python_compat import itemgetter, lmap, next, sort_inplace # enum order matters here! (prio: "disable" overrides "complete", etc.) ResyncMode = make_enum(['disable', 'complete', 'changed', 'ignore']) # pylint:disable=invalid-name ResyncMode.no_changed = [ ResyncMode.disable, ResyncMode.complete, ResyncMode.ignore ] ResyncOrder = make_enum(['append', 'preserve', 'fillgap', 'reorder']) # pylint:disable=invalid-name class BlockResyncState(object): def __init__(self, block_list_old, block_list_new): activity = Activity('Performing resynchronization of dataset') block_resync_tuple = DataProvider.resync_blocks( block_list_old, block_list_new) (self.block_list_added, self._block_list_missing, self._block_list_matching) = block_resync_tuple for block_missing in self._block_list_missing: # Files in matching blocks are already sorted sort_inplace(block_missing[DataProvider.FileList],
from grid_control.config import TriggerResync from grid_control.datasets import DataProvider, DataSplitter, DatasetError from grid_control.datasets.splitter_basic import HybridSplitter from grid_control.utils import split_opt from grid_control.utils.activity import Activity, ProgressActivity from grid_control.utils.data_structures import make_enum from grid_control.utils.thread_tools import start_thread from grid_control.utils.webservice import JSONRestClient from grid_control_cms.lumi_tools import parse_lumi_filter, str_lumi from grid_control_cms.sitedb import SiteDB from hpfwk import AbstractError from python_compat import itemgetter, lfilter, sorted CMSLocationFormat = make_enum(['hostname', 'siteDB', 'both']) # pylint:disable=invalid-name class CMSBaseProvider(DataProvider): # required format: <dataset path>[@<instance>][#<block>] def __init__(self, config, datasource_name, dataset_expr, dataset_nick=None, dataset_proc=None): dataset_config = config.change_view(default_on_change=TriggerResync(['datasets', 'parameters'])) self._lumi_filter = dataset_config.get_lookup(['lumi filter', '%s lumi filter' % datasource_name], default={}, parser=parse_lumi_filter, strfun=str_lumi) if not self._lumi_filter.empty(): config.set('%s processor' % datasource_name, 'LumiDataProcessor', '+=') DataProvider.__init__(self, config, datasource_name, dataset_expr, dataset_nick, dataset_proc) # LumiDataProcessor instantiated in DataProcessor.__ini__ will set lumi metadata as well self._lumi_query = dataset_config.get_bool( ['lumi metadata', '%s lumi metadata' % datasource_name], default=not self._lumi_filter.empty()) config.set('phedex sites matcher mode', 'ShellStyleMatcher', '?=')
# | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. import copy from grid_control.datasets.provider_base import DataProvider from grid_control.datasets.splitter_base import DataSplitter, PartitionError, PartitionResyncHandler from grid_control.utils import Result, TwoSidedIterator from grid_control.utils.activity import Activity from grid_control.utils.data_structures import make_enum from python_compat import itemgetter, lmap, next, sort_inplace # enum order matters here! (prio: "disable" overrides "complete", etc.) ResyncMode = make_enum(['disable', 'complete', 'changed', 'ignore']) # pylint:disable=invalid-name ResyncMode.no_changed = [ResyncMode.disable, ResyncMode.complete, ResyncMode.ignore] ResyncOrder = make_enum(['append', 'preserve', 'fillgap', 'reorder']) # pylint:disable=invalid-name class BlockResyncState(object): def __init__(self, block_list_old, block_list_new): activity = Activity('Performing resynchronization of dataset') block_resync_tuple = DataProvider.resync_blocks(block_list_old, block_list_new) (self.block_list_added, self._block_list_missing, self._block_list_matching) = block_resync_tuple for block_missing in self._block_list_missing: # Files in matching blocks are already sorted sort_inplace(block_missing[DataProvider.FileList], key=itemgetter(DataProvider.URL)) activity.finish() def get_block_change_info(self, partition): # Get block information (block_old, block_new, fi_list_missing, fi_list_matched)
# | limitations under the License. import os, sys, signal from grid_control.config.cinterface_base import ConfigInterface from grid_control.config.config_entry import ConfigError, join_config_locations from grid_control.config.cview_base import SimpleConfigView from grid_control.config.matcher_base import DictLookup, ListFilter, ListOrder, Matcher from grid_control.utils import resolve_path, resolve_paths from grid_control.utils.data_structures import make_enum from grid_control.utils.parsing import parse_bool, parse_dict_cfg, parse_list, parse_time, str_dict_cfg, str_time_short # pylint:disable=line-too-long from grid_control.utils.thread_tools import GCEvent from grid_control.utils.user_interface import UserInputInterface from hpfwk import APIError, ExceptionCollector, Plugin, clear_current_exception from python_compat import any, identity, ifilter, imap, lmap, sorted, unspecified CommandType = make_enum(['executable', 'command']) # pylint: disable=invalid-name class TypedConfigInterface(ConfigInterface): # Config interface class accessing typed data using an string interface provided by config_view def get_bool(self, option, default=unspecified, **kwargs): # Getting boolean config options - feature: true and false are not the only valid expressions def _str2obj(value): result = parse_bool(value) if result is None: raise ConfigError( 'Valid boolean expressions are: "true", "false"') return result return self._get_internal('bool', bool.__str__, _str2obj, None, option, default, **kwargs)
import os, time, logging from grid_control.gc_plugin import ConfigurablePlugin from grid_control.parameters.psource_base import ParameterError, ParameterInfo, ParameterMetadata, ParameterSource # pylint:disable=line-too-long from grid_control.utils import ensure_dir_exists from grid_control.utils.activity import Activity from grid_control.utils.algos import filter_dict, get_list_difference, reverse_dict from grid_control.utils.data_structures import make_enum from grid_control.utils.file_tools import GZipTextFile from grid_control.utils.parsing import str_time_short from grid_control.utils.user_interface import UserInputInterface from hpfwk import APIError from python_compat import ifilter, iidfilter, imap, irange, itemgetter, lfilter, lmap, md5_hex, set, sort_inplace, sorted # pylint:disable=line-too-long # TrackingInfo enum values == fast resync tuple indices TrackingInfo = make_enum(['ACTIVE', 'HASH', 'pnum'], use_hash=False) # pylint:disable=invalid-name class ParameterAdapter(ConfigurablePlugin): def __init__(self, config, source): ConfigurablePlugin.__init__(self, config) self._log = logging.getLogger('parameters.adapter') self._psrc = source def __repr__(self): return '%s(%r)' % (self.__class__.__name__, self._psrc) def can_finish(self): return self._psrc.can_finish() def can_submit(self, jobnum):
def _iter_blocks_raw(self): # List of (partial or complete) block dicts with format # { NEntries: 123, Dataset: '/path/to/data', Block: 'abcd-1234', Locations: ['site1','site2'], # Filelist: [{URL: '/path/to/file1', NEntries: 100}, {URL: '/path/to/file2', NEntries: 23}]} raise AbstractError def _raise_on_abort(self): if abort(): raise DatasetError( 'Received abort request during dataset retrieval') make_enum( # To uncover errors, the enums of DataProvider / DataSplitter do *NOT* match type wise [ 'NEntries', 'BlockName', 'Dataset', 'Locations', 'URL', 'FileList', 'Nickname', 'Metadata', 'Provider', 'Query' ], DataProvider) def _split_metadata_idx_list(block): def _get_metadata_hash(fi, idx): if idx < len(fi[DataProvider.Metadata]): return md5_hex(repr(fi[DataProvider.Metadata][idx])) fi_list = block[DataProvider.FileList] common_metadata_idx_list = lrange(len(block[DataProvider.Metadata])) if fi_list: common_metadata_hash_list = lmap( lambda idx: _get_metadata_hash(fi_list[0], idx), common_metadata_idx_list)
# | Unless required by applicable law or agreed to in writing, software # | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. import re, fnmatch, logging from grid_control.config.config_entry import join_config_locations from grid_control.gc_plugin import ConfigurablePlugin from grid_control.utils.data_structures import make_enum from grid_control.utils.parsing import str_dict_linear from hpfwk import AbstractError, Plugin from python_compat import lfilter, sorted, unspecified ListOrder = make_enum(['source', 'matcher']) # pylint: disable=invalid-name class MatcherHolder(object): def __init__(self, selector, case): self._case = case self._selector = selector self._assign_selector(selector) def __repr__(self): if self._case: return self.__class__.__name__ + '(%s)' % repr(self._selector) return self.__class__.__name__ + '_ci(%s)' % repr(self._selector) def match(self, value): raise AbstractError
# | http://www.apache.org/licenses/LICENSE-2.0 # | # | Unless required by applicable law or agreed to in writing, software # | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. import os, gzip, logging from grid_control.utils import DictFormat from grid_control.utils.data_structures import make_enum from grid_control.utils.file_tools import SafeFile from hpfwk import AbstractError, NestedException, Plugin, clear_current_exception, get_current_exception from python_compat import bytes2str, ifilter, izip JobResult = make_enum(['JOBNUM', 'MESSAGE', 'EXITCODE', 'RAW']) # pylint:disable=invalid-name FileInfo = make_enum(['Hash', 'NameLocal', 'NameDest', 'Path', 'Size']) # pylint:disable=invalid-name class JobResultError(NestedException): pass class OutputProcessor(Plugin): def process(self, dn): raise AbstractError class TaskOutputProcessor(Plugin): def process(self, dn, task): raise AbstractError
# | # | http://www.apache.org/licenses/LICENSE-2.0 # | # | Unless required by applicable law or agreed to in writing, software # | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. from grid_control.datasets.dproc_base import DataProcessor from grid_control.datasets.provider_base import DataProvider, DatasetError from grid_control.utils.data_structures import make_enum from python_compat import imap, itemgetter, md5_hex, set # Enum to specify how to react to multiple occurences of something DatasetUniqueMode = make_enum(['warn', 'abort', 'skip', 'ignore', 'record']) # pylint: disable=invalid-name DatasetCheckMode = make_enum(['warn', 'abort', 'ignore']) # pylint: disable=invalid-name class BaseConsistencyProcessor(DataProcessor): def _handle_error(self, msg, mode): if mode == DatasetCheckMode.warn: self._log.warning(msg) elif mode == DatasetCheckMode.abort: raise DatasetError(msg) class EntriesConsistencyDataProcessor(BaseConsistencyProcessor): alias_list = ['consistency'] def __init__(self, config, datasource_name):
# | You may obtain a copy of the License at # | # | http://www.apache.org/licenses/LICENSE-2.0 # | # | Unless required by applicable law or agreed to in writing, software # | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. import time, logging from grid_control.utils.data_structures import make_enum from hpfwk import AbstractError, NestedException, Plugin from python_compat import set ParameterInfo = make_enum(['ACTIVE', 'HASH', 'REQS', 'FILES']) # pylint:disable=invalid-name class ParameterError(NestedException): pass class ParameterMetadata(object): def __init__(self, value, untracked=False): (self.value, self.untracked) = (value, untracked) def __repr__(self): return self.get_value() def get_value(self): if self.untracked:
import os, sys, signal from grid_control.config.cinterface_base import ConfigInterface from grid_control.config.config_entry import ConfigError, join_config_locations from grid_control.config.cview_base import SimpleConfigView from grid_control.config.matcher_base import DictLookup, ListFilter, ListOrder, Matcher from grid_control.utils import resolve_path, resolve_paths from grid_control.utils.data_structures import make_enum from grid_control.utils.parsing import parse_bool, parse_dict_cfg, parse_list, parse_time, str_dict_cfg, str_time_short # pylint:disable=line-too-long from grid_control.utils.thread_tools import GCEvent from grid_control.utils.user_interface import UserInputInterface from hpfwk import APIError, ExceptionCollector, Plugin, clear_current_exception from python_compat import any, identity, ifilter, imap, lmap, sorted, unspecified CommandType = make_enum(['executable', 'command']) # pylint: disable=invalid-name class TypedConfigInterface(ConfigInterface): # Config interface class accessing typed data using an string interface provided by config_view def get_bool(self, option, default=unspecified, **kwargs): # Getting boolean config options - feature: true and false are not the only valid expressions def _str2obj(value): result = parse_bool(value) if result is None: raise ConfigError('Valid boolean expressions are: "true", "false"') return result return self._get_internal('bool', bool.__str__, _str2obj, None, option, default, **kwargs) def get_composited_plugin(self, option, default=unspecified, default_compositor=unspecified, option_compositor=None, cls=Plugin, require_plugin=True,
# | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. import os, sys, time, logging, threading from grid_control.gc_exceptions import GCError, GCLogHandler from grid_control.stream_base import ActivityMonitor from grid_control.utils.data_structures import UniqueList, make_enum from grid_control.utils.file_tools import SafeFile, VirtualFile from grid_control.utils.thread_tools import GCLock, with_lock from hpfwk import AbstractError, format_exception, ignore_exception, rethrow from python_compat import any, imap, irange, lmap, set, sorted, tarfile LogLevelEnum = make_enum( lmap(lambda level: logging.getLevelName(level).upper(), irange(51)), # pylint:disable=invalid-name use_hash=False, register=False) def dump_log_setup(level): # Display logging setup output = logging.getLogger('logging') def _display_logger(indent, logger, name): propagate_symbol = '+' if hasattr(logger, 'propagate') and not logger.propagate: propagate_symbol = 'o' desc = name if hasattr(logger, 'level'): desc += ' (level = %s)' % logging.getLevelName(logger.level) output.log(level, '%s%s %s', '| ' * indent, propagate_symbol, desc)
from grid_control.utils.algos import dict_union from grid_control.utils.data_structures import make_enum from grid_control.utils.file_tools import SafeFile, VirtualFile from hpfwk import AbstractError, NestedException, clear_current_exception, ignore_exception from python_compat import ichain, identity, imap, izip, lchain, lmap, set, sorted class BackendError(NestedException): pass BackendJobState = make_enum([ # pylint:disable=invalid-name 'ABORTED', # job was aborted by the WMS 'CANCELLED', # job was cancelled 'DONE', # job is finished 'QUEUED', # job is at WMS and is assigned a place to run 'RUNNING', # job is running 'UNKNOWN', # job status is unknown 'WAITING', # job is at WMS but was not yet assigned some place to run ]) WallTimeMode = make_enum(['hard', 'soft', 'ignore']) # pylint:disable=invalid-name class WMS(NamedPlugin): alias_list = ['NullWMS'] config_section_list = NamedPlugin.config_section_list + ['wms', 'backend'] config_tag_name = 'wms' def __init__(self, config, name): name = (name or self.__class__.__name__).upper().replace('.', '_')
# | # | Unless required by applicable law or agreed to in writing, software # | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. import os, gzip, logging from grid_control.utils import DictFormat from grid_control.utils.data_structures import make_enum from grid_control.utils.file_tools import SafeFile from hpfwk import AbstractError, NestedException, Plugin, clear_current_exception, get_current_exception from python_compat import bytes2str, ifilter, izip JobResult = make_enum(['JOBNUM', 'MESSAGE', 'EXITCODE', 'RAW']) # pylint:disable=invalid-name FileInfo = make_enum(['Hash', 'NameLocal', 'NameDest', 'Path', 'Size']) # pylint:disable=invalid-name class JobResultError(NestedException): pass class OutputProcessor(Plugin): def process(self, dn): raise AbstractError class TaskOutputProcessor(Plugin): def process(self, dn, task): raise AbstractError
from grid_control_gui.report_bar import ProgressBarActivity except Exception: clear_current_exception() ProgressBarActivity = ProgressActivity # pylint:disable=invalid-name log = logging.getLogger('se_output_download') # pylint:disable=invalid-name logging.getLogger('logging.process').disabled = True get_thread_state = resolve_fun('threading:Thread.is_alive', 'threading:Thread.isAlive') # pylint:disable=invalid-name JobDownloadStatus = make_enum([ 'JOB_OK', 'JOB_ALREADY', 'JOB_NO_OUTPUT', # pylint:disable=invalid-name 'JOB_PROCESSING', 'JOB_FAILED', 'JOB_RETRY', 'JOB_INCOMPLETE' ]) FileDownloadStatus = make_enum([ 'FILE_OK', 'FILE_EXISTS', 'FILE_TIMEOUT', # pylint:disable=invalid-name 'FILE_SE_BLACKLIST', 'FILE_HASH_FAILED', 'FILE_TRANSFER_FAILED', 'FILE_MKDIR_FAILED' ])
# | # | Unless required by applicable law or agreed to in writing, software # | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. import re, fnmatch, logging from grid_control.config.config_entry import join_config_locations from grid_control.gc_plugin import ConfigurablePlugin from grid_control.utils.data_structures import make_enum from grid_control.utils.parsing import str_dict_linear from hpfwk import AbstractError, Plugin from python_compat import lfilter, sorted, unspecified ListOrder = make_enum(['source', 'matcher']) # pylint: disable=invalid-name class MatcherHolder(object): def __init__(self, selector, case): self._case = case self._selector = selector self._assign_selector(selector) def __repr__(self): if self._case: return self.__class__.__name__ + '(%s)' % repr(self._selector) return self.__class__.__name__ + '_ci(%s)' % repr(self._selector) def match(self, value): raise AbstractError
# | http://www.apache.org/licenses/LICENSE-2.0 # | # | Unless required by applicable law or agreed to in writing, software # | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. from grid_control.datasets.dproc_base import DataProcessor from grid_control.datasets.provider_base import DataProvider, DatasetError from grid_control.utils.data_structures import make_enum from python_compat import imap, itemgetter, md5_hex, set # Enum to specify how to react to multiple occurences of something DatasetUniqueMode = make_enum(['warn', 'abort', 'skip', 'ignore', 'record']) # pylint: disable=invalid-name DatasetCheckMode = make_enum(['warn', 'abort', 'ignore']) # pylint: disable=invalid-name class BaseConsistencyProcessor(DataProcessor): def _handle_error(self, msg, mode): if mode == DatasetCheckMode.warn: self._log.warning(msg) elif mode == DatasetCheckMode.abort: raise DatasetError(msg) class EntriesConsistencyDataProcessor(BaseConsistencyProcessor): alias_list = ['consistency'] def __init__(self, config, datasource_name):
matplotlib = None from grid_control.output_processor import JobInfoProcessor, JobResult from grid_control.report import ImageReport from grid_control.utils.data_structures import make_enum from hpfwk import clear_current_exception from python_compat import BytesBuffer, irange, izip JobMetrics = make_enum([ # pylint:disable=invalid-name 'EVENT_COUNT', 'FILESIZE_IN_TOTAL', 'FILESIZE_OUT_TOTAL', 'TS_CMSSW_CMSRUN1_DONE', 'TS_CMSSW_CMSRUN1_START', 'TS_DEPLOYMENT_DONE', 'TS_DEPLOYMENT_START', 'TS_EXECUTION_DONE', 'TS_EXECUTION_START', 'TS_SE_IN_DONE', 'TS_SE_IN_START', 'TS_SE_OUT_DONE', 'TS_SE_OUT_START', 'TS_WRAPPER_DONE', 'TS_WRAPPER_START', ]) class PlotReport(ImageReport): alias_list = ['plot'] def __init__(self, config, name, job_db, task=None): ImageReport.__init__(self, config, name, job_db, task)
# | # | Unless required by applicable law or agreed to in writing, software # | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. import logging from grid_control.backends.backend_tools import BackendError, BackendExecutor from grid_control.job_db import Job from grid_control.utils import abort from grid_control.utils.data_structures import make_enum from hpfwk import AbstractError from python_compat import set CheckInfo = make_enum(['WMSID', 'RAW_STATUS', 'QUEUE', 'WN', 'SITE']) # pylint:disable=invalid-name CheckStatus = make_enum(['OK', 'ERROR']) # pylint:disable=invalid-name # TODO: Error Handler Plugins - logging, exception, errorcode - with abort / continue def expand_status_map(map_job_status2status_str_list): result = {} for job_status, status_str_list in map_job_status2status_str_list.items(): result.update(dict.fromkeys(status_str_list, job_status)) return result class CheckJobs(BackendExecutor): def execute(self, wms_id_list): # yields list of (wms_id, job_status, job_info)
# | limitations under the License. from grid_control.config import TriggerResync from grid_control.datasets import DataProvider, DataSplitter, DatasetError from grid_control.datasets.splitter_basic import HybridSplitter from grid_control.utils import split_opt from grid_control.utils.activity import Activity, ProgressActivity from grid_control.utils.data_structures import make_enum from grid_control.utils.thread_tools import start_thread from grid_control.utils.webservice import JSONRestClient from grid_control_cms.lumi_tools import parse_lumi_filter, str_lumi from grid_control_cms.sitedb import SiteDB from hpfwk import AbstractError from python_compat import itemgetter, lfilter, sorted CMSLocationFormat = make_enum(['hostname', 'siteDB', 'both']) # pylint:disable=invalid-name class CMSBaseProvider(DataProvider): # required format: <dataset path>[@<instance>][#<block>] def __init__(self, config, datasource_name, dataset_expr, dataset_nick=None, dataset_proc=None): dataset_config = config.change_view( default_on_change=TriggerResync(['datasets', 'parameters'])) self._lumi_filter = dataset_config.get_lookup( ['lumi filter', '%s lumi filter' % datasource_name], default={},
# | http://www.apache.org/licenses/LICENSE-2.0 # | # | Unless required by applicable law or agreed to in writing, software # | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. from grid_control.datasets import DataProcessor, DataProvider, DataSplitter, DatasetError, PartitionProcessor # pylint:disable=line-too-long from grid_control.parameters import ParameterMetadata from grid_control.utils.algos import safe_index from grid_control.utils.data_structures import make_enum from grid_control_cms.lumi_tools import filter_lumi_filter, format_lumi, parse_lumi_filter, select_lumi, select_run, str_lumi # pylint:disable=line-too-long from python_compat import any, ichain, imap, izip, set LumiKeep = make_enum(['RunLumi', 'Run', 'none']) # pylint:disable=invalid-name LumiMode = make_enum(['strict', 'weak']) # pylint:disable=invalid-name class LumiDataProcessor(DataProcessor): alias_list = ['lumi'] def __init__(self, config, datasource_name): DataProcessor.__init__(self, config, datasource_name) self._lumi_filter = config.get_lookup( ['lumi filter', '%s lumi filter' % datasource_name], default={}, parser=parse_lumi_filter, strfun=str_lumi) if self._lumi_filter.empty(): lumi_keep_default = LumiKeep.none
from grid_control.backends.aspect_cancel import CancelAndPurgeJobs from grid_control.backends.aspect_status import CheckJobsMissingState from grid_control.backends.backend_tools import unpack_wildcard_tar from grid_control.backends.broker_base import Broker from grid_control.backends.condor_wms.processhandler import ProcessHandler from grid_control.backends.wms import BackendError, BasicWMS, WMS, WallTimeMode from grid_control.backends.wms_condor import CondorCancelJobs, CondorCheckJobs from grid_control.backends.wms_local import LocalPurgeJobs, SandboxHelper from grid_control.utils import Result, ensure_dir_exists, get_path_share, remove_files, resolve_install_path, safe_write, split_blackwhite_list # pylint:disable=line-too-long from grid_control.utils.activity import Activity from grid_control.utils.data_structures import make_enum from python_compat import imap, irange, lfilter, lmap, lzip, md5_hex # if the ssh stuff proves too hack'y: http://www.lag.net/paramiko/ PoolType = make_enum(['LOCAL', 'SPOOL', 'SSH', 'GSISSH']) # pylint:disable=invalid-name class CondorJDLWriter(object): def __init__(self, config): self._email = config.get(['notifyemail', 'email'], '', on_change=None) self._classad_list = config.get_list(['classaddata', 'classad data'], [], on_change=None) self._jdl_list = config.get_list(['jdldata', 'jdl data'], [], on_change=None) self._pool_query_dict = config.get_dict('poolArgs query', {})[0] def get_jdl(self): jdl_str_list = [] if self._email: jdl_str_list.append('notify_user = ' + self._email) # properly inject any information retrieval keys into ClassAds # regular attributes do not need injecting
# | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. import os, sys, time, logging, threading from grid_control.gc_exceptions import GCError, GCLogHandler from grid_control.stream_base import ActivityMonitor from grid_control.utils.data_structures import UniqueList, make_enum from grid_control.utils.file_tools import SafeFile, VirtualFile from grid_control.utils.thread_tools import GCLock, with_lock from hpfwk import AbstractError, format_exception, ignore_exception, rethrow from python_compat import any, imap, irange, lmap, set, sorted, tarfile LogLevelEnum = make_enum(lmap(lambda level: logging.getLevelName(level).upper(), irange(51)), # pylint:disable=invalid-name use_hash=False, register=False) def clean_logger(logger_name=None): logger = logging.getLogger(logger_name) logger.handlers = [] return logger def dump_log_setup(level): # Display logging setup output = logging.getLogger('logging') def _display_logger(indent, logger, name): propagate_symbol = '+' if hasattr(logger, 'propagate') and not logger.propagate:
partition[ds_prop] = block[dp_prop] if DataProvider.Metadata in block: partition[DataSplitter.MetadataHeader] = block[DataProvider.Metadata] # Helper for very simple splitter if fi_list: partition[DataSplitter.FileList] = lmap(itemgetter(DataProvider.URL), fi_list) partition[DataSplitter.NEntries] = sum(imap(itemgetter(DataProvider.NEntries), fi_list)) if DataProvider.Metadata in block: partition[DataSplitter.Metadata] = lmap(itemgetter(DataProvider.Metadata), fi_list) return partition def _get_part_opt(self, *args): return join_config_locations(['', self._datasource_name], *args) make_enum(['Dataset', 'Locations', 'NEntries', 'Skipped', 'FileList', 'Nickname', 'DatasetID', # DatasetID is legacy 'CommonPrefix', 'Invalid', 'BlockName', 'MetadataHeader', 'Metadata', 'Comment'], DataSplitter, use_hash=False) class PartitionResyncHandler(ConfigurablePlugin): def resync(self, splitter, reader, block_list_old, block_list_new): raise AbstractError class PartitionReader(Plugin): def __init__(self, partition_len): self._lock = GCLock() self._partition_len = partition_len def get_partition_checked(self, partition_num): if partition_num >= self._partition_len:
# | # | Unless required by applicable law or agreed to in writing, software # | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. from grid_control.datasets import DataProcessor, DataProvider, DataSplitter, DatasetError, PartitionProcessor # pylint:disable=line-too-long from grid_control.parameters import ParameterMetadata from grid_control.utils.algos import safe_index from grid_control.utils.data_structures import make_enum from grid_control_cms.lumi_tools import filter_lumi_filter, format_lumi, parse_lumi_filter, select_lumi, select_run, str_lumi # pylint:disable=line-too-long from python_compat import any, ichain, imap, izip, set LumiKeep = make_enum(['RunLumi', 'Run', 'none']) # pylint:disable=invalid-name LumiMode = make_enum(['strict', 'weak']) # pylint:disable=invalid-name class LumiDataProcessor(DataProcessor): alias_list = ['lumi'] def __init__(self, config, datasource_name): DataProcessor.__init__(self, config, datasource_name) self._lumi_filter = config.get_lookup(['lumi filter', '%s lumi filter' % datasource_name], default={}, parser=parse_lumi_filter, strfun=str_lumi) if self._lumi_filter.empty(): lumi_keep_default = LumiKeep.none else: lumi_keep_default = LumiKeep.Run config.set_bool('%s lumi metadata' % datasource_name, True)
import os, re, time, tempfile from grid_control.backends.aspect_cancel import CancelAndPurgeJobs from grid_control.backends.aspect_status import CheckJobsMissingState from grid_control.backends.backend_tools import unpack_wildcard_tar from grid_control.backends.broker_base import Broker from grid_control.backends.condor_wms.processhandler import ProcessHandler from grid_control.backends.wms import BackendError, BasicWMS, WMS, WallTimeMode from grid_control.backends.wms_condor import CondorCancelJobs, CondorCheckJobs from grid_control.backends.wms_local import LocalPurgeJobs, SandboxHelper from grid_control.utils import Result, ensure_dir_exists, get_path_share, remove_files, resolve_install_path, safe_write, split_blackwhite_list # pylint:disable=line-too-long from grid_control.utils.activity import Activity from grid_control.utils.data_structures import make_enum from python_compat import imap, irange, lfilter, lmap, lzip, md5_hex # if the ssh stuff proves too hack'y: http://www.lag.net/paramiko/ PoolType = make_enum(['LOCAL', 'SPOOL', 'SSH', 'GSISSH']) # pylint:disable=invalid-name class CondorJDLWriter(object): def __init__(self, config): self._email = config.get(['notifyemail', 'email'], '', on_change=None) self._classad_list = config.get_list(['classaddata', 'classad data'], [], on_change=None) self._jdl_list = config.get_list(['jdldata', 'jdl data'], [], on_change=None) self._pool_query_dict = config.get_dict('poolArgs query', {})[0] def get_jdl(self): jdl_str_list = [] if self._email:
url = self._url if api: url += '/%s' % api try: return self._session.request(mode, url=url, headers=header_dict, params=params, data=data, cert=self._cert) except Exception: raise RestError('Unable to query %r (params: %r, cert: %r, session: %r)' % (url, params, self._cert, self._session.__class__.__name__)) class RestSession(Plugin): def request(self, mode, url, headers, params=None, data=None, cert=None): raise AbstractError make_enum(['GET', 'PUT', 'POST', 'DELETE'], RestSession) class JSONRestClient(RestClient): def __init__(self, cert=None, url=None, default_headers=None, process_result=None): RestClient.__init__(self, cert, url, default_headers or {'Content-Type': 'application/json', 'Accept': 'application/json'}, process_result=process_result or self._process_json_result, process_data=json.dumps) def _process_json_result(self, value): if not value: raise RestError('Received empty reply') try: return parse_json(value) except Exception: raise RestError('Received invalid JSON reply: %r' % value)