from grid_control.output_processor import JobInfoProcessor, JobResult from grid_control.report import ImageReport from grid_control.utils.data_structures import make_enum from hpfwk import clear_current_exception from python_compat import BytesBuffer, irange, izip JobMetrics = make_enum([ # pylint:disable=invalid-name 'EVENT_COUNT', 'FILESIZE_IN_TOTAL', 'FILESIZE_OUT_TOTAL', 'TS_CMSSW_CMSRUN1_DONE', 'TS_CMSSW_CMSRUN1_START', 'TS_DEPLOYMENT_DONE', 'TS_DEPLOYMENT_START', 'TS_EXECUTION_DONE', 'TS_EXECUTION_START', 'TS_SE_IN_DONE', 'TS_SE_IN_START', 'TS_SE_OUT_DONE', 'TS_SE_OUT_START', 'TS_WRAPPER_DONE', 'TS_WRAPPER_START', ]) class PlotReport(ImageReport): alias_list = ['plot'] def __init__(self, config, name, job_db, task=None): ImageReport.__init__(self, config, name, job_db, task)
def set(self, key, value):
self._dict[key.lower()] = value
def set_dict(self, value):
self._dict = value
def update(self, state):
self.state = state
self.changed = time.time()
self.history[self.attempt] = self.get_job_location()
make_enum([
'INIT', 'SUBMITTED', 'DISABLED', 'READY', 'WAITING', 'QUEUED', 'ABORTED',
'RUNNING', 'CANCEL', 'UNKNOWN', 'CANCELLED', 'DONE', 'FAILED', 'SUCCESS',
'IGNORED'
], Job)
class JobClassHolder(object):
def __init__(self, *state_list):
self.state_list = tuple(state_list)
def lookup_job_class_name(cls, state_list):
for prop_name in dir(cls):
prop = getattr(cls, prop_name)
if isinstance(prop, JobClassHolder):
if sorted(getattr(prop, 'state_list')) == sorted(state_list):
return prop_name
from grid_control.utils.algos import dict_union
from grid_control.utils.data_structures import make_enum
from grid_control.utils.file_tools import SafeFile, VirtualFile
from hpfwk import AbstractError, NestedException, clear_current_exception, ignore_exception
from python_compat import ichain, identity, imap, izip, lchain, lmap, set, sorted
class BackendError(NestedException):
pass
BackendJobState = make_enum([ # pylint:disable=invalid-name
'ABORTED', # job was aborted by the WMS
'CANCELLED', # job was cancelled
'DONE', # job is finished
'QUEUED', # job is at WMS and is assigned a place to run
'RUNNING', # job is running
'UNKNOWN', # job status is unknown
'WAITING', # job is at WMS but was not yet assigned some place to run
])
WallTimeMode = make_enum(['hard', 'soft', 'ignore']) # pylint:disable=invalid-name
class WMS(NamedPlugin):
alias_list = ['NullWMS']
config_section_list = NamedPlugin.config_section_list + ['wms', 'backend']
config_tag_name = 'wms'
def __init__(self, config, name):
name = (name or self.__class__.__name__).upper().replace('.', '_')
def get_job_location(self):
job_location_str = (self._dict.get('site') or '') + '/' + (self._dict.get('queue') or '')
return job_location_str.strip('/') or 'N/A'
def set(self, key, value):
self._dict[key.lower()] = value
def set_dict(self, value):
self._dict = value
def update(self, state):
self.state = state
self.changed = time.time()
self.history[self.attempt] = self.get_job_location()
make_enum(['INIT', 'SUBMITTED', 'DISABLED', 'READY', 'WAITING', 'QUEUED', 'ABORTED',
'RUNNING', 'CANCEL', 'UNKNOWN', 'CANCELLED', 'DONE', 'FAILED', 'SUCCESS', 'IGNORED'], Job)
class JobClassHolder(object):
def __init__(self, *state_list):
self.state_list = tuple(state_list)
def lookup_job_class_name(cls, state_list):
for prop_name in dir(cls):
prop = getattr(cls, prop_name)
if isinstance(prop, JobClassHolder):
if sorted(getattr(prop, 'state_list')) == sorted(state_list):
return prop_name
lookup_job_class_name = classmethod(lookup_job_class_name)
from grid_control_gui.ansi import ANSI
except Exception:
clear_current_exception()
ANSI = None # pylint:disable=invalid-name
try:
from grid_control_gui.report_bar import ProgressBarActivity
except Exception:
clear_current_exception()
ProgressBarActivity = ProgressActivity # pylint:disable=invalid-name
log = logging.getLogger('se_output_download') # pylint:disable=invalid-name
logging.getLogger('logging.process').disabled = True
get_thread_state = resolve_fun('threading:Thread.is_alive', 'threading:Thread.isAlive') # pylint:disable=invalid-name
JobDownloadStatus = make_enum(['JOB_OK', 'JOB_ALREADY', 'JOB_NO_OUTPUT', # pylint:disable=invalid-name
'JOB_PROCESSING', 'JOB_FAILED', 'JOB_RETRY', 'JOB_INCOMPLETE'])
FileDownloadStatus = make_enum(['FILE_OK', 'FILE_EXISTS', 'FILE_TIMEOUT', # pylint:disable=invalid-name
'FILE_SE_BLACKLIST', 'FILE_HASH_FAILED', 'FILE_TRANSFER_FAILED', 'FILE_MKDIR_FAILED'])
def accepted_se(opts, fi):
return any(imap(fi[FileInfo.Path].__contains__, opts.select_se)) or not opts.select_se
def check_token(token):
if time.time() - check_token.last_check > 10:
check_token.last_check = time.time()
if not token.can_submit(20 * 60, True):
log.critical('\nPlease renew access token')
sys.exit(os.EX_UNAVAILABLE)
check_token.last_check = 0 # <global-state>
# | Unless required by applicable law or agreed to in writing, software
# | distributed under the License is distributed on an "AS IS" BASIS,
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
import copy
from grid_control.datasets.provider_base import DataProvider
from grid_control.datasets.splitter_base import DataSplitter, PartitionError, PartitionResyncHandler
from grid_control.utils import Result, TwoSidedIterator
from grid_control.utils.activity import Activity
from grid_control.utils.data_structures import make_enum
from python_compat import itemgetter, lmap, next, sort_inplace
# enum order matters here! (prio: "disable" overrides "complete", etc.)
ResyncMode = make_enum(['disable', 'complete', 'changed', 'ignore']) # pylint:disable=invalid-name
ResyncMode.no_changed = [
ResyncMode.disable, ResyncMode.complete, ResyncMode.ignore
]
ResyncOrder = make_enum(['append', 'preserve', 'fillgap', 'reorder']) # pylint:disable=invalid-name
class BlockResyncState(object):
def __init__(self, block_list_old, block_list_new):
activity = Activity('Performing resynchronization of dataset')
block_resync_tuple = DataProvider.resync_blocks(
block_list_old, block_list_new)
(self.block_list_added, self._block_list_missing,
self._block_list_matching) = block_resync_tuple
for block_missing in self._block_list_missing: # Files in matching blocks are already sorted
sort_inplace(block_missing[DataProvider.FileList],
from grid_control.config import TriggerResync
from grid_control.datasets import DataProvider, DataSplitter, DatasetError
from grid_control.datasets.splitter_basic import HybridSplitter
from grid_control.utils import split_opt
from grid_control.utils.activity import Activity, ProgressActivity
from grid_control.utils.data_structures import make_enum
from grid_control.utils.thread_tools import start_thread
from grid_control.utils.webservice import JSONRestClient
from grid_control_cms.lumi_tools import parse_lumi_filter, str_lumi
from grid_control_cms.sitedb import SiteDB
from hpfwk import AbstractError
from python_compat import itemgetter, lfilter, sorted
CMSLocationFormat = make_enum(['hostname', 'siteDB', 'both']) # pylint:disable=invalid-name
class CMSBaseProvider(DataProvider):
# required format: <dataset path>[@<instance>][#<block>]
def __init__(self, config, datasource_name, dataset_expr, dataset_nick=None, dataset_proc=None):
dataset_config = config.change_view(default_on_change=TriggerResync(['datasets', 'parameters']))
self._lumi_filter = dataset_config.get_lookup(['lumi filter', '%s lumi filter' % datasource_name],
default={}, parser=parse_lumi_filter, strfun=str_lumi)
if not self._lumi_filter.empty():
config.set('%s processor' % datasource_name, 'LumiDataProcessor', '+=')
DataProvider.__init__(self, config, datasource_name, dataset_expr, dataset_nick, dataset_proc)
# LumiDataProcessor instantiated in DataProcessor.__ini__ will set lumi metadata as well
self._lumi_query = dataset_config.get_bool(
['lumi metadata', '%s lumi metadata' % datasource_name], default=not self._lumi_filter.empty())
config.set('phedex sites matcher mode', 'ShellStyleMatcher', '?=')
# | distributed under the License is distributed on an "AS IS" BASIS,
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
import copy
from grid_control.datasets.provider_base import DataProvider
from grid_control.datasets.splitter_base import DataSplitter, PartitionError, PartitionResyncHandler
from grid_control.utils import Result, TwoSidedIterator
from grid_control.utils.activity import Activity
from grid_control.utils.data_structures import make_enum
from python_compat import itemgetter, lmap, next, sort_inplace
# enum order matters here! (prio: "disable" overrides "complete", etc.)
ResyncMode = make_enum(['disable', 'complete', 'changed', 'ignore']) # pylint:disable=invalid-name
ResyncMode.no_changed = [ResyncMode.disable, ResyncMode.complete, ResyncMode.ignore]
ResyncOrder = make_enum(['append', 'preserve', 'fillgap', 'reorder']) # pylint:disable=invalid-name
class BlockResyncState(object):
def __init__(self, block_list_old, block_list_new):
activity = Activity('Performing resynchronization of dataset')
block_resync_tuple = DataProvider.resync_blocks(block_list_old, block_list_new)
(self.block_list_added, self._block_list_missing, self._block_list_matching) = block_resync_tuple
for block_missing in self._block_list_missing: # Files in matching blocks are already sorted
sort_inplace(block_missing[DataProvider.FileList], key=itemgetter(DataProvider.URL))
activity.finish()
def get_block_change_info(self, partition):
# Get block information (block_old, block_new, fi_list_missing, fi_list_matched)
# | limitations under the License.
import os, sys, signal
from grid_control.config.cinterface_base import ConfigInterface
from grid_control.config.config_entry import ConfigError, join_config_locations
from grid_control.config.cview_base import SimpleConfigView
from grid_control.config.matcher_base import DictLookup, ListFilter, ListOrder, Matcher
from grid_control.utils import resolve_path, resolve_paths
from grid_control.utils.data_structures import make_enum
from grid_control.utils.parsing import parse_bool, parse_dict_cfg, parse_list, parse_time, str_dict_cfg, str_time_short # pylint:disable=line-too-long
from grid_control.utils.thread_tools import GCEvent
from grid_control.utils.user_interface import UserInputInterface
from hpfwk import APIError, ExceptionCollector, Plugin, clear_current_exception
from python_compat import any, identity, ifilter, imap, lmap, sorted, unspecified
CommandType = make_enum(['executable', 'command']) # pylint: disable=invalid-name
class TypedConfigInterface(ConfigInterface):
# Config interface class accessing typed data using an string interface provided by config_view
def get_bool(self, option, default=unspecified, **kwargs):
# Getting boolean config options - feature: true and false are not the only valid expressions
def _str2obj(value):
result = parse_bool(value)
if result is None:
raise ConfigError(
'Valid boolean expressions are: "true", "false"')
return result
return self._get_internal('bool', bool.__str__, _str2obj, None, option,
default, **kwargs)
import os, time, logging
from grid_control.gc_plugin import ConfigurablePlugin
from grid_control.parameters.psource_base import ParameterError, ParameterInfo, ParameterMetadata, ParameterSource # pylint:disable=line-too-long
from grid_control.utils import ensure_dir_exists
from grid_control.utils.activity import Activity
from grid_control.utils.algos import filter_dict, get_list_difference, reverse_dict
from grid_control.utils.data_structures import make_enum
from grid_control.utils.file_tools import GZipTextFile
from grid_control.utils.parsing import str_time_short
from grid_control.utils.user_interface import UserInputInterface
from hpfwk import APIError
from python_compat import ifilter, iidfilter, imap, irange, itemgetter, lfilter, lmap, md5_hex, set, sort_inplace, sorted # pylint:disable=line-too-long
# TrackingInfo enum values == fast resync tuple indices
TrackingInfo = make_enum(['ACTIVE', 'HASH', 'pnum'], use_hash=False) # pylint:disable=invalid-name
class ParameterAdapter(ConfigurablePlugin):
def __init__(self, config, source):
ConfigurablePlugin.__init__(self, config)
self._log = logging.getLogger('parameters.adapter')
self._psrc = source
def __repr__(self):
return '%s(%r)' % (self.__class__.__name__, self._psrc)
def can_finish(self):
return self._psrc.can_finish()
def can_submit(self, jobnum):
def _iter_blocks_raw(self):
# List of (partial or complete) block dicts with format
# { NEntries: 123, Dataset: '/path/to/data', Block: 'abcd-1234', Locations: ['site1','site2'],
# Filelist: [{URL: '/path/to/file1', NEntries: 100}, {URL: '/path/to/file2', NEntries: 23}]}
raise AbstractError
def _raise_on_abort(self):
if abort():
raise DatasetError(
'Received abort request during dataset retrieval')
make_enum( # To uncover errors, the enums of DataProvider / DataSplitter do *NOT* match type wise
[
'NEntries', 'BlockName', 'Dataset', 'Locations', 'URL', 'FileList',
'Nickname', 'Metadata', 'Provider', 'Query'
], DataProvider)
def _split_metadata_idx_list(block):
def _get_metadata_hash(fi, idx):
if idx < len(fi[DataProvider.Metadata]):
return md5_hex(repr(fi[DataProvider.Metadata][idx]))
fi_list = block[DataProvider.FileList]
common_metadata_idx_list = lrange(len(block[DataProvider.Metadata]))
if fi_list:
common_metadata_hash_list = lmap(
lambda idx: _get_metadata_hash(fi_list[0], idx),
common_metadata_idx_list)
# | Unless required by applicable law or agreed to in writing, software # | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. import re, fnmatch, logging from grid_control.config.config_entry import join_config_locations from grid_control.gc_plugin import ConfigurablePlugin from grid_control.utils.data_structures import make_enum from grid_control.utils.parsing import str_dict_linear from hpfwk import AbstractError, Plugin from python_compat import lfilter, sorted, unspecified ListOrder = make_enum(['source', 'matcher']) # pylint: disable=invalid-name class MatcherHolder(object): def __init__(self, selector, case): self._case = case self._selector = selector self._assign_selector(selector) def __repr__(self): if self._case: return self.__class__.__name__ + '(%s)' % repr(self._selector) return self.__class__.__name__ + '_ci(%s)' % repr(self._selector) def match(self, value): raise AbstractError
# | http://www.apache.org/licenses/LICENSE-2.0
# |
# | Unless required by applicable law or agreed to in writing, software
# | distributed under the License is distributed on an "AS IS" BASIS,
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
import os, gzip, logging
from grid_control.utils import DictFormat
from grid_control.utils.data_structures import make_enum
from grid_control.utils.file_tools import SafeFile
from hpfwk import AbstractError, NestedException, Plugin, clear_current_exception, get_current_exception
from python_compat import bytes2str, ifilter, izip
JobResult = make_enum(['JOBNUM', 'MESSAGE', 'EXITCODE', 'RAW']) # pylint:disable=invalid-name
FileInfo = make_enum(['Hash', 'NameLocal', 'NameDest', 'Path', 'Size']) # pylint:disable=invalid-name
class JobResultError(NestedException):
pass
class OutputProcessor(Plugin):
def process(self, dn):
raise AbstractError
class TaskOutputProcessor(Plugin):
def process(self, dn, task):
raise AbstractError
# |
# | http://www.apache.org/licenses/LICENSE-2.0
# |
# | Unless required by applicable law or agreed to in writing, software
# | distributed under the License is distributed on an "AS IS" BASIS,
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
from grid_control.datasets.dproc_base import DataProcessor
from grid_control.datasets.provider_base import DataProvider, DatasetError
from grid_control.utils.data_structures import make_enum
from python_compat import imap, itemgetter, md5_hex, set
# Enum to specify how to react to multiple occurences of something
DatasetUniqueMode = make_enum(['warn', 'abort', 'skip', 'ignore', 'record']) # pylint: disable=invalid-name
DatasetCheckMode = make_enum(['warn', 'abort', 'ignore']) # pylint: disable=invalid-name
class BaseConsistencyProcessor(DataProcessor):
def _handle_error(self, msg, mode):
if mode == DatasetCheckMode.warn:
self._log.warning(msg)
elif mode == DatasetCheckMode.abort:
raise DatasetError(msg)
class EntriesConsistencyDataProcessor(BaseConsistencyProcessor):
alias_list = ['consistency']
def __init__(self, config, datasource_name):
# | You may obtain a copy of the License at
# |
# | http://www.apache.org/licenses/LICENSE-2.0
# |
# | Unless required by applicable law or agreed to in writing, software
# | distributed under the License is distributed on an "AS IS" BASIS,
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
import time, logging
from grid_control.utils.data_structures import make_enum
from hpfwk import AbstractError, NestedException, Plugin
from python_compat import set
ParameterInfo = make_enum(['ACTIVE', 'HASH', 'REQS', 'FILES']) # pylint:disable=invalid-name
class ParameterError(NestedException):
pass
class ParameterMetadata(object):
def __init__(self, value, untracked=False):
(self.value, self.untracked) = (value, untracked)
def __repr__(self):
return self.get_value()
def get_value(self):
if self.untracked:
import os, sys, signal
from grid_control.config.cinterface_base import ConfigInterface
from grid_control.config.config_entry import ConfigError, join_config_locations
from grid_control.config.cview_base import SimpleConfigView
from grid_control.config.matcher_base import DictLookup, ListFilter, ListOrder, Matcher
from grid_control.utils import resolve_path, resolve_paths
from grid_control.utils.data_structures import make_enum
from grid_control.utils.parsing import parse_bool, parse_dict_cfg, parse_list, parse_time, str_dict_cfg, str_time_short # pylint:disable=line-too-long
from grid_control.utils.thread_tools import GCEvent
from grid_control.utils.user_interface import UserInputInterface
from hpfwk import APIError, ExceptionCollector, Plugin, clear_current_exception
from python_compat import any, identity, ifilter, imap, lmap, sorted, unspecified
CommandType = make_enum(['executable', 'command']) # pylint: disable=invalid-name
class TypedConfigInterface(ConfigInterface):
# Config interface class accessing typed data using an string interface provided by config_view
def get_bool(self, option, default=unspecified, **kwargs):
# Getting boolean config options - feature: true and false are not the only valid expressions
def _str2obj(value):
result = parse_bool(value)
if result is None:
raise ConfigError('Valid boolean expressions are: "true", "false"')
return result
return self._get_internal('bool', bool.__str__, _str2obj, None, option, default, **kwargs)
def get_composited_plugin(self, option, default=unspecified, default_compositor=unspecified,
option_compositor=None, cls=Plugin, require_plugin=True,
# | distributed under the License is distributed on an "AS IS" BASIS,
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
import os, sys, time, logging, threading
from grid_control.gc_exceptions import GCError, GCLogHandler
from grid_control.stream_base import ActivityMonitor
from grid_control.utils.data_structures import UniqueList, make_enum
from grid_control.utils.file_tools import SafeFile, VirtualFile
from grid_control.utils.thread_tools import GCLock, with_lock
from hpfwk import AbstractError, format_exception, ignore_exception, rethrow
from python_compat import any, imap, irange, lmap, set, sorted, tarfile
LogLevelEnum = make_enum(
lmap(lambda level: logging.getLevelName(level).upper(), irange(51)), # pylint:disable=invalid-name
use_hash=False,
register=False)
def dump_log_setup(level):
# Display logging setup
output = logging.getLogger('logging')
def _display_logger(indent, logger, name):
propagate_symbol = '+'
if hasattr(logger, 'propagate') and not logger.propagate:
propagate_symbol = 'o'
desc = name
if hasattr(logger, 'level'):
desc += ' (level = %s)' % logging.getLevelName(logger.level)
output.log(level, '%s%s %s', '| ' * indent, propagate_symbol, desc)
from grid_control.utils.algos import dict_union
from grid_control.utils.data_structures import make_enum
from grid_control.utils.file_tools import SafeFile, VirtualFile
from hpfwk import AbstractError, NestedException, clear_current_exception, ignore_exception
from python_compat import ichain, identity, imap, izip, lchain, lmap, set, sorted
class BackendError(NestedException):
pass
BackendJobState = make_enum([ # pylint:disable=invalid-name
'ABORTED', # job was aborted by the WMS
'CANCELLED', # job was cancelled
'DONE', # job is finished
'QUEUED', # job is at WMS and is assigned a place to run
'RUNNING', # job is running
'UNKNOWN', # job status is unknown
'WAITING', # job is at WMS but was not yet assigned some place to run
])
WallTimeMode = make_enum(['hard', 'soft', 'ignore']) # pylint:disable=invalid-name
class WMS(NamedPlugin):
alias_list = ['NullWMS']
config_section_list = NamedPlugin.config_section_list + ['wms', 'backend']
config_tag_name = 'wms'
def __init__(self, config, name):
name = (name or self.__class__.__name__).upper().replace('.', '_')
# | # | Unless required by applicable law or agreed to in writing, software # | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. import os, gzip, logging from grid_control.utils import DictFormat from grid_control.utils.data_structures import make_enum from grid_control.utils.file_tools import SafeFile from hpfwk import AbstractError, NestedException, Plugin, clear_current_exception, get_current_exception from python_compat import bytes2str, ifilter, izip JobResult = make_enum(['JOBNUM', 'MESSAGE', 'EXITCODE', 'RAW']) # pylint:disable=invalid-name FileInfo = make_enum(['Hash', 'NameLocal', 'NameDest', 'Path', 'Size']) # pylint:disable=invalid-name class JobResultError(NestedException): pass class OutputProcessor(Plugin): def process(self, dn): raise AbstractError class TaskOutputProcessor(Plugin): def process(self, dn, task): raise AbstractError
from grid_control_gui.report_bar import ProgressBarActivity
except Exception:
clear_current_exception()
ProgressBarActivity = ProgressActivity # pylint:disable=invalid-name
log = logging.getLogger('se_output_download') # pylint:disable=invalid-name
logging.getLogger('logging.process').disabled = True
get_thread_state = resolve_fun('threading:Thread.is_alive',
'threading:Thread.isAlive') # pylint:disable=invalid-name
JobDownloadStatus = make_enum([
'JOB_OK',
'JOB_ALREADY',
'JOB_NO_OUTPUT', # pylint:disable=invalid-name
'JOB_PROCESSING',
'JOB_FAILED',
'JOB_RETRY',
'JOB_INCOMPLETE'
])
FileDownloadStatus = make_enum([
'FILE_OK',
'FILE_EXISTS',
'FILE_TIMEOUT', # pylint:disable=invalid-name
'FILE_SE_BLACKLIST',
'FILE_HASH_FAILED',
'FILE_TRANSFER_FAILED',
'FILE_MKDIR_FAILED'
])
# |
# | Unless required by applicable law or agreed to in writing, software
# | distributed under the License is distributed on an "AS IS" BASIS,
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
import re, fnmatch, logging
from grid_control.config.config_entry import join_config_locations
from grid_control.gc_plugin import ConfigurablePlugin
from grid_control.utils.data_structures import make_enum
from grid_control.utils.parsing import str_dict_linear
from hpfwk import AbstractError, Plugin
from python_compat import lfilter, sorted, unspecified
ListOrder = make_enum(['source', 'matcher']) # pylint: disable=invalid-name
class MatcherHolder(object):
def __init__(self, selector, case):
self._case = case
self._selector = selector
self._assign_selector(selector)
def __repr__(self):
if self._case:
return self.__class__.__name__ + '(%s)' % repr(self._selector)
return self.__class__.__name__ + '_ci(%s)' % repr(self._selector)
def match(self, value):
raise AbstractError
# | http://www.apache.org/licenses/LICENSE-2.0 # | # | Unless required by applicable law or agreed to in writing, software # | distributed under the License is distributed on an "AS IS" BASIS, # | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # | See the License for the specific language governing permissions and # | limitations under the License. from grid_control.datasets.dproc_base import DataProcessor from grid_control.datasets.provider_base import DataProvider, DatasetError from grid_control.utils.data_structures import make_enum from python_compat import imap, itemgetter, md5_hex, set # Enum to specify how to react to multiple occurences of something DatasetUniqueMode = make_enum(['warn', 'abort', 'skip', 'ignore', 'record']) # pylint: disable=invalid-name DatasetCheckMode = make_enum(['warn', 'abort', 'ignore']) # pylint: disable=invalid-name class BaseConsistencyProcessor(DataProcessor): def _handle_error(self, msg, mode): if mode == DatasetCheckMode.warn: self._log.warning(msg) elif mode == DatasetCheckMode.abort: raise DatasetError(msg) class EntriesConsistencyDataProcessor(BaseConsistencyProcessor): alias_list = ['consistency'] def __init__(self, config, datasource_name):
matplotlib = None
from grid_control.output_processor import JobInfoProcessor, JobResult
from grid_control.report import ImageReport
from grid_control.utils.data_structures import make_enum
from hpfwk import clear_current_exception
from python_compat import BytesBuffer, irange, izip
JobMetrics = make_enum([ # pylint:disable=invalid-name
'EVENT_COUNT',
'FILESIZE_IN_TOTAL',
'FILESIZE_OUT_TOTAL',
'TS_CMSSW_CMSRUN1_DONE',
'TS_CMSSW_CMSRUN1_START',
'TS_DEPLOYMENT_DONE',
'TS_DEPLOYMENT_START',
'TS_EXECUTION_DONE',
'TS_EXECUTION_START',
'TS_SE_IN_DONE',
'TS_SE_IN_START',
'TS_SE_OUT_DONE',
'TS_SE_OUT_START',
'TS_WRAPPER_DONE',
'TS_WRAPPER_START',
])
class PlotReport(ImageReport):
alias_list = ['plot']
def __init__(self, config, name, job_db, task=None):
ImageReport.__init__(self, config, name, job_db, task)
# |
# | Unless required by applicable law or agreed to in writing, software
# | distributed under the License is distributed on an "AS IS" BASIS,
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
import logging
from grid_control.backends.backend_tools import BackendError, BackendExecutor
from grid_control.job_db import Job
from grid_control.utils import abort
from grid_control.utils.data_structures import make_enum
from hpfwk import AbstractError
from python_compat import set
CheckInfo = make_enum(['WMSID', 'RAW_STATUS', 'QUEUE', 'WN', 'SITE']) # pylint:disable=invalid-name
CheckStatus = make_enum(['OK', 'ERROR']) # pylint:disable=invalid-name
# TODO: Error Handler Plugins - logging, exception, errorcode - with abort / continue
def expand_status_map(map_job_status2status_str_list):
result = {}
for job_status, status_str_list in map_job_status2status_str_list.items():
result.update(dict.fromkeys(status_str_list, job_status))
return result
class CheckJobs(BackendExecutor):
def execute(self,
wms_id_list): # yields list of (wms_id, job_status, job_info)
# | limitations under the License.
from grid_control.config import TriggerResync
from grid_control.datasets import DataProvider, DataSplitter, DatasetError
from grid_control.datasets.splitter_basic import HybridSplitter
from grid_control.utils import split_opt
from grid_control.utils.activity import Activity, ProgressActivity
from grid_control.utils.data_structures import make_enum
from grid_control.utils.thread_tools import start_thread
from grid_control.utils.webservice import JSONRestClient
from grid_control_cms.lumi_tools import parse_lumi_filter, str_lumi
from grid_control_cms.sitedb import SiteDB
from hpfwk import AbstractError
from python_compat import itemgetter, lfilter, sorted
CMSLocationFormat = make_enum(['hostname', 'siteDB', 'both']) # pylint:disable=invalid-name
class CMSBaseProvider(DataProvider):
# required format: <dataset path>[@<instance>][#<block>]
def __init__(self,
config,
datasource_name,
dataset_expr,
dataset_nick=None,
dataset_proc=None):
dataset_config = config.change_view(
default_on_change=TriggerResync(['datasets', 'parameters']))
self._lumi_filter = dataset_config.get_lookup(
['lumi filter', '%s lumi filter' % datasource_name],
default={},
# | http://www.apache.org/licenses/LICENSE-2.0
# |
# | Unless required by applicable law or agreed to in writing, software
# | distributed under the License is distributed on an "AS IS" BASIS,
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
from grid_control.datasets import DataProcessor, DataProvider, DataSplitter, DatasetError, PartitionProcessor # pylint:disable=line-too-long
from grid_control.parameters import ParameterMetadata
from grid_control.utils.algos import safe_index
from grid_control.utils.data_structures import make_enum
from grid_control_cms.lumi_tools import filter_lumi_filter, format_lumi, parse_lumi_filter, select_lumi, select_run, str_lumi # pylint:disable=line-too-long
from python_compat import any, ichain, imap, izip, set
LumiKeep = make_enum(['RunLumi', 'Run', 'none']) # pylint:disable=invalid-name
LumiMode = make_enum(['strict', 'weak']) # pylint:disable=invalid-name
class LumiDataProcessor(DataProcessor):
alias_list = ['lumi']
def __init__(self, config, datasource_name):
DataProcessor.__init__(self, config, datasource_name)
self._lumi_filter = config.get_lookup(
['lumi filter', '%s lumi filter' % datasource_name],
default={},
parser=parse_lumi_filter,
strfun=str_lumi)
if self._lumi_filter.empty():
lumi_keep_default = LumiKeep.none
from grid_control.backends.aspect_cancel import CancelAndPurgeJobs
from grid_control.backends.aspect_status import CheckJobsMissingState
from grid_control.backends.backend_tools import unpack_wildcard_tar
from grid_control.backends.broker_base import Broker
from grid_control.backends.condor_wms.processhandler import ProcessHandler
from grid_control.backends.wms import BackendError, BasicWMS, WMS, WallTimeMode
from grid_control.backends.wms_condor import CondorCancelJobs, CondorCheckJobs
from grid_control.backends.wms_local import LocalPurgeJobs, SandboxHelper
from grid_control.utils import Result, ensure_dir_exists, get_path_share, remove_files, resolve_install_path, safe_write, split_blackwhite_list # pylint:disable=line-too-long
from grid_control.utils.activity import Activity
from grid_control.utils.data_structures import make_enum
from python_compat import imap, irange, lfilter, lmap, lzip, md5_hex
# if the ssh stuff proves too hack'y: http://www.lag.net/paramiko/
PoolType = make_enum(['LOCAL', 'SPOOL', 'SSH', 'GSISSH']) # pylint:disable=invalid-name
class CondorJDLWriter(object):
def __init__(self, config):
self._email = config.get(['notifyemail', 'email'], '', on_change=None)
self._classad_list = config.get_list(['classaddata', 'classad data'], [], on_change=None)
self._jdl_list = config.get_list(['jdldata', 'jdl data'], [], on_change=None)
self._pool_query_dict = config.get_dict('poolArgs query', {})[0]
def get_jdl(self):
jdl_str_list = []
if self._email:
jdl_str_list.append('notify_user = ' + self._email)
# properly inject any information retrieval keys into ClassAds
# regular attributes do not need injecting
# | distributed under the License is distributed on an "AS IS" BASIS,
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
import os, sys, time, logging, threading
from grid_control.gc_exceptions import GCError, GCLogHandler
from grid_control.stream_base import ActivityMonitor
from grid_control.utils.data_structures import UniqueList, make_enum
from grid_control.utils.file_tools import SafeFile, VirtualFile
from grid_control.utils.thread_tools import GCLock, with_lock
from hpfwk import AbstractError, format_exception, ignore_exception, rethrow
from python_compat import any, imap, irange, lmap, set, sorted, tarfile
LogLevelEnum = make_enum(lmap(lambda level: logging.getLevelName(level).upper(), irange(51)), # pylint:disable=invalid-name
use_hash=False, register=False)
def clean_logger(logger_name=None):
logger = logging.getLogger(logger_name)
logger.handlers = []
return logger
def dump_log_setup(level):
# Display logging setup
output = logging.getLogger('logging')
def _display_logger(indent, logger, name):
propagate_symbol = '+'
if hasattr(logger, 'propagate') and not logger.propagate:
partition[ds_prop] = block[dp_prop] if DataProvider.Metadata in block: partition[DataSplitter.MetadataHeader] = block[DataProvider.Metadata] # Helper for very simple splitter if fi_list: partition[DataSplitter.FileList] = lmap(itemgetter(DataProvider.URL), fi_list) partition[DataSplitter.NEntries] = sum(imap(itemgetter(DataProvider.NEntries), fi_list)) if DataProvider.Metadata in block: partition[DataSplitter.Metadata] = lmap(itemgetter(DataProvider.Metadata), fi_list) return partition def _get_part_opt(self, *args): return join_config_locations(['', self._datasource_name], *args) make_enum(['Dataset', 'Locations', 'NEntries', 'Skipped', 'FileList', 'Nickname', 'DatasetID', # DatasetID is legacy 'CommonPrefix', 'Invalid', 'BlockName', 'MetadataHeader', 'Metadata', 'Comment'], DataSplitter, use_hash=False) class PartitionResyncHandler(ConfigurablePlugin): def resync(self, splitter, reader, block_list_old, block_list_new): raise AbstractError class PartitionReader(Plugin): def __init__(self, partition_len): self._lock = GCLock() self._partition_len = partition_len def get_partition_checked(self, partition_num): if partition_num >= self._partition_len:
# |
# | Unless required by applicable law or agreed to in writing, software
# | distributed under the License is distributed on an "AS IS" BASIS,
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
from grid_control.datasets import DataProcessor, DataProvider, DataSplitter, DatasetError, PartitionProcessor # pylint:disable=line-too-long
from grid_control.parameters import ParameterMetadata
from grid_control.utils.algos import safe_index
from grid_control.utils.data_structures import make_enum
from grid_control_cms.lumi_tools import filter_lumi_filter, format_lumi, parse_lumi_filter, select_lumi, select_run, str_lumi # pylint:disable=line-too-long
from python_compat import any, ichain, imap, izip, set
LumiKeep = make_enum(['RunLumi', 'Run', 'none']) # pylint:disable=invalid-name
LumiMode = make_enum(['strict', 'weak']) # pylint:disable=invalid-name
class LumiDataProcessor(DataProcessor):
alias_list = ['lumi']
def __init__(self, config, datasource_name):
DataProcessor.__init__(self, config, datasource_name)
self._lumi_filter = config.get_lookup(['lumi filter', '%s lumi filter' % datasource_name],
default={}, parser=parse_lumi_filter, strfun=str_lumi)
if self._lumi_filter.empty():
lumi_keep_default = LumiKeep.none
else:
lumi_keep_default = LumiKeep.Run
config.set_bool('%s lumi metadata' % datasource_name, True)
import os, re, time, tempfile
from grid_control.backends.aspect_cancel import CancelAndPurgeJobs
from grid_control.backends.aspect_status import CheckJobsMissingState
from grid_control.backends.backend_tools import unpack_wildcard_tar
from grid_control.backends.broker_base import Broker
from grid_control.backends.condor_wms.processhandler import ProcessHandler
from grid_control.backends.wms import BackendError, BasicWMS, WMS, WallTimeMode
from grid_control.backends.wms_condor import CondorCancelJobs, CondorCheckJobs
from grid_control.backends.wms_local import LocalPurgeJobs, SandboxHelper
from grid_control.utils import Result, ensure_dir_exists, get_path_share, remove_files, resolve_install_path, safe_write, split_blackwhite_list # pylint:disable=line-too-long
from grid_control.utils.activity import Activity
from grid_control.utils.data_structures import make_enum
from python_compat import imap, irange, lfilter, lmap, lzip, md5_hex
# if the ssh stuff proves too hack'y: http://www.lag.net/paramiko/
PoolType = make_enum(['LOCAL', 'SPOOL', 'SSH', 'GSISSH']) # pylint:disable=invalid-name
class CondorJDLWriter(object):
def __init__(self, config):
self._email = config.get(['notifyemail', 'email'], '', on_change=None)
self._classad_list = config.get_list(['classaddata', 'classad data'],
[],
on_change=None)
self._jdl_list = config.get_list(['jdldata', 'jdl data'], [],
on_change=None)
self._pool_query_dict = config.get_dict('poolArgs query', {})[0]
def get_jdl(self):
jdl_str_list = []
if self._email:
url = self._url
if api:
url += '/%s' % api
try:
return self._session.request(mode, url=url, headers=header_dict,
params=params, data=data, cert=self._cert)
except Exception:
raise RestError('Unable to query %r (params: %r, cert: %r, session: %r)' %
(url, params, self._cert, self._session.__class__.__name__))
class RestSession(Plugin):
def request(self, mode, url, headers, params=None, data=None, cert=None):
raise AbstractError
make_enum(['GET', 'PUT', 'POST', 'DELETE'], RestSession)
class JSONRestClient(RestClient):
def __init__(self, cert=None, url=None, default_headers=None, process_result=None):
RestClient.__init__(self, cert, url,
default_headers or {'Content-Type': 'application/json', 'Accept': 'application/json'},
process_result=process_result or self._process_json_result, process_data=json.dumps)
def _process_json_result(self, value):
if not value:
raise RestError('Received empty reply')
try:
return parse_json(value)
except Exception:
raise RestError('Received invalid JSON reply: %r' % value)
