def __init__(self, config, name):
# Determine ROOT path from previous settings / environment / config file
def _check_root_dn(loc, obj):
if os.path.isdir(obj):
return obj
raise ConfigError(
'Either set environment variable "ROOTSYS" or set option "root path"!'
)
self._root_dn = config.get_dn('root path',
os.environ.get('ROOTSYS', ''),
persistent=True,
on_change=TriggerInit('sandbox'),
on_valid=_check_root_dn)
logging.getLogger('task').info('Using the following ROOT path: %s',
self._root_dn)
# Special handling for executables bundled with ROOT
self._executable = config.get('executable',
on_change=TriggerInit('sandbox'))
exe_full = os.path.join(self._root_dn, 'bin',
self._executable.lstrip('/'))
self._is_builtin = os.path.exists(exe_full)
if self._is_builtin:
config.set('send executable', 'False')
# store resolved built-in executable path?
# Apply default handling from UserTask
UserTask.__init__(self, config, name)
self._update_map_error_code2msg(get_path_share('gc-run.root.sh'))
# TODO: Collect lib files needed by executable
self._lib_fn_list = []
def __init__(self, config, name):
DataTask.__init__(self, config, name)
config.set('area files matcher mode', 'ShellStyleMatcher')
# SCRAM settings
scram_arch_default = unspecified
scram_project = config.get_list('scram project', [])
if scram_project: # manual scram setup
if len(scram_project) != 2:
raise ConfigError('%r needs exactly 2 arguments: <PROJECT> <VERSION>' % 'scram project')
self._project_area = None
self._project_area_selector_list = None
self._scram_project = scram_project[0]
self._scram_project_version = scram_project[1]
# ensure project area is not used
if 'project area' in config.get_option_list():
raise ConfigError('Cannot specify both %r and %r' % ('scram project', 'project area'))
else: # scram setup used from project area
self._project_area = config.get_dn('project area')
self._always_matcher = Matcher.create_instance('AlwaysMatcher', config, [''])
self._project_area_base_fn = config.get_bool('area files basename', True,
on_change=TriggerInit('sandbox'))
self._project_area_matcher = config.get_matcher('area files',
'-.* -config bin lib python module data *.xml *.sql *.db *.cfi *.cff *.py -CVS -work.* *.pcm',
default_matcher='BlackWhiteMatcher', on_change=TriggerInit('sandbox'))
self._log.info('Project area found in: %s', self._project_area)
# try to determine scram settings from environment settings
scram_path = os.path.join(self._project_area, '.SCRAM')
scram_env = self._parse_scram_file(os.path.join(scram_path, 'Environment'))
try:
self._scram_project = scram_env['SCRAM_PROJECTNAME']
self._scram_project_version = scram_env['SCRAM_PROJECTVERSION']
except:
raise ConfigError('Installed program in project area not recognized.')
def filter_arch_dir(dn):
return os.path.isdir(os.path.join(scram_path, dn))
for arch_dir in sorted(ifilter(filter_arch_dir, os.listdir(scram_path))):
scram_arch_default = arch_dir
self._scram_version = config.get('scram version', 'scramv1')
self._scram_arch = config.get('scram arch', scram_arch_default)
self._scram_req_list = []
if config.get_bool('scram arch requirements', True, on_change=None):
self._scram_req_list.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scram_arch))
if config.get_bool('scram project requirements', False, on_change=None):
self._scram_req_list.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scram_project))
if config.get_bool('scram project version requirements', False, on_change=None):
self._scram_req_list.append((WMS.SOFTWARE, 'VO-cms-%s' % self._scram_project_version))
def __init__(self, config, prefix='', executable_default=unspecified):
init_sandbox = TriggerInit('sandbox')
self._executable_send = config.get_bool('%s send executable' % prefix, True,
on_change=init_sandbox)
if self._executable_send:
self._executable = config.get_fn('%s executable' % prefix, executable_default,
on_change=init_sandbox)
else:
self._executable = config.get('%s executable' % prefix, executable_default,
on_change=init_sandbox)
self._arguments = config.get('%s arguments' % prefix, '', on_change=init_sandbox)
def __init__(self, config, name):
name = (name or self.__class__.__name__).upper().replace('.', '_')
NamedPlugin.__init__(self, config, name)
self._wait_idle = config.get_int('wait idle', 60, on_change=None)
self._wait_work = config.get_int('wait work', 10, on_change=None)
self._job_parser = config.get_plugin('job parser', 'JobInfoProcessor',
cls='JobInfoProcessor', on_change=None)
self._remote_event_handler = config.get_composited_plugin(
['remote monitor', 'remote event handler'], '', 'MultiRemoteEventHandler',
cls=RemoteEventHandler, bind_kwargs={'tags': [self]},
require_plugin=False, on_change=TriggerInit('sandbox')) or RemoteEventHandler(config, 'dummy')
def __init__(self, config,
name): # Read configuration options and init vars
NamedPlugin.__init__(self, config, name)
init_sandbox = TriggerInit('sandbox')
self._var_checker = NoVarCheck(config)
# Task requirements
# Move this into parameter manager?
jobs_config = config.change_view(view_class='TaggedConfigView',
add_sections=['jobs'],
add_tags=[self])
self.wall_time = jobs_config.get_time('wall time', on_change=None)
self._cpu_time = jobs_config.get_time('cpu time',
self.wall_time,
on_change=None)
self._cores = jobs_config.get_int(['cores', 'cpus'], 1, on_change=None)
self._memory = jobs_config.get_int('memory', -1, on_change=None)
self._job_timeout = jobs_config.get_time('node timeout',
-1,
on_change=init_sandbox)
# Compute / get task ID
self._task_id = config.get('task id',
'GC' + md5_hex(str(time.time()))[:12],
persistent=True)
self._task_date = config.get('task date',
time.strftime('%Y-%m-%d'),
persistent=True,
on_change=init_sandbox)
self._task_time = config.get('task time',
time.strftime('%H%M%S'),
persistent=True,
on_change=init_sandbox)
task_name_generator = config.get_plugin('task name generator',
'DefaultTaskName',
cls=TaskNamePlugin)
self._task_name = task_name_generator.get_name(self)
self._task_config_name = config.get_config_name()
self._job_name_generator = config.get_plugin('job name generator',
'DefaultJobName',
cls=JobNamePlugin)
# Storage setup
storage_config = config.change_view(view_class='TaggedConfigView',
set_classes=None,
set_names=None,
add_sections=['storage'],
add_tags=[self])
scratch_space_used = storage_config.get_int('scratch space used',
5000,
on_change=init_sandbox)
lz_space_used = storage_config.get_int('landing zone space used',
100,
on_change=init_sandbox)
self._task_var_dict = {
# Space limits
'SCRATCH_UL':
scratch_space_used,
'SCRATCH_LL':
storage_config.get_int('scratch space left',
1,
on_change=init_sandbox),
'LANDINGZONE_UL':
lz_space_used,
'LANDINGZONE_LL':
storage_config.get_int('landing zone space left',
1,
on_change=init_sandbox),
}
storage_config.set('se output pattern', 'job_@GC_JOB_ID@_@X@')
self._se_min_size = storage_config.get_int('se min size',
-1,
on_change=init_sandbox)
self._disk_min = max(scratch_space_used, lz_space_used)
self._sb_in_fn_list = config.get_path_list('input files', [],
on_change=init_sandbox)
self._sb_out_fn_list = config.get_list('output files', [],
on_change=init_sandbox)
self._do_gzip_std_output = config.get_bool('gzip output',
True,
on_change=init_sandbox)
self._subst_files = config.get_list('subst files', [],
on_change=init_sandbox)
self._dependencies = lmap(
str.lower, config.get_list('depends', [], on_change=init_sandbox))
# Get error messages from gc-run.lib comments
self.map_error_code2msg = {}
self._update_map_error_code2msg(get_path_share('gc-run.lib'))
# Init parameter source manager
psrc_repository = {}
self._setup_repository(config, psrc_repository)
pfactory = config.get_plugin('internal parameter factory',
'BasicParameterFactory',
cls=ParameterFactory,
bind_kwargs={
'tags': [self],
'inherit': True
})
self._source = config.get_plugin(
'parameter adapter',
'TrackedParameterAdapter',
cls=ParameterAdapter,
pargs=(pfactory.get_psrc(psrc_repository), ))
self._log.log(logging.DEBUG3, 'Using parameter adapter %s',
repr(self._source))
self._log.info('Current task ID: %s', self._task_id)
self._log.info('Task started on: %s', self._task_date)