def can_submit(self, needed_time, can_currently_submit):
if self._get_timeleft(cached=True) < 0:
raise UserError(
'Your access token (%s) expired %s ago! (Required lifetime: %s)'
% (self.get_object_name(),
str_time_long(-self._get_timeleft(cached=True)),
str_time_long(self._min_life_time)))
if not self._check_time_left(self._min_life_time):
raise UserError(
'Your access token (%s) only has %d seconds left! (Required are %s)'
% (self.get_object_name(), self._get_timeleft(cached=True),
str_time_long(self._min_life_time)))
if self._ignore_time or (needed_time < 0):
return True
if not self._check_time_left(self._min_life_time +
needed_time) and can_currently_submit:
self._log.log_time(
logging.WARNING,
'Access token (%s) lifetime (%s) does not meet the access and walltime (%s) requirements!',
self.get_object_name(),
str_time_long(self._get_timeleft(cached=False)),
str_time_long(self._min_life_time + needed_time))
self._log.log_time(logging.WARNING, 'Disabling job submission')
return False
return True
def userRefresh(config, old_obj, cur_obj, cur_entry, obj2str):
if (old_obj == '') and (cur_obj != ''):
raise UserError('It is currently not possible to attach a dataset to a non-dataset task!')
self._log.info('Dataset setup was changed - forcing resync...')
config.setState(True, 'resync', detail = 'dataset')
config.setState(True, 'init', detail = 'config') # This will trigger a write of the new options
return cur_obj
def genTarball(outFile, fileList):
tar = tarfile.open(outFile, 'w:gz')
activity = None
for (pathAbs, pathRel, pathStatus) in fileList:
if pathStatus is True: # Existing file
tar.add(pathAbs, pathRel, recursive=False)
elif pathStatus is False: # Existing file
if not os.path.exists(pathAbs):
raise UserError('File %s does not exist!' % pathRel)
tar.add(pathAbs, pathRel, recursive=False)
elif pathStatus is None: # Directory
del activity
msg = QM(
len(pathRel) > 50, pathRel[:15] + '...' + pathRel[-32:],
pathRel)
activity = ActivityLog('Generating tarball: %s' % msg)
else: # File handle
info, handle = pathStatus.getTarInfo()
info.mtime = time.time()
info.mode = stat.S_IRUSR + stat.S_IWUSR + stat.S_IRGRP + stat.S_IROTH
if info.name.endswith('.sh') or info.name.endswith('.py'):
info.mode += stat.S_IXUSR + stat.S_IXGRP + stat.S_IXOTH
tar.addfile(info, handle)
handle.close()
del activity
tar.close()
def __init__(self, arg, **kwargs):
idList = imap(lambda x: x.split('-'), arg.split(','))
try:
parse = lambda x: utils.QM(x != '', int, str)
self._ranges = lmap(
lambda x: (parse(x[0])(x[0]), parse(x[-1])(x[-1])), idList)
except Exception:
raise UserError('Job identifiers must be integers or ranges.')
def __init__(self, url):
self._reader_url = '%s/%s' % (url, 'DBSReader')
self._writer_url = '%s/%s' % (url, 'DBSWriter')
self._migrate_url = '%s/%s' % (url, 'DBSMigrate')
self._proxy_path = os.environ.get('X509_USER_PROXY', '')
if not os.path.exists(self._proxy_path):
raise UserError(
'VOMS proxy needed to query DBS3! Environment variable X509_USER_PROXY is "%s"'
% self._proxy_path)
self._jrc = JSONRestClient(cert=self._proxy_path)
def __init__(self, arg, **kwargs):
id_list = imap(lambda x: x.split('-'), arg.split(','))
try:
def _parse(value):
if value != '':
return int
return str
self._ranges = lmap(lambda x: (_parse(x[0])(x[0]), _parse(x[-1])(x[-1])), id_list)
except Exception:
raise UserError('Job identifiers must be integers or ranges.')
def _setupJobParameters(self, config):
data_config = config.changeView(viewClass = 'TaggedConfigView', addSections = ['dataset'])
self.dataSplitter = None
self._data_refresh = -1
def userRefresh(config, old_obj, cur_obj, cur_entry, obj2str):
if (old_obj == '') and (cur_obj != ''):
raise UserError('It is currently not possible to attach a dataset to a non-dataset task!')
self._log.info('Dataset setup was changed - forcing resync...')
config.setState(True, 'resync', detail = 'dataset')
config.setState(True, 'init', detail = 'config') # This will trigger a write of the new options
return cur_obj
dataProvider = data_config.getCompositePlugin('dataset', '', ':MultiDatasetProvider:',
cls = DataProvider, requirePlugin = False, onChange = userRefresh)
self._forceRefresh = config.getState('resync', detail = 'dataset')
config.setState(False, 'resync', detail = 'dataset')
if not dataProvider:
return
tmp_config = data_config.changeView(viewClass = 'TaggedConfigView', setClasses = None, setNames = None, setTags = [], addSections = ['storage'])
tmp_config.set('se output pattern', '@NICK@_job_@GC_JOB_ID@_@X@')
tmp_config = data_config.changeView(viewClass = 'TaggedConfigView', setClasses = None, setNames = None, setTags = [], addSections = ['parameters'])
tmp_config.set('default lookup', 'DATASETNICK')
splitterName = data_config.get('dataset splitter', 'FileBoundarySplitter')
splitterClass = dataProvider.checkSplitter(DataSplitter.getClass(splitterName))
self.dataSplitter = splitterClass(data_config)
# Create and register dataset parameter source
partProcessor = data_config.getCompositePlugin('partition processor',
'TFCPartitionProcessor LocationPartitionProcessor MetaPartitionProcessor BasicPartitionProcessor',
'MultiPartitionProcessor', cls = PartitionProcessor, onChange = triggerResync(['parameters']))
DataParameterSource = ParameterSource.getClass('DataParameterSource')
self._dataPS = DataParameterSource(data_config.getWorkPath(), 'data',
dataProvider, self.dataSplitter, partProcessor)
DataParameterSource.datasetsAvailable['data'] = self._dataPS
# Select dataset refresh rate
self._data_refresh = data_config.getTime('dataset refresh', -1, onChange = None)
if self._data_refresh > 0:
self._dataPS.resyncSetup(interval = max(self._data_refresh, dataProvider.queryLimit()))
utils.vprint('Dataset source will be queried every %s' % strTime(self._data_refresh), -1)
else:
self._dataPS.resyncSetup(interval = 0)
if self._forceRefresh:
self._dataPS.resyncSetup(force = True)
def externalRefresh(sig, frame):
self._dataPS.resyncSetup(force = True)
signal.signal(signal.SIGUSR2, externalRefresh)
if self.dataSplitter.getMaxJobs() == 0:
raise UserError('There are no events to process')
def canSubmit(self, neededTime, canCurrentlySubmit):
if not self._checkTimeleft(self._lowerLimit):
raise UserError(
'Your access token (%s) only has %d seconds left! (Required are %s)'
% (self.getObjectName(), self._getTimeleft(cached=True),
strTime(self._lowerLimit)))
if self._ignoreTime:
return True
if not self._checkTimeleft(self._lowerLimit +
neededTime) and canCurrentlySubmit:
self._logUser.warning(
'Access token (%s) lifetime (%s) does not meet the access and walltime (%s) requirements!',
self.getObjectName(), strTime(self._getTimeleft(cached=False)),
strTime(self._lowerLimit + neededTime))
self._logUser.warning('Disabling job submission')
return False
return True
def __init__(self, config, datasource_name, repository, keep_old=True):
BaseDataParameterSource.__init__(self, config, datasource_name,
repository)
# hide provider property set by __new__
self._provider = self.provider
del self.provider
if self._provider.need_init_query():
self._provider.get_block_list_cached(show_stats=False)
data_src_text = 'Dataset source %r' % datasource_name
# Select dataset refresh rate
data_refresh = config.get_time('%s refresh' % datasource_name,
-1,
on_change=None)
if data_refresh >= 0:
data_refresh = max(data_refresh,
self._provider.get_query_interval())
self._log.info('%s will be queried every %s', data_src_text,
str_time_long(data_refresh))
self.setup_resync(interval=data_refresh,
force=config.get_state('resync', detail='datasets'))
splitter_name = config.get('%s splitter' % datasource_name,
'FileBoundarySplitter')
splitter_cls = self._provider.check_splitter(
DataSplitter.get_class(splitter_name))
self._splitter = splitter_cls(config, datasource_name)
# Settings:
(self._dn, self._keep_old) = (config.get_work_path(), keep_old)
ensure_dir_exists(self._dn, 'partition map directory', DatasetError)
self._set_reader(self._init_reader())
if not self.get_parameter_len():
if data_refresh < 0:
raise UserError('%s does not provide jobs to process' %
data_src_text)
self._log.warning('%s does not provide jobs to process',
data_src_text)
def _setupJobParameters(self, config, psrc_repository):
TaskModule._setupJobParameters(self, config, psrc_repository)
data_config = config.changeView(viewClass='TaggedConfigView',
addSections=['dataset'])
self._dataSplitter = None
dataProvider = data_config.getCompositePlugin(
'dataset',
'',
':MultiDatasetProvider:',
cls=DataProvider,
requirePlugin=False,
onChange=triggerResync(['datasets', 'parameters']))
self._forceRefresh = config.getState('resync', detail='datasets')
config.setState(False, 'resync', detail='datasets')
if not dataProvider:
return
tmp_config = data_config.changeView(viewClass='TaggedConfigView',
setClasses=None,
setNames=None,
setTags=[],
addSections=['storage'])
tmp_config.set('se output pattern', '@NICK@_job_@GC_JOB_ID@_@X@')
tmp_config = data_config.changeView(viewClass='TaggedConfigView',
setClasses=None,
setNames=None,
setTags=[],
addSections=['parameters'])
tmp_config.set('default lookup', 'DATASETNICK')
splitterName = data_config.get('dataset splitter',
'FileBoundarySplitter')
splitterClass = dataProvider.checkSplitter(
DataSplitter.getClass(splitterName))
self._dataSplitter = splitterClass(data_config)
# Create and register dataset parameter source
self._partProcessor = data_config.getCompositePlugin(
'partition processor',
'TFCPartitionProcessor LocationPartitionProcessor MetaPartitionProcessor BasicPartitionProcessor',
'MultiPartitionProcessor',
cls=PartitionProcessor,
onChange=triggerResync(['parameters']))
dataPS = ParameterSource.createInstance('DataParameterSource',
data_config.getWorkPath(),
'data', dataProvider,
self._dataSplitter,
self._partProcessor,
psrc_repository)
# Select dataset refresh rate
data_refresh = data_config.getTime('dataset refresh',
-1,
onChange=None)
if data_refresh >= 0:
data_refresh = max(data_refresh, dataProvider.queryLimit())
self._log.info('Dataset source will be queried every %s',
strTime(data_refresh))
dataPS.resyncSetup(interval=data_refresh, force=self._forceRefresh)
def externalRefresh(sig, frame):
self._log.info(
'External signal triggered resync of dataset source')
dataPS.resyncSetup(force=True)
signal.signal(signal.SIGUSR2, externalRefresh)
if self._dataSplitter.getMaxJobs() == 0:
if data_refresh < 0:
raise UserError(
'Currently used dataset does not provide jobs to process')
self._log.warning(
'Currently used dataset does not provide jobs to process')
def create(cls, pconfig, repository, src='data'): # pylint:disable=arguments-differ
src_key = 'dataset:%s' % src
if src_key not in repository:
raise UserError('Dataset parameter source "%s" not setup!' % src)
return repository[src_key]
def create(cls, pconfig=None, src='data'): # pylint:disable=arguments-differ
if src not in DataParameterSource.datasetsAvailable:
raise UserError('Dataset parameter source "%s" not setup!' % src)
result = DataParameterSource.datasetsAvailable[src]
DataParameterSource.datasetsUsed.append(result)
return result
def __init__(self, config, name, job_db, task=None):
TableReport.__init__(self, config, name, job_db, task)
if not task:
raise UserError('Variables report needs task information!')
self._task = task
