def can_submit(self, needed_time, can_currently_submit): if self._get_timeleft(cached=True) < 0: raise UserError( 'Your access token (%s) expired %s ago! (Required lifetime: %s)' % (self.get_object_name(), str_time_long(-self._get_timeleft(cached=True)), str_time_long(self._min_life_time))) if not self._check_time_left(self._min_life_time): raise UserError( 'Your access token (%s) only has %d seconds left! (Required are %s)' % (self.get_object_name(), self._get_timeleft(cached=True), str_time_long(self._min_life_time))) if self._ignore_time or (needed_time < 0): return True if not self._check_time_left(self._min_life_time + needed_time) and can_currently_submit: self._log.log_time( logging.WARNING, 'Access token (%s) lifetime (%s) does not meet the access and walltime (%s) requirements!', self.get_object_name(), str_time_long(self._get_timeleft(cached=False)), str_time_long(self._min_life_time + needed_time)) self._log.log_time(logging.WARNING, 'Disabling job submission') return False return True
def userRefresh(config, old_obj, cur_obj, cur_entry, obj2str): if (old_obj == '') and (cur_obj != ''): raise UserError('It is currently not possible to attach a dataset to a non-dataset task!') self._log.info('Dataset setup was changed - forcing resync...') config.setState(True, 'resync', detail = 'dataset') config.setState(True, 'init', detail = 'config') # This will trigger a write of the new options return cur_obj
def genTarball(outFile, fileList): tar = tarfile.open(outFile, 'w:gz') activity = None for (pathAbs, pathRel, pathStatus) in fileList: if pathStatus is True: # Existing file tar.add(pathAbs, pathRel, recursive=False) elif pathStatus is False: # Existing file if not os.path.exists(pathAbs): raise UserError('File %s does not exist!' % pathRel) tar.add(pathAbs, pathRel, recursive=False) elif pathStatus is None: # Directory del activity msg = QM( len(pathRel) > 50, pathRel[:15] + '...' + pathRel[-32:], pathRel) activity = ActivityLog('Generating tarball: %s' % msg) else: # File handle info, handle = pathStatus.getTarInfo() info.mtime = time.time() info.mode = stat.S_IRUSR + stat.S_IWUSR + stat.S_IRGRP + stat.S_IROTH if info.name.endswith('.sh') or info.name.endswith('.py'): info.mode += stat.S_IXUSR + stat.S_IXGRP + stat.S_IXOTH tar.addfile(info, handle) handle.close() del activity tar.close()
def __init__(self, arg, **kwargs): idList = imap(lambda x: x.split('-'), arg.split(',')) try: parse = lambda x: utils.QM(x != '', int, str) self._ranges = lmap( lambda x: (parse(x[0])(x[0]), parse(x[-1])(x[-1])), idList) except Exception: raise UserError('Job identifiers must be integers or ranges.')
def __init__(self, url): self._reader_url = '%s/%s' % (url, 'DBSReader') self._writer_url = '%s/%s' % (url, 'DBSWriter') self._migrate_url = '%s/%s' % (url, 'DBSMigrate') self._proxy_path = os.environ.get('X509_USER_PROXY', '') if not os.path.exists(self._proxy_path): raise UserError( 'VOMS proxy needed to query DBS3! Environment variable X509_USER_PROXY is "%s"' % self._proxy_path) self._jrc = JSONRestClient(cert=self._proxy_path)
def __init__(self, arg, **kwargs): id_list = imap(lambda x: x.split('-'), arg.split(',')) try: def _parse(value): if value != '': return int return str self._ranges = lmap(lambda x: (_parse(x[0])(x[0]), _parse(x[-1])(x[-1])), id_list) except Exception: raise UserError('Job identifiers must be integers or ranges.')
def _setupJobParameters(self, config): data_config = config.changeView(viewClass = 'TaggedConfigView', addSections = ['dataset']) self.dataSplitter = None self._data_refresh = -1 def userRefresh(config, old_obj, cur_obj, cur_entry, obj2str): if (old_obj == '') and (cur_obj != ''): raise UserError('It is currently not possible to attach a dataset to a non-dataset task!') self._log.info('Dataset setup was changed - forcing resync...') config.setState(True, 'resync', detail = 'dataset') config.setState(True, 'init', detail = 'config') # This will trigger a write of the new options return cur_obj dataProvider = data_config.getCompositePlugin('dataset', '', ':MultiDatasetProvider:', cls = DataProvider, requirePlugin = False, onChange = userRefresh) self._forceRefresh = config.getState('resync', detail = 'dataset') config.setState(False, 'resync', detail = 'dataset') if not dataProvider: return tmp_config = data_config.changeView(viewClass = 'TaggedConfigView', setClasses = None, setNames = None, setTags = [], addSections = ['storage']) tmp_config.set('se output pattern', '@NICK@_job_@GC_JOB_ID@_@X@') tmp_config = data_config.changeView(viewClass = 'TaggedConfigView', setClasses = None, setNames = None, setTags = [], addSections = ['parameters']) tmp_config.set('default lookup', 'DATASETNICK') splitterName = data_config.get('dataset splitter', 'FileBoundarySplitter') splitterClass = dataProvider.checkSplitter(DataSplitter.getClass(splitterName)) self.dataSplitter = splitterClass(data_config) # Create and register dataset parameter source partProcessor = data_config.getCompositePlugin('partition processor', 'TFCPartitionProcessor LocationPartitionProcessor MetaPartitionProcessor BasicPartitionProcessor', 'MultiPartitionProcessor', cls = PartitionProcessor, onChange = triggerResync(['parameters'])) DataParameterSource = ParameterSource.getClass('DataParameterSource') self._dataPS = DataParameterSource(data_config.getWorkPath(), 'data', dataProvider, self.dataSplitter, partProcessor) DataParameterSource.datasetsAvailable['data'] = self._dataPS # Select dataset refresh rate self._data_refresh = data_config.getTime('dataset refresh', -1, onChange = None) if self._data_refresh > 0: self._dataPS.resyncSetup(interval = max(self._data_refresh, dataProvider.queryLimit())) utils.vprint('Dataset source will be queried every %s' % strTime(self._data_refresh), -1) else: self._dataPS.resyncSetup(interval = 0) if self._forceRefresh: self._dataPS.resyncSetup(force = True) def externalRefresh(sig, frame): self._dataPS.resyncSetup(force = True) signal.signal(signal.SIGUSR2, externalRefresh) if self.dataSplitter.getMaxJobs() == 0: raise UserError('There are no events to process')
def canSubmit(self, neededTime, canCurrentlySubmit): if not self._checkTimeleft(self._lowerLimit): raise UserError( 'Your access token (%s) only has %d seconds left! (Required are %s)' % (self.getObjectName(), self._getTimeleft(cached=True), strTime(self._lowerLimit))) if self._ignoreTime: return True if not self._checkTimeleft(self._lowerLimit + neededTime) and canCurrentlySubmit: self._logUser.warning( 'Access token (%s) lifetime (%s) does not meet the access and walltime (%s) requirements!', self.getObjectName(), strTime(self._getTimeleft(cached=False)), strTime(self._lowerLimit + neededTime)) self._logUser.warning('Disabling job submission') return False return True
def __init__(self, config, datasource_name, repository, keep_old=True): BaseDataParameterSource.__init__(self, config, datasource_name, repository) # hide provider property set by __new__ self._provider = self.provider del self.provider if self._provider.need_init_query(): self._provider.get_block_list_cached(show_stats=False) data_src_text = 'Dataset source %r' % datasource_name # Select dataset refresh rate data_refresh = config.get_time('%s refresh' % datasource_name, -1, on_change=None) if data_refresh >= 0: data_refresh = max(data_refresh, self._provider.get_query_interval()) self._log.info('%s will be queried every %s', data_src_text, str_time_long(data_refresh)) self.setup_resync(interval=data_refresh, force=config.get_state('resync', detail='datasets')) splitter_name = config.get('%s splitter' % datasource_name, 'FileBoundarySplitter') splitter_cls = self._provider.check_splitter( DataSplitter.get_class(splitter_name)) self._splitter = splitter_cls(config, datasource_name) # Settings: (self._dn, self._keep_old) = (config.get_work_path(), keep_old) ensure_dir_exists(self._dn, 'partition map directory', DatasetError) self._set_reader(self._init_reader()) if not self.get_parameter_len(): if data_refresh < 0: raise UserError('%s does not provide jobs to process' % data_src_text) self._log.warning('%s does not provide jobs to process', data_src_text)
def _setupJobParameters(self, config, psrc_repository): TaskModule._setupJobParameters(self, config, psrc_repository) data_config = config.changeView(viewClass='TaggedConfigView', addSections=['dataset']) self._dataSplitter = None dataProvider = data_config.getCompositePlugin( 'dataset', '', ':MultiDatasetProvider:', cls=DataProvider, requirePlugin=False, onChange=triggerResync(['datasets', 'parameters'])) self._forceRefresh = config.getState('resync', detail='datasets') config.setState(False, 'resync', detail='datasets') if not dataProvider: return tmp_config = data_config.changeView(viewClass='TaggedConfigView', setClasses=None, setNames=None, setTags=[], addSections=['storage']) tmp_config.set('se output pattern', '@NICK@_job_@GC_JOB_ID@_@X@') tmp_config = data_config.changeView(viewClass='TaggedConfigView', setClasses=None, setNames=None, setTags=[], addSections=['parameters']) tmp_config.set('default lookup', 'DATASETNICK') splitterName = data_config.get('dataset splitter', 'FileBoundarySplitter') splitterClass = dataProvider.checkSplitter( DataSplitter.getClass(splitterName)) self._dataSplitter = splitterClass(data_config) # Create and register dataset parameter source self._partProcessor = data_config.getCompositePlugin( 'partition processor', 'TFCPartitionProcessor LocationPartitionProcessor MetaPartitionProcessor BasicPartitionProcessor', 'MultiPartitionProcessor', cls=PartitionProcessor, onChange=triggerResync(['parameters'])) dataPS = ParameterSource.createInstance('DataParameterSource', data_config.getWorkPath(), 'data', dataProvider, self._dataSplitter, self._partProcessor, psrc_repository) # Select dataset refresh rate data_refresh = data_config.getTime('dataset refresh', -1, onChange=None) if data_refresh >= 0: data_refresh = max(data_refresh, dataProvider.queryLimit()) self._log.info('Dataset source will be queried every %s', strTime(data_refresh)) dataPS.resyncSetup(interval=data_refresh, force=self._forceRefresh) def externalRefresh(sig, frame): self._log.info( 'External signal triggered resync of dataset source') dataPS.resyncSetup(force=True) signal.signal(signal.SIGUSR2, externalRefresh) if self._dataSplitter.getMaxJobs() == 0: if data_refresh < 0: raise UserError( 'Currently used dataset does not provide jobs to process') self._log.warning( 'Currently used dataset does not provide jobs to process')
def create(cls, pconfig, repository, src='data'): # pylint:disable=arguments-differ src_key = 'dataset:%s' % src if src_key not in repository: raise UserError('Dataset parameter source "%s" not setup!' % src) return repository[src_key]
def create(cls, pconfig=None, src='data'): # pylint:disable=arguments-differ if src not in DataParameterSource.datasetsAvailable: raise UserError('Dataset parameter source "%s" not setup!' % src) result = DataParameterSource.datasetsAvailable[src] DataParameterSource.datasetsUsed.append(result) return result
def __init__(self, config, name, job_db, task=None): TableReport.__init__(self, config, name, job_db, task) if not task: raise UserError('Variables report needs task information!') self._task = task