def __init__(self, jobDB, task, jobs = None, configString = ''):
Report.__init__(self, jobDB, task, jobs, configString)
catJobs = {}
catDescDict = {}
# Assignment of jobs to categories (depending on variables and using datasetnick if available)
jobConfig = {}
varList = []
for jobNum in self._jobs:
if task:
jobConfig = task.getJobConfig(jobNum)
varList = sorted(ifilter(lambda var: '!' not in repr(var), jobConfig.keys()))
if 'DATASETSPLIT' in varList:
varList.remove('DATASETSPLIT')
varList.append('DATASETNICK')
catKey = str.join('|', imap(lambda var: '%s=%s' % (var, jobConfig[var]), varList))
catJobs.setdefault(catKey, []).append(jobNum)
if catKey not in catDescDict:
catDescDict[catKey] = dict(imap(lambda var: (var, jobConfig[var]), varList))
# Kill redundant keys from description
commonVars = dict(imap(lambda var: (var, jobConfig[var]), varList)) # seed with last varList
for catKey in catDescDict:
for key in list(commonVars.keys()):
if key not in catDescDict[catKey].keys():
commonVars.pop(key)
elif commonVars[key] != catDescDict[catKey][key]:
commonVars.pop(key)
for catKey in catDescDict:
for commonKey in commonVars:
catDescDict[catKey].pop(commonKey)
# Generate job-category map with efficient int keys - catNum becomes the new catKey
self._job2cat = {}
self._catDescDict = {}
for catNum, catKey in enumerate(sorted(catJobs)):
self._catDescDict[catNum] = catDescDict[catKey]
self._job2cat.update(dict.fromkeys(catJobs[catKey], catNum))
def __init__(self, jobDB, task, jobs = None, configString = ''):
Report.__init__(self, jobDB, task, jobs, configString)
catJobs = {}
catDescDict = {}
# Assignment of jobs to categories (depending on variables and using datasetnick if available)
for jobNum in self._jobs:
jobConfig = task.getJobConfig(jobNum)
varList = sorted(filter(lambda var: '!' not in repr(var), jobConfig.keys()))
if 'DATASETSPLIT' in varList:
varList.remove('DATASETSPLIT')
varList.append('DATASETNICK')
catKey = str.join('|', map(lambda var: '%s=%s' % (var, jobConfig[var]), varList))
catJobs.setdefault(catKey, []).append(jobNum)
if catKey not in catDescDict:
catDescDict[catKey] = dict(map(lambda var: (var, jobConfig[var]), varList))
# Kill redundant keys from description
commonVars = dict(map(lambda var: (var, jobConfig[var]), varList)) # seed with last varList
for catKey in catDescDict:
for key in commonVars.keys():
if key not in catDescDict[catKey].keys():
commonVars.pop(key)
elif commonVars[key] != catDescDict[catKey][key]:
commonVars.pop(key)
for catKey in catDescDict:
for commonKey in commonVars:
catDescDict[catKey].pop(commonKey)
# Generate job-category map with efficient int keys - catNum becomes the new catKey
self._job2cat = {}
self._catDescDict = {}
for catNum, catKey in enumerate(sorted(catJobs)):
self._catDescDict[catNum] = catDescDict[catKey]
self._job2cat.update(dict.fromkeys(catJobs[catKey], catNum))
def __init__(self, jobDB, task, jobs = None, configString = ''):
Report.__init__(self, jobDB, task, jobs, configString)
self._levelMap = {'wms': 2, 'endpoint': 3, 'site': 4, 'queue': 5}
self._useHistory = ('history' in configString)
configString = configString.replace('history', '')
self._idxList = lmap(lambda x: self._levelMap[x.lower()], configString.split())
self._idxList.reverse()
self._stateMap = [(None, 'WAITING'), (Job.RUNNING, 'RUNNING'),
(Job.FAILED, 'FAILED'), (Job.SUCCESS, 'SUCCESS')]
def __init__(self, config, name, task, eventhandler):
NamedPlugin.__init__(self, config, name)
(self._task, self._eventhandler) = (task, eventhandler)
self._log_user = logging.getLogger('user')
self._log_user_time = logging.getLogger('user.time')
self._njobs_limit = config.getInt('jobs', -1, onChange = None)
self._njobs_inflight = config.getInt('in flight', -1, onChange = None)
self._njobs_inqueue = config.getInt('in queue', -1, onChange = None)
self._chunks_submit = config.getInt('chunks submit', 100, onChange = None)
self._chunks_check = config.getInt('chunks check', 100, onChange = None)
self._chunks_retrieve = config.getInt('chunks retrieve', 100, onChange = None)
self._job_timeout = config.getTime('queue timeout', -1, onChange = None)
self._job_retries = config.getInt('max retry', -1, onChange = None)
selected = JobSelector.create(config.get('selected', '', onChange = None), task = self._task)
self.jobDB = config.getPlugin('job database', 'JobDB',
cls = JobDB, pargs = (self.getMaxJobs(self._task), selected))
self._disabled_jobs_logfile = config.getWorkPath('disabled')
self._outputProcessor = config.getPlugin('output processor', 'SandboxProcessor',
cls = TaskOutputProcessor, pargs = (task,))
self._do_shuffle = config.getBool('shuffle', False, onChange = None)
self._continuous = config.getBool('continuous', False, onChange = None)
self._reportClass = Report.getClass(config.get('abort report', 'LocationReport', onChange = None))
self._showBlocker = True
def __init__(self, config, name, job_db, task=None):
# needed in destructor:
self._task_info = {}
self._output_fn = None
Report.__init__(self, config, name, job_db, task)
task_id = 'Unknown'
name = 'Unknown'
if task:
desc = task.get_description()
task_id = desc.task_id
name = desc.task_name
self._task_info = {
'task id': task_id,
'name': name,
}
output_dn = clean_path(config.get('report file directory', on_change=None))
self._output_fn = os.path.join(output_dn, 'states' + task_id + '.json')
def __init__(self, config, name, task, eventhandler):
NamedPlugin.__init__(self, config, name)
(self._task, self._eventhandler) = (task, eventhandler)
self._log_user = logging.getLogger('user')
self._log_user_time = logging.getLogger('user.time')
self._njobs_limit = config.getInt('jobs', -1, onChange=None)
self._njobs_inflight = config.getInt('in flight', -1, onChange=None)
self._njobs_inqueue = config.getInt('in queue', -1, onChange=None)
self._chunks_enabled = config.getBool('chunks enabled',
True,
onChange=None)
self._chunks_submit = config.getInt('chunks submit',
100,
onChange=None)
self._chunks_check = config.getInt('chunks check', 100, onChange=None)
self._chunks_retrieve = config.getInt('chunks retrieve',
100,
onChange=None)
self._job_timeout = config.getTime('queue timeout', -1, onChange=None)
self._job_retries = config.getInt('max retry', -1, onChange=None)
selected = JobSelector.create(config.get('selected', '',
onChange=None),
task=self._task)
self.jobDB = config.getPlugin('job database',
'JobDB',
cls=JobDB,
pargs=(self.getMaxJobs(self._task),
selected))
self._disabled_jobs_logfile = config.getWorkPath('disabled')
self._outputProcessor = config.getPlugin('output processor',
'SandboxProcessor',
cls=TaskOutputProcessor,
pargs=(task, ))
self._do_shuffle = config.getBool('shuffle', False, onChange=None)
self._reportClass = Report.getClass(
config.get('abort report', 'LocationReport', onChange=None))
self._showBlocker = True
def __init__(self, jobDB, task, jobs = None, configString = ''): Report.__init__(self, jobDB, task, jobs, configString) self._selector = JobSelector.create(configString, task = task)
def __init__(self, jobDB, task, jobs=None, configString=''):
Report.__init__(self, jobDB, task, jobs, configString)
self._selector = JobSelector.create(configString, task=task)
def __init__(self, jobDB, task, jobs=None, configString=''):
Report.__init__(self, jobDB, task, jobs, configString)
self._dollar_per_hour = 0.013
if configString:
self._dollar_per_hour = float(configString)
def __init__(self, jobDB, task, jobs=None, configString=''):
Report.__init__(self, jobDB, task, jobs, configString)
self._bar = BasicProgressBar(0, len(jobDB), 65)
def __init__(self, jobDB, task, jobs = None, configString = ''): Report.__init__(self, jobDB, task, jobs, configString) self._dollar_per_hour = 0.013 if configString: self._dollar_per_hour = float(configString)
def __init__(self, jobDB, task, jobs = None, configString = ''): Report.__init__(self, jobDB, task, jobs, configString) self._bar = JobProgressBar(len(jobDB), 65, jobsOnFinish = True)
def __init__(self, jobDB, task, jobs=None, configString=''):
Report.__init__(self, jobDB, task, jobs, configString)
self._bar = JobProgressBar(len(jobDB), 65, jobsOnFinish=True)