class WorkerAdjuster(object):
# constructor
def __init__(self, queue_config_mapper):
self.queue_configMapper = queue_config_mapper
self.pluginFactory = PluginFactory()
self.dbProxy = DBProxy()
self.throttlerMap = dict()
self.apf_mon = Apfmon(self.queue_configMapper)
try:
self.maxNewWorkers = harvester_config.submitter.maxNewWorkers
except AttributeError:
self.maxNewWorkers = None
# define number of workers to submit based on various information
def define_num_workers(self, static_num_workers, site_name):
tmp_log = core_utils.make_logger(_logger,
'site={0}'.format(site_name),
method_name='define_num_workers')
tmp_log.debug('start')
tmp_log.debug('static_num_workers: {0}'.format(static_num_workers))
dyn_num_workers = copy.deepcopy(static_num_workers)
try:
# get queue status
queue_stat = self.dbProxy.get_cache("panda_queues.json", None)
if queue_stat is None:
queue_stat = dict()
else:
queue_stat = queue_stat.data
# get job statistics
job_stats = self.dbProxy.get_cache("job_statistics.json", None)
if job_stats is None:
job_stats = dict()
else:
job_stats = job_stats.data
# define num of new workers
for queue_name in static_num_workers:
# get queue
queue_config = self.queue_configMapper.get_queue(queue_name)
worker_limits_dict = self.dbProxy.get_worker_limits(queue_name)
max_workers = worker_limits_dict.get('maxWorkers', 0)
n_queue_limit = worker_limits_dict.get('nQueueLimitWorker', 0)
n_queue_limit_per_rt = worker_limits_dict[
'nQueueLimitWorkerPerRT']
n_queue_total, n_ready_total, n_running_total = 0, 0, 0
apf_msg = None
apf_data = None
for job_type, jt_values in iteritems(
static_num_workers[queue_name]):
for resource_type, tmp_val in iteritems(jt_values):
tmp_log.debug(
'Processing queue {0} job_type {1} resource_type {2} with static_num_workers {3}'
.format(queue_name, job_type, resource_type,
tmp_val))
# set 0 to num of new workers when the queue is disabled
if queue_name in queue_stat and queue_stat[queue_name][
'status'] in [
'offline', 'standby', 'maintenance'
]:
dyn_num_workers[queue_name][job_type][
resource_type]['nNewWorkers'] = 0
ret_msg = 'set n_new_workers=0 since status={0}'.format(
queue_stat[queue_name]['status'])
tmp_log.debug(ret_msg)
apf_msg = 'Not submitting workers since queue status = {0}'.format(
queue_stat[queue_name]['status'])
continue
# protection against not-up-to-date queue config
if queue_config is None:
dyn_num_workers[queue_name][job_type][
resource_type]['nNewWorkers'] = 0
ret_msg = 'set n_new_workers=0 due to missing queue_config'
tmp_log.debug(ret_msg)
apf_msg = 'Not submitting workers because of missing queue_config'
continue
# get throttler
if queue_name not in self.throttlerMap:
if hasattr(queue_config, 'throttler'):
throttler = self.pluginFactory.get_plugin(
queue_config.throttler)
else:
throttler = None
self.throttlerMap[queue_name] = throttler
# check throttler
throttler = self.throttlerMap[queue_name]
if throttler is not None:
to_throttle, tmp_msg = throttler.to_be_throttled(
queue_config)
if to_throttle:
dyn_num_workers[queue_name][job_type][
resource_type]['nNewWorkers'] = 0
ret_msg = 'set n_new_workers=0 by {0}:{1}'.format(
throttler.__class__.__name__, tmp_msg)
tmp_log.debug(ret_msg)
continue
# check stats
n_queue = tmp_val['nQueue']
n_ready = tmp_val['nReady']
n_running = tmp_val['nRunning']
if resource_type != 'ANY' and job_type != 'ANY' and job_type is not None:
n_queue_total += n_queue
n_ready_total += n_ready
n_running_total += n_running
if queue_config.runMode == 'slave':
n_new_workers_def = tmp_val['nNewWorkers']
if n_new_workers_def == 0:
dyn_num_workers[queue_name][job_type][
resource_type]['nNewWorkers'] = 0
ret_msg = 'set n_new_workers=0 by panda in slave mode'
tmp_log.debug(ret_msg)
continue
else:
n_new_workers_def = None
# define num of new workers based on static site config
n_new_workers = 0
if n_queue >= n_queue_limit_per_rt > 0:
# enough queued workers
ret_msg = 'No n_new_workers since n_queue({0})>=n_queue_limit_per_rt({1})'.format(
n_queue, n_queue_limit_per_rt)
tmp_log.debug(ret_msg)
pass
elif (n_queue + n_ready +
n_running) >= max_workers > 0:
# enough workers in the system
ret_msg = 'No n_new_workers since n_queue({0}) + n_ready({1}) + n_running({2}) '.format(
n_queue, n_ready, n_running)
ret_msg += '>= max_workers({0})'.format(
max_workers)
tmp_log.debug(ret_msg)
pass
else:
max_queued_workers = None
if n_queue_limit_per_rt > 0: # there is a limit set for the queue
max_queued_workers = n_queue_limit_per_rt
# Reset the maxQueueWorkers according to particular
if n_new_workers_def is not None: # don't surpass limits given centrally
maxQueuedWorkers_slave = n_new_workers_def + n_queue
if max_queued_workers is not None:
max_queued_workers = min(
maxQueuedWorkers_slave,
max_queued_workers)
else:
max_queued_workers = maxQueuedWorkers_slave
elif queue_config.mapType == 'NoJob': # for pull mode, limit to activated jobs
# limit the queue to the number of activated jobs to avoid empty pilots
try:
n_activated = max(
job_stats[queue_name]['activated'],
1) # avoid no activity queues
queue_limit = max_queued_workers
max_queued_workers = min(
n_activated, max_queued_workers)
tmp_log.debug(
'limiting max_queued_workers to min(n_activated={0}, queue_limit={1})'
.format(n_activated, queue_limit))
except KeyError:
tmp_log.warning(
'n_activated not defined, defaulting to configured queue limits'
)
pass
if max_queued_workers is None: # no value found, use default value
max_queued_workers = 1
# new workers
n_new_workers = max(max_queued_workers - n_queue,
0)
tmp_log.debug(
'setting n_new_workers to {0} in max_queued_workers calculation'
.format(n_new_workers))
if max_workers > 0:
n_new_workers = min(
n_new_workers,
max(
max_workers - n_queue - n_ready -
n_running, 0))
tmp_log.debug(
'setting n_new_workers to {0} to respect max_workers'
.format(n_new_workers))
if queue_config.maxNewWorkersPerCycle > 0:
n_new_workers = min(
n_new_workers,
queue_config.maxNewWorkersPerCycle)
tmp_log.debug(
'setting n_new_workers to {0} in order to respect maxNewWorkersPerCycle'
.format(n_new_workers))
if self.maxNewWorkers is not None and self.maxNewWorkers > 0:
n_new_workers = min(n_new_workers,
self.maxNewWorkers)
tmp_log.debug(
'setting n_new_workers to {0} in order to respect universal maxNewWorkers'
.format(n_new_workers))
dyn_num_workers[queue_name][job_type][resource_type][
'nNewWorkers'] = n_new_workers
# adjust n_new_workers for UCORE to let aggregations over RT respect nQueueLimitWorker and max_workers
if queue_config is None:
max_new_workers_per_cycle = 0
ret_msg = 'set max_new_workers_per_cycle=0 in UCORE aggregation due to missing queue_config'
tmp_log.debug(ret_msg)
else:
max_new_workers_per_cycle = queue_config.maxNewWorkersPerCycle
if len(dyn_num_workers[queue_name]) > 1:
total_new_workers_rts = 0
for _jt in dyn_num_workers[queue_name]:
for _rt in dyn_num_workers[queue_name][_jt]:
if _jt != 'ANY' and _rt != 'ANY':
total_new_workers_rts = total_new_workers_rts + dyn_num_workers[
queue_name][_jt][_rt]['nNewWorkers']
n_new_workers_max_agg = min(
max(n_queue_limit - n_queue_total, 0),
max(
max_workers - n_queue_total - n_ready_total -
n_running_total, 0))
if max_new_workers_per_cycle >= 0:
n_new_workers_max_agg = min(n_new_workers_max_agg,
max_new_workers_per_cycle)
if self.maxNewWorkers is not None and self.maxNewWorkers > 0:
n_new_workers_max_agg = min(n_new_workers_max_agg,
self.maxNewWorkers)
# exceeded max, to adjust
if total_new_workers_rts > n_new_workers_max_agg:
if n_new_workers_max_agg == 0:
for job_type in dyn_num_workers[queue_name]:
for resource_type in dyn_num_workers[
queue_name][job_type]:
dyn_num_workers[queue_name][job_type][
resource_type]['nNewWorkers'] = 0
tmp_log.debug(
'No n_new_workers since n_new_workers_max_agg=0 for UCORE'
)
else:
tmp_log.debug(
'n_new_workers_max_agg={0} for UCORE'.format(
n_new_workers_max_agg))
_d = dyn_num_workers[queue_name].copy()
del _d['ANY']
# TODO: needs to be recalculated
simple_rt_nw_list = []
for job_type in _d: # jt: job type
for resource_type in _d[
job_type]: # rt: resource type
simple_rt_nw_list.append([
(resource_type, job_type),
_d[job_type][resource_type].get(
'nNewWorkers', 0), 0
])
_countdown = n_new_workers_max_agg
for _rt_list in simple_rt_nw_list:
(resource_type,
job_type), n_new_workers_orig, _r = _rt_list
n_new_workers, remainder = divmod(
n_new_workers_orig * n_new_workers_max_agg,
total_new_workers_rts)
dyn_num_workers[queue_name][
job_type].setdefault(
resource_type, {
'nReady': 0,
'nRunning': 0,
'nQueue': 0,
'nNewWorkers': 0
})
dyn_num_workers[queue_name][job_type][
resource_type][
'nNewWorkers'] = n_new_workers
_rt_list[2] = remainder
_countdown -= n_new_workers
_s_list = sorted(simple_rt_nw_list,
key=(lambda x: x[1]))
sorted_rt_nw_list = sorted(_s_list,
key=(lambda x: x[2]),
reverse=True)
for (
resource_type, job_type
), n_new_workers_orig, remainder in sorted_rt_nw_list:
if _countdown <= 0:
break
dyn_num_workers[queue_name][job_type][
resource_type]['nNewWorkers'] += 1
_countdown -= 1
for job_type in dyn_num_workers[queue_name]:
for resource_type in dyn_num_workers[queue_name][
job_type]:
if job_type == 'ANY' or resource_type == 'ANY':
continue
n_new_workers = dyn_num_workers[queue_name][
job_type][resource_type]['nNewWorkers']
tmp_log.debug(
'setting n_new_workers to {0} of job_type {1} resource_type {2} in order to respect RT aggregations for UCORE'
.format(n_new_workers, job_type,
resource_type))
if not apf_msg:
apf_data = copy.deepcopy(dyn_num_workers[queue_name])
self.apf_mon.update_label(queue_name, apf_msg, apf_data)
# dump
tmp_log.debug('defined {0}'.format(str(dyn_num_workers)))
return dyn_num_workers
except Exception:
# dump error
err_msg = core_utils.dump_error_message(tmp_log)
return None
class WorkerAdjuster:
# constructor
def __init__(self, queue_config_mapper):
self.queueConfigMapper = queue_config_mapper
self.pluginFactory = PluginFactory()
self.dbProxy = DBProxy()
self.throttlerMap = dict()
# define number of workers to submit based on various information
def define_num_workers(self, static_num_workers, site_name):
tmpLog = core_utils.make_logger(_logger, 'site={0}'.format(site_name), method_name='define_num_workers')
tmpLog.debug('start')
tmpLog.debug('static_num_workers: {0}'.format(static_num_workers))
dyn_num_workers = copy.deepcopy(static_num_workers)
try:
# get queue status
queueStat = self.dbProxy.get_cache("panda_queues.json", None)
if queueStat is None:
queueStat = dict()
else:
queueStat = queueStat.data
# define num of new workers
for queueName in static_num_workers:
for resource_type, tmpVal in iteritems(static_num_workers[queueName]):
tmpLog.debug('Processing queue {0} resource {1} with static_num_workers {2}'.
format(queueName, resource_type, tmpVal))
# set 0 to num of new workers when the queue is disabled
if queueName in queueStat and queueStat[queueName]['status'] in ['offline', 'standby',
'maintenance']:
dyn_num_workers[queueName][resource_type]['nNewWorkers'] = 0
retMsg = 'set nNewWorkers=0 since status={0}'.format(queueStat[queueName]['status'])
tmpLog.debug(retMsg)
continue
# get queue
queueConfig = self.queueConfigMapper.get_queue(queueName)
# get throttler
if queueName not in self.throttlerMap:
if hasattr(queueConfig, 'throttler'):
throttler = self.pluginFactory.get_plugin(queueConfig.throttler)
else:
throttler = None
self.throttlerMap[queueName] = throttler
# check throttler
throttler = self.throttlerMap[queueName]
if throttler is not None:
toThrottle, tmpMsg = throttler.to_be_throttled(queueConfig)
if toThrottle:
dyn_num_workers[queueName][resource_type]['nNewWorkers'] = 0
retMsg = 'set nNewWorkers=0 by {0}:{1}'.format(throttler.__class__.__name__, tmpMsg)
tmpLog.debug(retMsg)
continue
# check stats
nQueue = tmpVal['nQueue']
nReady = tmpVal['nReady']
nRunning = tmpVal['nRunning']
nQueueLimit = queueConfig.nQueueLimitWorker
maxWorkers = queueConfig.maxWorkers
if queueConfig.runMode == 'slave':
nNewWorkersDef = tmpVal['nNewWorkers']
if nNewWorkersDef == 0:
dyn_num_workers[queueName][resource_type]['nNewWorkers'] = 0
retMsg = 'set nNewWorkers=0 by panda in slave mode'
tmpLog.debug(retMsg)
continue
else:
nNewWorkersDef = None
# define num of new workers based on static site config
nNewWorkers = 0
if nQueue >= nQueueLimit > 0:
# enough queued workers
retMsg = 'No nNewWorkers since nQueue({0})>=nQueueLimit({1})'.format(nQueue, nQueueLimit)
tmpLog.debug(retMsg)
pass
elif (nQueue + nReady + nRunning) >= maxWorkers > 0:
# enough workers in the system
retMsg = 'No nNewWorkers since nQueue({0}) + nReady({1}) + nRunning({2}) '.format(nQueue,
nReady,
nRunning)
retMsg += '>= maxWorkers({0})'.format(maxWorkers)
tmpLog.debug(retMsg)
pass
else:
maxQueuedWorkers = None
if nQueueLimit > 0: # there is a limit set for the queue
maxQueuedWorkers = nQueueLimit
if nNewWorkersDef is not None: # don't surpass limits given centrally
maxQueuedWorkers_slave = nNewWorkersDef + nQueue
if maxQueuedWorkers is not None:
maxQueuedWorkers = min(maxQueuedWorkers_slave, maxQueuedWorkers)
else:
maxQueuedWorkers = maxQueuedWorkers_slave
if maxQueuedWorkers is None: # no value found, use default value
maxQueuedWorkers = 1
# new workers
nNewWorkers = max(maxQueuedWorkers - nQueue, 0)
tmpLog.debug('setting nNewWorkers to {0} in maxQueuedWorkers calculation'
.format(nNewWorkers))
if maxWorkers > 0:
nNewWorkers = min(nNewWorkers, max(maxWorkers - nQueue - nReady - nRunning, 0))
tmpLog.debug('setting nNewWorkers to {0} to respect maxWorkers'
.format(nNewWorkers))
if queueConfig.maxNewWorkersPerCycle > 0:
nNewWorkers = min(nNewWorkers, queueConfig.maxNewWorkersPerCycle)
tmpLog.debug('setting nNewWorkers to {0} in order to respect maxNewWorkersPerCycle'
.format(nNewWorkers))
dyn_num_workers[queueName][resource_type]['nNewWorkers'] = nNewWorkers
# dump
tmpLog.debug('defined {0}'.format(str(dyn_num_workers)))
return dyn_num_workers
except:
# dump error
errMsg = core_utils.dump_error_message(tmpLog)
return None
class ARCSubmitter(PluginBase):
'''Submitter for ARC CE'''
def __init__(self, **kwarg):
'''Set up DB connection and credentials'''
PluginBase.__init__(self, **kwarg)
self.dbproxy = DBProxy()
self.schedulerid = harvester_config.master.harvester_id
# Credential dictionary role: proxy file
self.certs = dict(zip([r.split('=')[1] for r in list(harvester_config.credmanager.voms)],
list(harvester_config.credmanager.outCertFile)))
self.cred_type = arc.initializeCredentialsType(arc.initializeCredentialsType.SkipCredentials)
def _run_submit(self, thr):
'''Run a thread to do the submission'''
try:
thr.start()
except:
pass
# Be careful to wait longer than submission timeout
thr.join(thr.userconfig.Timeout() + 60.0)
if thr.isAlive():
# abort due to timeout and try again
raise Exception("Submission timeout")
if thr.job is None:
raise Exception("Submission failed")
return thr.job
def _arc_submit(self, xrsl, arcces, userconfig, log):
'''Check the available CEs and submit'''
queuelist = []
for arcce in arcces:
(ce_endpoint, ce_queue) = arcce
aris = arc.URL(str(ce_endpoint))
ce_host = aris.Host()
if aris.Protocol() == 'https':
aris.ChangePath('/arex')
infoendpoints = [arc.Endpoint(aris.str(),
arc.Endpoint.COMPUTINGINFO,
'org.ogf.glue.emies.resourceinfo')]
else:
aris = 'ldap://'+aris.Host()+'/mds-vo-name=local,o=grid'
infoendpoints = [arc.Endpoint(aris,
arc.Endpoint.COMPUTINGINFO,
'org.nordugrid.ldapng')]
# retriever contains a list of CE endpoints
retriever = arc.ComputingServiceRetriever(userconfig, infoendpoints)
retriever.wait()
# targets is the list of queues
# parse target.ComputingService.ID for the CE hostname
# target.ComputingShare.Name is the queue name
targets = retriever.GetExecutionTargets()
# Filter only sites for this process
for target in targets:
if not target.ComputingService.ID:
log.info("Target {0} does not have ComputingService ID defined, skipping".format(target.ComputingService.Name))
continue
# If EMI-ES infoendpoint, force EMI-ES submission
if infoendpoints[0].InterfaceName == 'org.ogf.glue.emies.resourceinfo' \
and target.ComputingEndpoint.InterfaceName != 'org.ogf.glue.emies.activitycreation':
log.debug("Rejecting target interface {0} because not EMI-ES".format(target.ComputingEndpoint.InterfaceName))
continue
# Check for matching host and queue
targethost = re.sub(':arex$', '', re.sub('urn:ogf:ComputingService:', '', target.ComputingService.ID))
targetqueue = target.ComputingShare.Name
if targethost != ce_host:
log.debug('Rejecting target host {0} as it does not match {1}'.format(targethost, ce_host))
continue
if targetqueue != ce_queue:
log.debug('Rejecting target queue {0} as it does not match {1}'.format(targetqueue, ce_queue))
continue
queuelist.append(target)
log.debug("Adding target {0}:{1}".format(targethost, targetqueue))
# check if any queues are available, if not leave and try again next time
if not queuelist:
raise Exception("No free queues available")
log.debug("preparing submission")
jobdescs = arc.JobDescriptionList()
if not arc.JobDescription_Parse(str(xrsl), jobdescs):
raise Exception("Failed to prepare job description")
# Run the submission in a separate thread
thr = SubmitThr(queuelist, jobdescs, userconfig)
return self._run_submit(thr)
def _set_logdir(self, site):
date = time.strftime('%Y-%m-%d')
return os.path.join(date, site)
# submit workers
def submit_workers(self, workspec_list):
retlist = []
# Get queue info from DB
pandaqueues = self.dbproxy.get_cache("panda_queues.json", None)
if pandaqueues is None:
raise Exception("Failed to get panda queue info from database")
pandaqueues = pandaqueues.data
osmap = self.dbproxy.get_cache("ddmendpoints_objectstores.json", None)
if osmap is None:
raise Exception("Failed to get Object Store info from database")
osmap = osmap.data
for workspec in workspec_list:
arclog = arc_utils.ARCLogger(baselogger, workspec.workerID)
tmplog = arclog.log
# Assume for aCT that jobs are always pre-fetched (no late-binding)
for jobspec in workspec.get_jobspec_list():
tmplog.debug("JobSpec: {0}".format(jobspec.values_map()))
if jobspec.computingSite not in pandaqueues:
retlist.append((False, "No queue information for {0}".format(jobspec.computingSite)))
continue
# Get CEs from panda queue info
# List of (endpoint, queue) tuples
arcces = []
for endpoint in pandaqueues[jobspec.computingSite]['queues']:
ce_endpoint = endpoint['ce_endpoint']
if not re.search('://', ce_endpoint):
ce_endpoint = 'gsiftp://%s' % ce_endpoint
ce_queue = endpoint['ce_queue_name']
arcces.append((ce_endpoint, ce_queue))
if not arcces:
retlist.append((False, "No CEs defined for %{0}".format(jobspec.computingSite)))
continue
# Set true pilot or not
queueconfigmapper = QueueConfigMapper()
queueconfig = queueconfigmapper.get_queue(jobspec.computingSite)
pandaqueues[jobspec.computingSite]['truepilot'] = 'running' in queueconfig.noHeartbeat
# Set log URL for GTAG env in job description
logbaseurl = queueconfig.submitter.get('logBaseURL')
logsubdir = self._set_logdir(jobspec.computingSite)
logfileurl = '/'.join([logbaseurl, logsubdir, '%d.out' % jobspec.PandaID]) if logbaseurl else None
tmplog.debug("Converting to ARC XRSL format")
arcxrsl = ARCParser(jobspec.jobParams,
jobspec.computingSite,
pandaqueues[jobspec.computingSite],
logfileurl,
self.schedulerid,
osmap,
'/tmp', # tmpdir, TODO common tmp dir
None, #jobSpec.eventranges, # TODO event ranges
tmplog)
arcxrsl.parse()
xrsl = arcxrsl.getXrsl()
tmplog.debug("ARC xrsl: {0}".format(xrsl))
# Set the files to be downloaded at the end of the job
downloadfiles = 'gmlog/errors'
if 'logFile' in jobspec.jobParams:
downloadfiles += ';%s' %jobspec.jobParams['logFile'].replace('.tgz', '')
if not pandaqueues[jobspec.computingSite]['truepilot']:
downloadfiles += ';jobSmallFiles.tgz'
# Set certificate
userconfig = arc.UserConfig(self.cred_type)
proxyrole = ''
if jobspec.jobParams['prodSourceLabel'] == 'user':
userconfig.ProxyPath(str(self.certs['pilot']))
proxyrole = 'pilot'
else:
userconfig.ProxyPath(str(self.certs['production']))
proxyrole = 'production'
tmplog.debug("Submitting using {0} proxy at {1}".format(proxyrole, userconfig.ProxyPath()))
try:
tmplog.debug("Submission targets: {0}".format(arcces))
arcjob = self._arc_submit(xrsl, arcces, userconfig, tmplog)
tmplog.info("ARC CE job id {0}".format(arcjob.JobID))
arc_utils.arcjob2workspec(arcjob, workspec)
workspec.workAttributes['arcdownloadfiles'] = downloadfiles
workspec.workAttributes['proxyrole'] = proxyrole
workspec.workAttributes['logsubdir'] = logsubdir
workspec.batchID = arcjob.JobID
tmplog.debug(workspec.workAttributes)
result = (True, '')
except Exception as exc:
tmplog.error(traceback.format_exc())
result = (False, "Failed to submit ARC job: {0}".format(str(exc)))
retlist.append(result)
return retlist
class ARCSubmitter(PluginBase):
'''Submitter for ARC CE'''
def __init__(self, **kwarg):
'''Set up DB connection and credentials'''
PluginBase.__init__(self, **kwarg)
self.dbproxy = DBProxy()
self.schedulerid = harvester_config.master.harvester_id
# Credential dictionary role: proxy file
self.certs = dict(
zip([
r.split('=')[1]
for r in list(harvester_config.credmanager.voms)
], list(harvester_config.credmanager.outCertFile)))
self.cred_type = arc.initializeCredentialsType(
arc.initializeCredentialsType.SkipCredentials)
def _run_submit(self, thr):
'''Run a thread to do the submission'''
try:
thr.start()
except:
pass
# Be careful to wait longer than submission timeout
thr.join(thr.userconfig.Timeout() + 60.0)
if thr.isAlive():
# abort due to timeout and try again
raise Exception("Submission timeout")
if thr.job is None:
raise Exception("Submission failed")
return thr.job
def _arc_submit(self, xrsl, arcces, userconfig, log):
'''Check the available CEs and submit'''
queuelist = []
for arcce in arcces:
(ce_endpoint, ce_queue) = arcce
aris = arc.URL(str(ce_endpoint))
ce_host = aris.Host()
if aris.Protocol() == 'https':
aris.ChangePath('/arex')
infoendpoints = [
arc.Endpoint(aris.str(), arc.Endpoint.COMPUTINGINFO,
'org.ogf.glue.emies.resourceinfo')
]
else:
aris = 'ldap://' + aris.Host() + '/mds-vo-name=local,o=grid'
infoendpoints = [
arc.Endpoint(aris, arc.Endpoint.COMPUTINGINFO,
'org.nordugrid.ldapng')
]
# retriever contains a list of CE endpoints
retriever = arc.ComputingServiceRetriever(userconfig,
infoendpoints)
retriever.wait()
# targets is the list of queues
# parse target.ComputingService.ID for the CE hostname
# target.ComputingShare.Name is the queue name
targets = retriever.GetExecutionTargets()
# Filter only sites for this process
for target in targets:
if not target.ComputingService.ID:
log.info(
"Target {0} does not have ComputingService ID defined, skipping"
.format(target.ComputingService.Name))
continue
# If EMI-ES infoendpoint, force EMI-ES submission
if infoendpoints[0].InterfaceName == 'org.ogf.glue.emies.resourceinfo' \
and target.ComputingEndpoint.InterfaceName != 'org.ogf.glue.emies.activitycreation':
log.debug(
"Rejecting target interface {0} because not EMI-ES".
format(target.ComputingEndpoint.InterfaceName))
continue
# Check for matching host and queue
targethost = re.sub(
':arex$', '',
re.sub('urn:ogf:ComputingService:', '',
target.ComputingService.ID))
targetqueue = target.ComputingShare.Name
if targethost != ce_host:
log.debug(
'Rejecting target host {0} as it does not match {1}'.
format(targethost, ce_host))
continue
if targetqueue != ce_queue:
log.debug(
'Rejecting target queue {0} as it does not match {1}'.
format(targetqueue, ce_queue))
continue
queuelist.append(target)
log.debug("Adding target {0}:{1}".format(
targethost, targetqueue))
# check if any queues are available, if not leave and try again next time
if not queuelist:
raise Exception("No free queues available")
log.debug("preparing submission")
jobdescs = arc.JobDescriptionList()
if not arc.JobDescription_Parse(str(xrsl), jobdescs):
raise Exception("Failed to prepare job description")
# Run the submission in a separate thread
thr = SubmitThr(queuelist, jobdescs, userconfig)
return self._run_submit(thr)
def _set_logdir(self, site):
date = time.strftime('%Y-%m-%d')
return os.path.join(date, site)
# submit workers
def submit_workers(self, workspec_list):
retlist = []
# Get queue info from DB
pandaqueues = self.dbproxy.get_cache("panda_queues.json", None)
if pandaqueues is None:
raise Exception("Failed to get panda queue info from database")
pandaqueues = pandaqueues.data
osmap = self.dbproxy.get_cache("ddmendpoints_objectstores.json", None)
if osmap is None:
raise Exception("Failed to get Object Store info from database")
osmap = osmap.data
for workspec in workspec_list:
arclog = arc_utils.ARCLogger(baselogger, workspec.workerID)
tmplog = arclog.log
# Assume for aCT that jobs are always pre-fetched (no late-binding)
for jobspec in workspec.get_jobspec_list():
tmplog.debug("JobSpec: {0}".format(jobspec.values_map()))
if jobspec.computingSite not in pandaqueues:
retlist.append(
(False, "No queue information for {0}".format(
jobspec.computingSite)))
continue
# Get CEs from panda queue info
# List of (endpoint, queue) tuples
arcces = []
for endpoint in pandaqueues[jobspec.computingSite]['queues']:
ce_endpoint = endpoint['ce_endpoint']
if not re.search('://', ce_endpoint):
ce_endpoint = 'gsiftp://%s' % ce_endpoint
ce_queue = endpoint['ce_queue_name']
arcces.append((ce_endpoint, ce_queue))
if not arcces:
retlist.append((False, "No CEs defined for %{0}".format(
jobspec.computingSite)))
continue
# Set true pilot or not
queueconfigmapper = QueueConfigMapper()
queueconfig = queueconfigmapper.get_queue(
jobspec.computingSite)
pandaqueues[jobspec.computingSite][
'truepilot'] = 'running' in queueconfig.noHeartbeat
# Set log URL for GTAG env in job description
logbaseurl = queueconfig.submitter.get('logBaseURL')
logsubdir = self._set_logdir(jobspec.computingSite)
logfileurl = '/'.join(
[logbaseurl, logsubdir,
'%d.out' % jobspec.PandaID]) if logbaseurl else None
tmplog.debug("Converting to ARC XRSL format")
arcxrsl = ARCParser(
jobspec.jobParams,
jobspec.computingSite,
pandaqueues[jobspec.computingSite],
logfileurl,
self.schedulerid,
osmap,
'/tmp', # tmpdir, TODO common tmp dir
None, #jobSpec.eventranges, # TODO event ranges
tmplog)
arcxrsl.parse()
xrsl = arcxrsl.getXrsl()
tmplog.debug("ARC xrsl: {0}".format(xrsl))
# Set the files to be downloaded at the end of the job
downloadfiles = 'gmlog/errors'
if 'logFile' in jobspec.jobParams:
downloadfiles += ';%s' % jobspec.jobParams[
'logFile'].replace('.tgz', '')
if not pandaqueues[jobspec.computingSite]['truepilot']:
downloadfiles += ';jobSmallFiles.tgz'
# Set certificate
userconfig = arc.UserConfig(self.cred_type)
proxyrole = ''
if jobspec.jobParams['prodSourceLabel'] == 'user':
userconfig.ProxyPath(str(self.certs['pilot']))
proxyrole = 'pilot'
else:
userconfig.ProxyPath(str(self.certs['production']))
proxyrole = 'production'
tmplog.debug("Submitting using {0} proxy at {1}".format(
proxyrole, userconfig.ProxyPath()))
try:
tmplog.debug("Submission targets: {0}".format(arcces))
arcjob = self._arc_submit(xrsl, arcces, userconfig, tmplog)
tmplog.info("ARC CE job id {0}".format(arcjob.JobID))
arc_utils.arcjob2workspec(arcjob, workspec)
workspec.workAttributes['arcdownloadfiles'] = downloadfiles
workspec.workAttributes['proxyrole'] = proxyrole
workspec.workAttributes['logsubdir'] = logsubdir
workspec.batchID = arcjob.JobID
tmplog.debug(workspec.workAttributes)
result = (True, '')
except Exception as exc:
tmplog.error(traceback.format_exc())
result = (False,
"Failed to submit ARC job: {0}".format(str(exc)))
retlist.append(result)
return retlist
class WorkerAdjuster(object):
# constructor
def __init__(self, queue_config_mapper):
self.queueConfigMapper = queue_config_mapper
self.pluginFactory = PluginFactory()
self.dbProxy = DBProxy()
self.throttlerMap = dict()
self.apf_mon = Apfmon(self.queueConfigMapper)
try:
self.maxNewWorkers = harvester_config.submitter.maxNewWorkers
except AttributeError:
self.maxNewWorkers = None
# define number of workers to submit based on various information
def define_num_workers(self, static_num_workers, site_name):
tmpLog = core_utils.make_logger(_logger, 'site={0}'.format(site_name), method_name='define_num_workers')
tmpLog.debug('start')
tmpLog.debug('static_num_workers: {0}'.format(static_num_workers))
dyn_num_workers = copy.deepcopy(static_num_workers)
try:
# get queue status
queueStat = self.dbProxy.get_cache("panda_queues.json", None)
if queueStat is None:
queueStat = dict()
else:
queueStat = queueStat.data
# get job statistics
job_stats = self.dbProxy.get_cache("job_statistics.json", None)
if job_stats is None:
job_stats = dict()
else:
job_stats = job_stats.data
# define num of new workers
for queueName in static_num_workers:
# get queue
queueConfig = self.queueConfigMapper.get_queue(queueName)
workerLimits_dict = self.dbProxy.get_worker_limits(queueName)
maxWorkers = workerLimits_dict.get('maxWorkers', 0)
nQueueLimit = workerLimits_dict.get('nQueueLimitWorker', 0)
nQueueLimitPerRT = workerLimits_dict['nQueueLimitWorkerPerRT']
nQueue_total, nReady_total, nRunning_total = 0, 0, 0
apf_msg = None
apf_data = None
for resource_type, tmpVal in iteritems(static_num_workers[queueName]):
tmpLog.debug('Processing queue {0} resource {1} with static_num_workers {2}'.
format(queueName, resource_type, tmpVal))
# set 0 to num of new workers when the queue is disabled
if queueName in queueStat and queueStat[queueName]['status'] in ['offline', 'standby',
'maintenance']:
dyn_num_workers[queueName][resource_type]['nNewWorkers'] = 0
retMsg = 'set nNewWorkers=0 since status={0}'.format(queueStat[queueName]['status'])
tmpLog.debug(retMsg)
apf_msg = 'Not submitting workers since queue status = {0}'.format(queueStat[queueName]['status'])
continue
# protection against not-up-to-date queue config
if queueConfig is None:
dyn_num_workers[queueName][resource_type]['nNewWorkers'] = 0
retMsg = 'set nNewWorkers=0 due to missing queueConfig'
tmpLog.debug(retMsg)
apf_msg = 'Not submitting workers because of missing queueConfig'
continue
# get throttler
if queueName not in self.throttlerMap:
if hasattr(queueConfig, 'throttler'):
throttler = self.pluginFactory.get_plugin(queueConfig.throttler)
else:
throttler = None
self.throttlerMap[queueName] = throttler
# check throttler
throttler = self.throttlerMap[queueName]
if throttler is not None:
toThrottle, tmpMsg = throttler.to_be_throttled(queueConfig)
if toThrottle:
dyn_num_workers[queueName][resource_type]['nNewWorkers'] = 0
retMsg = 'set nNewWorkers=0 by {0}:{1}'.format(throttler.__class__.__name__, tmpMsg)
tmpLog.debug(retMsg)
continue
# check stats
nQueue = tmpVal['nQueue']
nReady = tmpVal['nReady']
nRunning = tmpVal['nRunning']
if resource_type != 'ANY':
nQueue_total += nQueue
nReady_total += nReady
nRunning_total += nRunning
if queueConfig.runMode == 'slave':
nNewWorkersDef = tmpVal['nNewWorkers']
if nNewWorkersDef == 0:
dyn_num_workers[queueName][resource_type]['nNewWorkers'] = 0
retMsg = 'set nNewWorkers=0 by panda in slave mode'
tmpLog.debug(retMsg)
continue
else:
nNewWorkersDef = None
# define num of new workers based on static site config
nNewWorkers = 0
if nQueue >= nQueueLimitPerRT > 0:
# enough queued workers
retMsg = 'No nNewWorkers since nQueue({0})>=nQueueLimitPerRT({1})'.format(nQueue, nQueueLimitPerRT)
tmpLog.debug(retMsg)
pass
elif (nQueue + nReady + nRunning) >= maxWorkers > 0:
# enough workers in the system
retMsg = 'No nNewWorkers since nQueue({0}) + nReady({1}) + nRunning({2}) '.format(nQueue,
nReady,
nRunning)
retMsg += '>= maxWorkers({0})'.format(maxWorkers)
tmpLog.debug(retMsg)
pass
else:
maxQueuedWorkers = None
if nQueueLimitPerRT > 0: # there is a limit set for the queue
maxQueuedWorkers = nQueueLimitPerRT
# Reset the maxQueueWorkers according to particular
if nNewWorkersDef is not None: # don't surpass limits given centrally
maxQueuedWorkers_slave = nNewWorkersDef + nQueue
if maxQueuedWorkers is not None:
maxQueuedWorkers = min(maxQueuedWorkers_slave, maxQueuedWorkers)
else:
maxQueuedWorkers = maxQueuedWorkers_slave
elif queueConfig.mapType == 'NoJob': # for pull mode, limit to activated jobs
# limit the queue to the number of activated jobs to avoid empty pilots
try:
n_activated = max(job_stats[queueName]['activated'], 1) # avoid no activity queues
queue_limit = maxQueuedWorkers
maxQueuedWorkers = min(n_activated, maxQueuedWorkers)
tmpLog.debug('limiting maxQueuedWorkers to min(n_activated={0}, queue_limit={1})'.
format(n_activated, queue_limit))
except KeyError:
tmpLog.warning('n_activated not defined, defaulting to configured queue limits')
pass
if maxQueuedWorkers is None: # no value found, use default value
maxQueuedWorkers = 1
# new workers
nNewWorkers = max(maxQueuedWorkers - nQueue, 0)
tmpLog.debug('setting nNewWorkers to {0} in maxQueuedWorkers calculation'
.format(nNewWorkers))
if maxWorkers > 0:
nNewWorkers = min(nNewWorkers, max(maxWorkers - nQueue - nReady - nRunning, 0))
tmpLog.debug('setting nNewWorkers to {0} to respect maxWorkers'
.format(nNewWorkers))
if queueConfig.maxNewWorkersPerCycle > 0:
nNewWorkers = min(nNewWorkers, queueConfig.maxNewWorkersPerCycle)
tmpLog.debug('setting nNewWorkers to {0} in order to respect maxNewWorkersPerCycle'
.format(nNewWorkers))
if self.maxNewWorkers is not None and self.maxNewWorkers > 0:
nNewWorkers = min(nNewWorkers, self.maxNewWorkers)
tmpLog.debug('setting nNewWorkers to {0} in order to respect universal maxNewWorkers'
.format(nNewWorkers))
dyn_num_workers[queueName][resource_type]['nNewWorkers'] = nNewWorkers
# adjust nNewWorkers for UCORE to let aggregations over RT respect nQueueLimitWorker and maxWorkers
if queueConfig is None:
maxNewWorkersPerCycle = 0
retMsg = 'set maxNewWorkersPerCycle=0 in UCORE aggregation due to missing queueConfig'
tmpLog.debug(retMsg)
else:
maxNewWorkersPerCycle = queueConfig.maxNewWorkersPerCycle
if len(dyn_num_workers[queueName]) > 1:
total_new_workers_rts = sum( dyn_num_workers[queueName][_rt]['nNewWorkers']
if _rt != 'ANY' else 0
for _rt in dyn_num_workers[queueName] )
nNewWorkers_max_agg = min(
max(nQueueLimit - nQueue_total, 0),
max(maxWorkers - nQueue_total - nReady_total - nRunning_total, 0),
)
if maxNewWorkersPerCycle >= 0:
nNewWorkers_max_agg = min(nNewWorkers_max_agg, maxNewWorkersPerCycle)
if self.maxNewWorkers is not None and self.maxNewWorkers > 0:
nNewWorkers_max_agg = min(nNewWorkers_max_agg, self.maxNewWorkers)
# exceeded max, to adjust
if total_new_workers_rts > nNewWorkers_max_agg:
if nNewWorkers_max_agg == 0:
for resource_type in dyn_num_workers[queueName]:
dyn_num_workers[queueName][resource_type]['nNewWorkers'] = 0
tmpLog.debug('No nNewWorkers since nNewWorkers_max_agg=0 for UCORE')
else:
tmpLog.debug('nNewWorkers_max_agg={0} for UCORE'.format(nNewWorkers_max_agg))
_d = dyn_num_workers[queueName].copy()
del _d['ANY']
simple_rt_nw_list = [ [_rt, _d[_rt].get('nNewWorkers', 0), 0] for _rt in _d ]
_countdown = nNewWorkers_max_agg
for _rt_list in simple_rt_nw_list:
resource_type, nNewWorkers_orig, _r = _rt_list
nNewWorkers, remainder = divmod(nNewWorkers_orig*nNewWorkers_max_agg, total_new_workers_rts)
dyn_num_workers[queueName][resource_type]['nNewWorkers'] = nNewWorkers
_rt_list[2] = remainder
_countdown -= nNewWorkers
_s_list = sorted(simple_rt_nw_list, key=(lambda x: x[1]))
sorted_rt_nw_list = sorted(_s_list, key=(lambda x: x[2]), reverse=True)
for resource_type, nNewWorkers_orig, remainder in sorted_rt_nw_list:
if _countdown <= 0:
break
dyn_num_workers[queueName][resource_type]['nNewWorkers'] += 1
_countdown -= 1
for resource_type in dyn_num_workers[queueName]:
if resource_type == 'ANY':
continue
nNewWorkers = dyn_num_workers[queueName][resource_type]['nNewWorkers']
tmpLog.debug('setting nNewWorkers to {0} of type {1} in order to respect RT aggregations for UCORE'
.format(nNewWorkers, resource_type))
if not apf_msg:
apf_data = copy.deepcopy(dyn_num_workers[queueName])
self.apf_mon.update_label(queueName, apf_msg, apf_data)
# dump
tmpLog.debug('defined {0}'.format(str(dyn_num_workers)))
return dyn_num_workers
except Exception:
# dump error
errMsg = core_utils.dump_error_message(tmpLog)
return None