class Propagator(AgentBase):
# constructor
def __init__(self, communicator, queue_config_mapper, single_mode=False):
AgentBase.__init__(self, single_mode)
self.dbProxy = DBProxy()
self.communicator = communicator
self.queueConfigMapper = queue_config_mapper
self._last_stats_update = None
self._last_metrics_update = None
# main loop
def run(self):
while True:
sw_main = core_utils.get_stopwatch()
mainLog = self.make_logger(_logger, 'id={0}'.format(self.get_pid()), method_name='run')
mainLog.debug('getting jobs to propagate')
sw = core_utils.get_stopwatch()
jobSpecs = self.dbProxy.get_jobs_to_propagate(harvester_config.propagator.maxJobs,
harvester_config.propagator.lockInterval,
harvester_config.propagator.updateInterval,
self.get_pid())
mainLog.debug('got {0} jobs {1}'.format(len(jobSpecs), sw.get_elapsed_time()))
# update jobs in central database
iJobs = 0
nJobs = harvester_config.propagator.nJobsInBulk
hbSuppressMap = dict()
while iJobs < len(jobSpecs):
jobList = jobSpecs[iJobs:iJobs + nJobs]
iJobs += nJobs
# collect jobs to update or check
jobListToSkip = []
jobListToUpdate = []
jobListToCheck = []
retList = []
for tmpJobSpec in jobList:
if tmpJobSpec.computingSite not in hbSuppressMap:
queueConfig = self.queueConfigMapper.get_queue(tmpJobSpec.computingSite,
tmpJobSpec.configID)
hbSuppressMap[tmpJobSpec.computingSite] = queueConfig.get_no_heartbeat_status()
# heartbeat is suppressed
if tmpJobSpec.get_status() in hbSuppressMap[tmpJobSpec.computingSite] and \
not tmpJobSpec.not_suppress_heartbeat():
# check running job to detect lost heartbeat
if tmpJobSpec.status == 'running':
jobListToCheck.append(tmpJobSpec)
else:
jobListToSkip.append(tmpJobSpec)
retList.append({'StatusCode': 0, 'command': None})
else:
jobListToUpdate.append(tmpJobSpec)
sw.reset()
retList += self.communicator.check_jobs(jobListToCheck)
mainLog.debug('check_jobs for {0} jobs {1}'.format(len(jobListToCheck), sw.get_elapsed_time()))
sw.reset()
retList += self.communicator.update_jobs(jobListToUpdate, self.get_pid())
mainLog.debug('update_jobs for {0} jobs took {1}'.format(len(jobListToUpdate),
sw.get_elapsed_time()))
# logging
for tmpJobSpec, tmpRet in zip(jobListToSkip+jobListToCheck+jobListToUpdate, retList):
if tmpRet['StatusCode'] == 0:
if tmpJobSpec in jobListToUpdate:
mainLog.debug('updated PandaID={0} status={1}'.format(tmpJobSpec.PandaID,
tmpJobSpec.status))
else:
mainLog.debug('skip updating PandaID={0} status={1}'.format(tmpJobSpec.PandaID,
tmpJobSpec.status))
# release job
tmpJobSpec.propagatorLock = None
if tmpJobSpec.is_final_status() and tmpJobSpec.status == tmpJobSpec.get_status():
# unset to disable further updating
tmpJobSpec.propagatorTime = None
tmpJobSpec.subStatus = 'done'
tmpJobSpec.modificationTime = datetime.datetime.utcnow()
elif tmpJobSpec.is_final_status() and not tmpJobSpec.all_events_done():
# trigger next propagation to update remaining events
tmpJobSpec.trigger_propagation()
else:
# check event availability
if tmpJobSpec.status == 'starting' and 'eventService' in tmpJobSpec.jobParams and \
tmpJobSpec.subStatus != 'submitted':
tmpEvStat, tmpEvRet = self.communicator.check_event_availability(tmpJobSpec)
if tmpEvStat:
if tmpEvRet is not None:
tmpJobSpec.nRemainingEvents = tmpEvRet
if tmpEvRet == 0:
mainLog.debug('kill PandaID={0} due to no event'.format(tmpJobSpec.PandaID))
tmpRet['command'] = 'tobekilled'
# got kill command
if 'command' in tmpRet and tmpRet['command'] in ['tobekilled']:
nWorkers = self.dbProxy.kill_workers_with_job(tmpJobSpec.PandaID)
if nWorkers == 0:
# no workers
tmpJobSpec.status = 'cancelled'
tmpJobSpec.subStatus = 'killed'
tmpJobSpec.set_pilot_error(PilotErrors.ERR_PANDAKILL,
PilotErrors.pilotError[PilotErrors.ERR_PANDAKILL])
tmpJobSpec.stateChangeTime = datetime.datetime.utcnow()
tmpJobSpec.trigger_propagation()
self.dbProxy.update_job(tmpJobSpec, {'propagatorLock': self.get_pid()})
else:
mainLog.error('failed to update PandaID={0} status={1}'.format(tmpJobSpec.PandaID,
tmpJobSpec.status))
mainLog.debug('getting workers to propagate')
sw.reset()
workSpecs = self.dbProxy.get_workers_to_propagate(harvester_config.propagator.maxWorkers,
harvester_config.propagator.updateInterval)
mainLog.debug('got {0} workers {1}'.format(len(workSpecs), sw.get_elapsed_time()))
# update workers in central database
sw.reset()
iWorkers = 0
nWorkers = harvester_config.propagator.nWorkersInBulk
while iWorkers < len(workSpecs):
workList = workSpecs[iWorkers:iWorkers + nWorkers]
iWorkers += nWorkers
retList, tmpErrStr = self.communicator.update_workers(workList)
# logging
if retList is None:
mainLog.error('failed to update workers with {0}'.format(tmpErrStr))
else:
for tmpWorkSpec, tmpRet in zip(workList, retList):
if tmpRet:
mainLog.debug('updated workerID={0} status={1}'.format(tmpWorkSpec.workerID,
tmpWorkSpec.status))
# update logs
for logFilePath, logOffset, logSize, logRemoteName in \
tmpWorkSpec.get_log_files_to_upload():
with open(logFilePath, 'rb') as logFileObj:
tmpStat, tmpErr = self.communicator.upload_file(logRemoteName, logFileObj,
logOffset, logSize)
if tmpStat:
tmpWorkSpec.update_log_files_to_upload(logFilePath, logOffset+logSize)
# disable further update
if tmpWorkSpec.is_final_status():
tmpWorkSpec.disable_propagation()
self.dbProxy.update_worker(tmpWorkSpec, {'workerID': tmpWorkSpec.workerID})
else:
mainLog.error('failed to update workerID={0} status={1}'.format(tmpWorkSpec.workerID,
tmpWorkSpec.status))
mainLog.debug('update_workers for {0} workers took {1}'.format(iWorkers,
sw.get_elapsed_time()))
mainLog.debug('getting commands')
commandSpecs = self.dbProxy.get_commands_for_receiver('propagator')
mainLog.debug('got {0} commands'.format(len(commandSpecs)))
for commandSpec in commandSpecs:
if commandSpec.command.startswith(CommandSpec.COM_reportWorkerStats):
# get worker stats
siteName = commandSpec.command.split(':')[-1]
workerStats = self.dbProxy.get_worker_stats(siteName)
if len(workerStats) == 0:
mainLog.error('failed to get worker stats for {0}'.format(siteName))
else:
# report worker stats
tmpRet, tmpStr = self.communicator.update_worker_stats(siteName, workerStats)
if tmpRet:
mainLog.debug('updated worker stats (command) for {0}'.format(siteName))
else:
mainLog.error('failed to update worker stats (command) for {0} err={1}'.format(siteName,
tmpStr))
if not self._last_stats_update or time.time() - self._last_stats_update > STATS_PERIOD:
# get active UPS queues. PanDA server needs to know about them and which harvester instance is taking
# care of them
active_ups_queues = self.queueConfigMapper.get_active_ups_queues()
# update worker stats for all sites
worker_stats_bulk = self.dbProxy.get_worker_stats_bulk(active_ups_queues)
if not worker_stats_bulk:
mainLog.error('failed to get worker stats in bulk')
else:
for site_name in worker_stats_bulk:
tmp_ret, tmp_str = self.communicator.update_worker_stats(site_name,
worker_stats_bulk[site_name])
if tmp_ret:
mainLog.debug('update of worker stats (bulk) for {0}'.format(site_name))
self._last_stats_update = time.time()
else:
mainLog.error('failed to update worker stats (bulk) for {0} err={1}'.format(site_name,
tmp_str))
if not self._last_metrics_update \
or datetime.datetime.utcnow() - self._last_metrics_update > datetime.timedelta(seconds=METRICS_PERIOD):
# get latest metrics from DB
service_metrics_list = self.dbProxy.get_service_metrics(self._last_metrics_update)
if not service_metrics_list:
mainLog.error('failed to get service metrics')
self._last_metrics_update = datetime.datetime.utcnow()
else:
tmp_ret, tmp_str = self.communicator.update_service_metrics(service_metrics_list)
if tmp_ret:
mainLog.debug('update of service metrics OK')
self._last_metrics_update = datetime.datetime.utcnow()
else:
mainLog.error('failed to update service metrics err={0}'.format(tmp_str))
# send dialog messages
mainLog.debug('getting dialog messages to propagate')
try:
maxDialogs = harvester_config.propagator.maxDialogs
except Exception:
maxDialogs = 50
diagSpecs = self.dbProxy.get_dialog_messages_to_send(maxDialogs,
harvester_config.propagator.lockInterval)
mainLog.debug('got {0} dialogs'.format(len(diagSpecs)))
if len(diagSpecs) > 0:
tmpStat, tmpStr = self.communicator.send_dialog_messages(diagSpecs)
if tmpStat:
diagIDs = [diagSpec.diagID for diagSpec in diagSpecs]
self.dbProxy.delete_dialog_messages(diagIDs)
mainLog.debug('sent {0} dialogs'.format(len(diagSpecs)))
else:
mainLog.error('failed to send dialogs err={0}'.format(tmpStr))
if sw_main.get_elapsed_time_in_sec() > harvester_config.propagator.lockInterval:
mainLog.warning('a single cycle was longer than lockInterval. done' + sw_main.get_elapsed_time())
else:
mainLog.debug('done' + sw_main.get_elapsed_time())
# check if being terminated
if self.terminated(harvester_config.propagator.sleepTime):
mainLog.debug('terminated')
return
class Propagator(AgentBase):
# constructor
def __init__(self, communicator, queue_config_mapper, single_mode=False):
AgentBase.__init__(self, single_mode)
self.dbProxy = DBProxy()
self.communicator = communicator
self.queueConfigMapper = queue_config_mapper
self._last_stats_update = None
# main loop
def run(self):
while True:
sw = core_utils.get_stopwatch()
mainLog = core_utils.make_logger(_logger,
'id={0}'.format(self.ident),
method_name='run')
mainLog.debug('getting jobs to propagate')
jobSpecs = self.dbProxy.get_jobs_to_propagate(
harvester_config.propagator.maxJobs,
harvester_config.propagator.lockInterval,
harvester_config.propagator.updateInterval, self.ident)
mainLog.debug('got {0} jobs'.format(len(jobSpecs)))
# update jobs in central database
iJobs = 0
nJobs = harvester_config.propagator.nJobsInBulk
hbSuppressMap = dict()
while iJobs < len(jobSpecs):
jobList = jobSpecs[iJobs:iJobs + nJobs]
iJobs += nJobs
# collect jobs to update or check
jobListToSkip = []
jobListToUpdate = []
jobListToCheck = []
retList = []
for tmpJobSpec in jobList:
if tmpJobSpec.computingSite not in hbSuppressMap:
queueConfig = self.queueConfigMapper.get_queue(
tmpJobSpec.computingSite)
hbSuppressMap[
tmpJobSpec.
computingSite] = queueConfig.get_no_heartbeat_status(
)
# heartbeat is suppressed
if tmpJobSpec.status in hbSuppressMap[
tmpJobSpec.computingSite]:
# check running job to detect lost heartbeat
if tmpJobSpec.status == 'running':
jobListToCheck.append(tmpJobSpec)
else:
jobListToSkip.append(tmpJobSpec)
retList.append({'StatusCode': 0, 'command': None})
else:
jobListToUpdate.append(tmpJobSpec)
retList += self.communicator.check_jobs(jobListToCheck)
retList += self.communicator.update_jobs(jobListToUpdate)
# logging
for tmpJobSpec, tmpRet in zip(
jobListToSkip + jobListToCheck + jobListToUpdate,
retList):
if tmpRet['StatusCode'] == 0:
if tmpJobSpec in jobListToUpdate:
mainLog.debug(
'updated PandaID={0} status={1}'.format(
tmpJobSpec.PandaID, tmpJobSpec.status))
else:
mainLog.debug(
'skip updating PandaID={0} status={1}'.format(
tmpJobSpec.PandaID, tmpJobSpec.status))
# release job
tmpJobSpec.propagatorLock = None
if tmpJobSpec.is_final_status(
) and tmpJobSpec.status == tmpJobSpec.get_status():
# unset to disable further updating
tmpJobSpec.propagatorTime = None
tmpJobSpec.subStatus = 'done'
else:
# check event availability
if tmpJobSpec.status == 'starting' and 'eventService' in tmpJobSpec.jobParams and \
tmpJobSpec.subStatus != 'submitted':
tmpEvStat, tmpEvRet = self.communicator.check_event_availability(
tmpJobSpec)
if tmpEvStat and tmpEvRet == 0:
mainLog.debug(
'kill PandaID={0} due to no event'.
format(tmpJobSpec.PandaID))
tmpRet['command'] = 'tobekilled'
# got kill command
if 'command' in tmpRet and tmpRet['command'] in [
'tobekilled'
]:
nWorkers = self.dbProxy.kill_workers_with_job(
tmpJobSpec.PandaID)
if nWorkers == 0:
# no remaining workers
tmpJobSpec.status = 'cancelled'
tmpJobSpec.subStatus = 'killed'
tmpJobSpec.stateChangeTime = datetime.datetime.utcnow(
)
tmpJobSpec.trigger_propagation()
self.dbProxy.update_job(tmpJobSpec,
{'propagatorLock': self.ident})
else:
mainLog.error(
'failed to update PandaID={0} status={1}'.format(
tmpJobSpec.PandaID, tmpJobSpec.status))
mainLog.debug('getting workers to propagate')
workSpecs = self.dbProxy.get_workers_to_propagate(
harvester_config.propagator.maxWorkers,
harvester_config.propagator.updateInterval)
mainLog.debug('got {0} workers'.format(len(workSpecs)))
# update workers in central database
iWorkers = 0
nWorkers = harvester_config.propagator.nWorkersInBulk
while iWorkers < len(workSpecs):
workList = workSpecs[iWorkers:iWorkers + nJobs]
iWorkers += nWorkers
retList, tmpErrStr = self.communicator.update_workers(workList)
# logging
if retList is None:
mainLog.error(
'failed to update workers with {0}'.format(tmpErrStr))
else:
for tmpWorkSpec, tmpRet in zip(workList, retList):
if tmpRet:
mainLog.debug(
'updated workerID={0} status={1}'.format(
tmpWorkSpec.workerID, tmpWorkSpec.status))
# update logs
for logFilePath, logOffset, logSize, logRemoteName in \
tmpWorkSpec.get_log_files_to_upload():
with open(logFilePath, 'rb') as logFileObj:
tmpStat, tmpErr = self.communicator.upload_file(
logRemoteName, logFileObj, logOffset,
logSize)
if tmpStat:
tmpWorkSpec.update_log_files_to_upload(
logFilePath, logOffset + logSize)
# disable further update
if tmpWorkSpec.is_final_status():
tmpWorkSpec.disable_propagation()
self.dbProxy.update_worker(
tmpWorkSpec,
{'workerID': tmpWorkSpec.workerID})
else:
mainLog.error(
'failed to update workerID={0} status={1}'.
format(tmpWorkSpec.workerID,
tmpWorkSpec.status))
mainLog.debug('getting commands')
commandSpecs = self.dbProxy.get_commands_for_receiver('propagator')
mainLog.debug('got {0} commands'.format(len(commandSpecs)))
for commandSpec in commandSpecs:
if commandSpec.command.startswith(
CommandSpec.COM_reportWorkerStats):
# get worker stats
siteName = commandSpec.command.split(':')[-1]
workerStats = self.dbProxy.get_worker_stats(siteName)
if len(workerStats) == 0:
mainLog.error(
'failed to get worker stats for {0}'.format(
siteName))
else:
# report worker stats
tmpRet, tmpStr = self.communicator.update_worker_stats(
siteName, workerStats)
if tmpRet:
mainLog.debug(
'updated worker stats (command) for {0}'.
format(siteName))
else:
mainLog.error(
'failed to update worker stats (command) for {0} err={1}'
.format(siteName, tmpStr))
if not self._last_stats_update or time.time(
) - self._last_stats_update > STATS_PERIOD:
# update worker stats for all sites
worker_stats_bulk = self.dbProxy.get_worker_stats_bulk()
if not worker_stats_bulk:
mainLog.error('failed to get worker stats in bulk')
else:
for site_name in worker_stats_bulk:
tmp_ret, tmp_str = self.communicator.update_worker_stats(
site_name, worker_stats_bulk[site_name])
if tmp_ret:
mainLog.debug(
'update of worker stats (bulk) for {0}'.format(
site_name))
self._last_stats_update = time.time()
else:
mainLog.error(
'failed to update worker stats (bulk) for {0} err={1}'
.format(site_name, tmp_str))
mainLog.debug('done' + sw.get_elapsed_time())
# check if being terminated
if self.terminated(harvester_config.propagator.sleepTime):
mainLog.debug('terminated')
return