def HarvesterReport():
log = ''
try:
from distutils.sysconfig import get_python_lib # pylint: disable=import-error
sys.path.append(get_python_lib() + '/pandacommon')
os.environ['PANDA_HOME'] = os.environ['VIRTUAL_ENV']
from collections import defaultdict # pylint: disable=import-error
from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy # pylint: disable=import-error
dbProxy = DBProxy()
workers = dbProxy.get_worker_stats_bulk(None)
rep = defaultdict(dict)
rtot = defaultdict(int)
for site, prodsourcelabels in workers.items():
for prodsourcelabel, resources in prodsourcelabels.items():
for resource, jobs in resources.items():
rep[f'{site}-{resource}'][prodsourcelabel
or 'empty'] = jobs
for state, count in jobs.items():
rtot[state] += count
log = f"All Harvester jobs: {sum(rtot.values())} prodSourceLabel: submitted/running\n"
for k in sorted(rep.keys()):
log += f"{k:>28.28}:"
for psl, jobs in rep[k].items():
log += f"{psl:>10}: {jobs['submitted']}/{jobs['running']}"
log += '\n'
log += f"{'Totals':>28}: submitted: {rtot['submitted']} running: {rtot['running']}\n\n"
except:
pass
return log
class Propagator(AgentBase):
# constructor
def __init__(self, communicator, queue_config_mapper, single_mode=False):
AgentBase.__init__(self, single_mode)
self.dbProxy = DBProxy()
self.communicator = communicator
self.queueConfigMapper = queue_config_mapper
self._last_stats_update = None
self._last_metrics_update = None
# main loop
def run(self):
while True:
sw_main = core_utils.get_stopwatch()
mainLog = self.make_logger(_logger, 'id={0}'.format(self.get_pid()), method_name='run')
mainLog.debug('getting jobs to propagate')
sw = core_utils.get_stopwatch()
jobSpecs = self.dbProxy.get_jobs_to_propagate(harvester_config.propagator.maxJobs,
harvester_config.propagator.lockInterval,
harvester_config.propagator.updateInterval,
self.get_pid())
mainLog.debug('got {0} jobs {1}'.format(len(jobSpecs), sw.get_elapsed_time()))
# update jobs in central database
iJobs = 0
nJobs = harvester_config.propagator.nJobsInBulk
hbSuppressMap = dict()
while iJobs < len(jobSpecs):
jobList = jobSpecs[iJobs:iJobs + nJobs]
iJobs += nJobs
# collect jobs to update or check
jobListToSkip = []
jobListToUpdate = []
jobListToCheck = []
retList = []
for tmpJobSpec in jobList:
if tmpJobSpec.computingSite not in hbSuppressMap:
queueConfig = self.queueConfigMapper.get_queue(tmpJobSpec.computingSite,
tmpJobSpec.configID)
hbSuppressMap[tmpJobSpec.computingSite] = queueConfig.get_no_heartbeat_status()
# heartbeat is suppressed
if tmpJobSpec.get_status() in hbSuppressMap[tmpJobSpec.computingSite] and \
not tmpJobSpec.not_suppress_heartbeat():
# check running job to detect lost heartbeat
if tmpJobSpec.status == 'running':
jobListToCheck.append(tmpJobSpec)
else:
jobListToSkip.append(tmpJobSpec)
retList.append({'StatusCode': 0, 'command': None})
else:
jobListToUpdate.append(tmpJobSpec)
sw.reset()
retList += self.communicator.check_jobs(jobListToCheck)
mainLog.debug('check_jobs for {0} jobs {1}'.format(len(jobListToCheck), sw.get_elapsed_time()))
sw.reset()
retList += self.communicator.update_jobs(jobListToUpdate, self.get_pid())
mainLog.debug('update_jobs for {0} jobs took {1}'.format(len(jobListToUpdate),
sw.get_elapsed_time()))
# logging
for tmpJobSpec, tmpRet in zip(jobListToSkip+jobListToCheck+jobListToUpdate, retList):
if tmpRet['StatusCode'] == 0:
if tmpJobSpec in jobListToUpdate:
mainLog.debug('updated PandaID={0} status={1}'.format(tmpJobSpec.PandaID,
tmpJobSpec.status))
else:
mainLog.debug('skip updating PandaID={0} status={1}'.format(tmpJobSpec.PandaID,
tmpJobSpec.status))
# release job
tmpJobSpec.propagatorLock = None
if tmpJobSpec.is_final_status() and tmpJobSpec.status == tmpJobSpec.get_status():
# unset to disable further updating
tmpJobSpec.propagatorTime = None
tmpJobSpec.subStatus = 'done'
tmpJobSpec.modificationTime = datetime.datetime.utcnow()
elif tmpJobSpec.is_final_status() and not tmpJobSpec.all_events_done():
# trigger next propagation to update remaining events
tmpJobSpec.trigger_propagation()
else:
# check event availability
if tmpJobSpec.status == 'starting' and 'eventService' in tmpJobSpec.jobParams and \
tmpJobSpec.subStatus != 'submitted':
tmpEvStat, tmpEvRet = self.communicator.check_event_availability(tmpJobSpec)
if tmpEvStat:
if tmpEvRet is not None:
tmpJobSpec.nRemainingEvents = tmpEvRet
if tmpEvRet == 0:
mainLog.debug('kill PandaID={0} due to no event'.format(tmpJobSpec.PandaID))
tmpRet['command'] = 'tobekilled'
# got kill command
if 'command' in tmpRet and tmpRet['command'] in ['tobekilled']:
nWorkers = self.dbProxy.kill_workers_with_job(tmpJobSpec.PandaID)
if nWorkers == 0:
# no workers
tmpJobSpec.status = 'cancelled'
tmpJobSpec.subStatus = 'killed'
tmpJobSpec.set_pilot_error(PilotErrors.ERR_PANDAKILL,
PilotErrors.pilotError[PilotErrors.ERR_PANDAKILL])
tmpJobSpec.stateChangeTime = datetime.datetime.utcnow()
tmpJobSpec.trigger_propagation()
self.dbProxy.update_job(tmpJobSpec, {'propagatorLock': self.get_pid()})
else:
mainLog.error('failed to update PandaID={0} status={1}'.format(tmpJobSpec.PandaID,
tmpJobSpec.status))
mainLog.debug('getting workers to propagate')
sw.reset()
workSpecs = self.dbProxy.get_workers_to_propagate(harvester_config.propagator.maxWorkers,
harvester_config.propagator.updateInterval)
mainLog.debug('got {0} workers {1}'.format(len(workSpecs), sw.get_elapsed_time()))
# update workers in central database
sw.reset()
iWorkers = 0
nWorkers = harvester_config.propagator.nWorkersInBulk
while iWorkers < len(workSpecs):
workList = workSpecs[iWorkers:iWorkers + nWorkers]
iWorkers += nWorkers
retList, tmpErrStr = self.communicator.update_workers(workList)
# logging
if retList is None:
mainLog.error('failed to update workers with {0}'.format(tmpErrStr))
else:
for tmpWorkSpec, tmpRet in zip(workList, retList):
if tmpRet:
mainLog.debug('updated workerID={0} status={1}'.format(tmpWorkSpec.workerID,
tmpWorkSpec.status))
# update logs
for logFilePath, logOffset, logSize, logRemoteName in \
tmpWorkSpec.get_log_files_to_upload():
with open(logFilePath, 'rb') as logFileObj:
tmpStat, tmpErr = self.communicator.upload_file(logRemoteName, logFileObj,
logOffset, logSize)
if tmpStat:
tmpWorkSpec.update_log_files_to_upload(logFilePath, logOffset+logSize)
# disable further update
if tmpWorkSpec.is_final_status():
tmpWorkSpec.disable_propagation()
self.dbProxy.update_worker(tmpWorkSpec, {'workerID': tmpWorkSpec.workerID})
else:
mainLog.error('failed to update workerID={0} status={1}'.format(tmpWorkSpec.workerID,
tmpWorkSpec.status))
mainLog.debug('update_workers for {0} workers took {1}'.format(iWorkers,
sw.get_elapsed_time()))
mainLog.debug('getting commands')
commandSpecs = self.dbProxy.get_commands_for_receiver('propagator')
mainLog.debug('got {0} commands'.format(len(commandSpecs)))
for commandSpec in commandSpecs:
if commandSpec.command.startswith(CommandSpec.COM_reportWorkerStats):
# get worker stats
siteName = commandSpec.command.split(':')[-1]
workerStats = self.dbProxy.get_worker_stats(siteName)
if len(workerStats) == 0:
mainLog.error('failed to get worker stats for {0}'.format(siteName))
else:
# report worker stats
tmpRet, tmpStr = self.communicator.update_worker_stats(siteName, workerStats)
if tmpRet:
mainLog.debug('updated worker stats (command) for {0}'.format(siteName))
else:
mainLog.error('failed to update worker stats (command) for {0} err={1}'.format(siteName,
tmpStr))
if not self._last_stats_update or time.time() - self._last_stats_update > STATS_PERIOD:
# get active UPS queues. PanDA server needs to know about them and which harvester instance is taking
# care of them
active_ups_queues = self.queueConfigMapper.get_active_ups_queues()
# update worker stats for all sites
worker_stats_bulk = self.dbProxy.get_worker_stats_bulk(active_ups_queues)
if not worker_stats_bulk:
mainLog.error('failed to get worker stats in bulk')
else:
for site_name in worker_stats_bulk:
tmp_ret, tmp_str = self.communicator.update_worker_stats(site_name,
worker_stats_bulk[site_name])
if tmp_ret:
mainLog.debug('update of worker stats (bulk) for {0}'.format(site_name))
self._last_stats_update = time.time()
else:
mainLog.error('failed to update worker stats (bulk) for {0} err={1}'.format(site_name,
tmp_str))
if not self._last_metrics_update \
or datetime.datetime.utcnow() - self._last_metrics_update > datetime.timedelta(seconds=METRICS_PERIOD):
# get latest metrics from DB
service_metrics_list = self.dbProxy.get_service_metrics(self._last_metrics_update)
if not service_metrics_list:
mainLog.error('failed to get service metrics')
self._last_metrics_update = datetime.datetime.utcnow()
else:
tmp_ret, tmp_str = self.communicator.update_service_metrics(service_metrics_list)
if tmp_ret:
mainLog.debug('update of service metrics OK')
self._last_metrics_update = datetime.datetime.utcnow()
else:
mainLog.error('failed to update service metrics err={0}'.format(tmp_str))
# send dialog messages
mainLog.debug('getting dialog messages to propagate')
try:
maxDialogs = harvester_config.propagator.maxDialogs
except Exception:
maxDialogs = 50
diagSpecs = self.dbProxy.get_dialog_messages_to_send(maxDialogs,
harvester_config.propagator.lockInterval)
mainLog.debug('got {0} dialogs'.format(len(diagSpecs)))
if len(diagSpecs) > 0:
tmpStat, tmpStr = self.communicator.send_dialog_messages(diagSpecs)
if tmpStat:
diagIDs = [diagSpec.diagID for diagSpec in diagSpecs]
self.dbProxy.delete_dialog_messages(diagIDs)
mainLog.debug('sent {0} dialogs'.format(len(diagSpecs)))
else:
mainLog.error('failed to send dialogs err={0}'.format(tmpStr))
if sw_main.get_elapsed_time_in_sec() > harvester_config.propagator.lockInterval:
mainLog.warning('a single cycle was longer than lockInterval. done' + sw_main.get_elapsed_time())
else:
mainLog.debug('done' + sw_main.get_elapsed_time())
# check if being terminated
if self.terminated(harvester_config.propagator.sleepTime):
mainLog.debug('terminated')
return
class aCTReport:
'''Print summary info on jobs in DB. Use --web to print html that is
automatically refreshed. Add filenames to query more than one aCT DB'''
def __init__(self, args):
self.output = ""
self.harvester = args.harvester
self.outfile = args.web
self.actconfs = args.conffiles or [''] # empty string for default behaviour
self.logger=aCTLogger.aCTLogger("aCTReport")
self.actlog=self.logger()
self.actlog.logger.setLevel(logging.ERROR)
self.criticallogger = aCTLogger.aCTLogger('aCTCritical', arclog=False)
self.criticallog = self.criticallogger()
if self.outfile:
self.log('<META HTTP-EQUIV="refresh" CONTENT="60"><pre>')
self.log(time.asctime() + '\n')
self.db=aCTDBArc.aCTDBArc(self.actlog)
self.pandadb=aCTDBPanda.aCTDBPanda(self.actlog)
def log(self, message=''):
self.output += message + '\n'
def ProcessReport(self):
if self.actconfs != ['']:
return # don't print processes for combined report
actprocscmd = 'ps ax -ww -o pid,etime,args'
try:
out = subprocess.run(actprocscmd.split(), check=True, encoding='utf-8', stdout=subprocess.PIPE).stdout
except subprocess.CalledProcessError as e:
self.log('Error: could not run ps command: %s' % e.stderr)
return
# Group processes by cluster
cluster_procs = {}
longprocesses = []
for line in out.split('\n'):
reg = re.match(r'\s*(\d*)\s*(.*) .*python.* .*(aCT\w*)\.py\s?(\S*)', line)
if reg:
pid, runningtime, process, cluster = reg.groups()
# ignore Main and this process
if process in ['aCTReport', 'aCTMain', 'aCTHeartbeatWatchdog']:
continue
if cluster == '':
cluster = '(no cluster defined)'
elif not re.match(r'\d\d:\d\d$', runningtime):
# Check for overrunning processes
longprocesses.append((process, pid, cluster, runningtime))
if cluster in cluster_procs:
cluster_procs[cluster].append(process)
else:
cluster_procs[cluster] = [process]
for proc in longprocesses:
self.log('WARNING: %s (pid %s) for %s running for more than one hour (%s), this process will be killed' % proc)
# Kill process and log a critical message to send email
# Too many emails, disable
#self.criticallog.critical('Killing process %s (pid %s) for %s running for more than one hour (%s)' % proc)
try:
os.kill(int(proc[1]), signal.SIGKILL)
except OSError:
pass
self.log()
self.log('Active processes per cluster:')
for cluster in sorted(cluster_procs):
procs = cluster_procs[cluster]
procs.sort()
self.log(f'{cluster:>38.38}: {" ".join(procs)}')
self.log()
def PandaReport(self):
rep={}
rtot={}
states = ["sent", "starting", "running", "slots", "tovalidate", "toresubmit",
"toclean", "finished", "done", "failed", "donefailed",
"tobekilled", "cancelled", "donecancelled"]
for conf in self.actconfs:
if conf:
os.environ['ACTCONFIGARC'] = conf
db=aCTDBArc.aCTDBArc(self.actlog)
c=db.db.conn.cursor()
c.execute("select sitename, actpandastatus, corecount from pandajobs")
rows=c.fetchall()
for r in rows:
site, state = (str(r[0]), str(r[1]))
if r[2] is None:
corecount=1
else:
corecount=int(r[2])
try:
rep[site][state]+=1
if state == "running":
rep[site]["slots"]+=1*corecount
except:
try:
rep[site][state]=1
if state == "running":
try:
rep[site]["slots"]+=1*corecount
except:
rep[site]["slots"]=corecount
except:
rep[site]={}
rep[site][state]=1
if state == "running":
rep[site]["slots"]=corecount
try:
rtot[state]+=1
if state == "running":
rtot["slots"]+=1*corecount
except:
rtot[state]=1
if state == "running":
rtot["slots"]=corecount
self.log(f"All Panda jobs: {sum([v for k,v in rtot.items() if k != 'slots'])}")
self.log(f"{'':29} {' '.join([f'{s:>9}' for s in states])}")
for k in sorted(rep.keys()):
log=f"{k:>28.28}:"
for s in states:
try:
log += f'{rep[k][s]:>10}'
except KeyError:
log += f'{"-":>10}'
self.log(log)
log = f'{"Totals":>28}:'
for s in states:
try:
log += f'{rtot[s]:>10}'
except:
log += f'{"-":>10}'
self.log(log+'\n\n')
def ArcJobReport(self):
rep={}
rtot={}
states = ["Undefined", "Accepted", "Preparing", "Submitting",
"Queuing", "Running", "Finishing", "Finished", "Hold", "Killed",
"Failed", "Deleted", "Other"]
for conf in self.actconfs:
if conf:
os.environ['ACTCONFIGARC'] = conf
db=aCTDBArc.aCTDBArc(self.actlog)
c=db.db.conn.cursor()
c.execute("select jobid,state from arcjobs")
rows=c.fetchall()
for r in rows:
reg=re.search('.+//([^:]+)',str(r[0]))
cl=""
try:
cl=reg.group(1)
except:
cl='WaitingSubmission'
jid=str(r[1])
if jid == 'None':
jid="Other"
try:
rep[cl][jid]+=1
except:
try:
rep[cl][jid]=1
except:
rep[cl]={}
rep[cl][jid]=1
try:
rtot[jid]+=1
except:
rtot[jid]=1
self.log(f"All ARC jobs: {sum(rtot.values())}")
self.log(f"{'':39} {' '.join([f'{s:>9}' for s in states])}")
for k in sorted(rep, key=lambda x: x.split('.')[-1]):
log=f"{k:>38.38}:"
for s in states:
try:
log += f'{rep[k][s]:>10}'
except KeyError:
log += f'{"-":>10}'
self.log(log)
log = f"{'Totals':>38}:"
for s in states:
try:
log += f'{rtot[s]:>10}'
except:
log += f'{"-":>10}'
self.log(log+'\n\n')
def CondorJobReport(self):
rep = {}
rtot = {}
condorjobstatemap = ['Undefined', # used before real state is known
'Idle',
'Running',
'Removed',
'Completed',
'Held',
'Transferring',
'Suspended']
for conf in self.actconfs:
if conf:
os.environ['ACTCONFIGARC'] = conf
db=aCTDBArc.aCTDBArc(self.actlog)
c = db.db.conn.cursor()
c.execute("select cluster, JobStatus from condorjobs")
rows = c.fetchall()
for r in rows:
cl = str(r[0])
if not cl:
cl = 'WaitingSubmission'
jid = r[1]
try:
rep[cl][jid]+=1
except:
try:
rep[cl][jid]=1
except:
rep[cl]={}
rep[cl][jid]=1
try:
rtot[jid]+=1
except:
rtot[jid]=1
self.log(f"All Condor jobs: {sum(rtot.values())}")
self.log(f"{'':39} {' '.join([f'{s:>9}' for s in condorjobstatemap])}")
for k in sorted(rep, key=lambda x: x.split('.')[-1]):
log=f"{k:>38.38}:"
for s in range(8):
try:
log += f'{rep[k][s]:>10}'
except KeyError:
log += f'{"-":>10}'
self.log(log)
log = f"{'Totals':>38}:"
for s in range(8):
try:
log += f'{rtot[s]:>10}'
except:
log += f'{"-":>10}'
self.log(log+'\n\n')
def StuckReport(self):
# Query for lost jobs older than lostlimit
lostlimit = 86400
select = "(arcstate='submitted' or arcstate='running') and " \
+ self.db.timeStampLessThan("tarcstate", lostlimit) + \
" and sendhb=1 and arcjobs.id=pandajobs.arcjobid order by tarcstate"
columns = ['cluster']
jobs = self.db.getArcJobsInfo(select, columns, tables='arcjobs,pandajobs')
if jobs:
self.log('Found %d jobs not updated in over %d seconds:\n' % (len(jobs), lostlimit))
clustercount = {}
for job in jobs:
try:
host = re.search('.+//([^:]+)', job['cluster']).group(1)
except:
pass
if host in clustercount:
clustercount[host] += 1
else:
clustercount[host] = 1
for cluster, count in clustercount.items():
self.log(f'{count} {cluster}')
self.log()
def HarvesterReport(self):
try:
from distutils.sysconfig import get_python_lib # pylint: disable=import-error
sys.path.append(get_python_lib()+'/pandacommon')
os.environ['PANDA_HOME']=os.environ['VIRTUAL_ENV']
from collections import defaultdict # pylint: disable=import-error
from pandaharvester.harvestercore.db_proxy_pool import DBProxyPool as DBProxy # pylint: disable=import-error
self.dbProxy = DBProxy()
workers = self.dbProxy.get_worker_stats_bulk(None)
rep = defaultdict(dict)
rtot = defaultdict(int)
for site, prodsourcelabels in workers.items():
for prodsourcelabel, resources in prodsourcelabels.items():
for resource, jobs in resources.items():
rep[f'{site}-{resource}'][prodsourcelabel or 'empty'] = jobs
for state, count in jobs.items():
rtot[state] += count
self.log(f"All Harvester jobs: {sum(rtot.values())} prodSourceLabel: submitted/running")
for k in sorted(rep.keys()):
log=f"{k:>28.28}:"
for psl, jobs in rep[k].items():
log += f"{psl:>10}: {jobs['submitted']}/{jobs['running']}"
self.log(log)
log = f"{'Totals':>28}: submitted: {rtot['submitted']} running: {rtot['running']}"
self.log(log+'\n\n')
except:
pass
def end(self):
if self.outfile:
self.log('</pre>')
class Propagator(AgentBase):
# constructor
def __init__(self, communicator, queue_config_mapper, single_mode=False):
AgentBase.__init__(self, single_mode)
self.dbProxy = DBProxy()
self.communicator = communicator
self.queueConfigMapper = queue_config_mapper
self._last_stats_update = None
# main loop
def run(self):
while True:
sw = core_utils.get_stopwatch()
mainLog = core_utils.make_logger(_logger,
'id={0}'.format(self.ident),
method_name='run')
mainLog.debug('getting jobs to propagate')
jobSpecs = self.dbProxy.get_jobs_to_propagate(
harvester_config.propagator.maxJobs,
harvester_config.propagator.lockInterval,
harvester_config.propagator.updateInterval, self.ident)
mainLog.debug('got {0} jobs'.format(len(jobSpecs)))
# update jobs in central database
iJobs = 0
nJobs = harvester_config.propagator.nJobsInBulk
hbSuppressMap = dict()
while iJobs < len(jobSpecs):
jobList = jobSpecs[iJobs:iJobs + nJobs]
iJobs += nJobs
# collect jobs to update or check
jobListToSkip = []
jobListToUpdate = []
jobListToCheck = []
retList = []
for tmpJobSpec in jobList:
if tmpJobSpec.computingSite not in hbSuppressMap:
queueConfig = self.queueConfigMapper.get_queue(
tmpJobSpec.computingSite)
hbSuppressMap[
tmpJobSpec.
computingSite] = queueConfig.get_no_heartbeat_status(
)
# heartbeat is suppressed
if tmpJobSpec.status in hbSuppressMap[
tmpJobSpec.computingSite]:
# check running job to detect lost heartbeat
if tmpJobSpec.status == 'running':
jobListToCheck.append(tmpJobSpec)
else:
jobListToSkip.append(tmpJobSpec)
retList.append({'StatusCode': 0, 'command': None})
else:
jobListToUpdate.append(tmpJobSpec)
retList += self.communicator.check_jobs(jobListToCheck)
retList += self.communicator.update_jobs(jobListToUpdate)
# logging
for tmpJobSpec, tmpRet in zip(
jobListToSkip + jobListToCheck + jobListToUpdate,
retList):
if tmpRet['StatusCode'] == 0:
if tmpJobSpec in jobListToUpdate:
mainLog.debug(
'updated PandaID={0} status={1}'.format(
tmpJobSpec.PandaID, tmpJobSpec.status))
else:
mainLog.debug(
'skip updating PandaID={0} status={1}'.format(
tmpJobSpec.PandaID, tmpJobSpec.status))
# release job
tmpJobSpec.propagatorLock = None
if tmpJobSpec.is_final_status(
) and tmpJobSpec.status == tmpJobSpec.get_status():
# unset to disable further updating
tmpJobSpec.propagatorTime = None
tmpJobSpec.subStatus = 'done'
else:
# check event availability
if tmpJobSpec.status == 'starting' and 'eventService' in tmpJobSpec.jobParams and \
tmpJobSpec.subStatus != 'submitted':
tmpEvStat, tmpEvRet = self.communicator.check_event_availability(
tmpJobSpec)
if tmpEvStat and tmpEvRet == 0:
mainLog.debug(
'kill PandaID={0} due to no event'.
format(tmpJobSpec.PandaID))
tmpRet['command'] = 'tobekilled'
# got kill command
if 'command' in tmpRet and tmpRet['command'] in [
'tobekilled'
]:
nWorkers = self.dbProxy.kill_workers_with_job(
tmpJobSpec.PandaID)
if nWorkers == 0:
# no remaining workers
tmpJobSpec.status = 'cancelled'
tmpJobSpec.subStatus = 'killed'
tmpJobSpec.stateChangeTime = datetime.datetime.utcnow(
)
tmpJobSpec.trigger_propagation()
self.dbProxy.update_job(tmpJobSpec,
{'propagatorLock': self.ident})
else:
mainLog.error(
'failed to update PandaID={0} status={1}'.format(
tmpJobSpec.PandaID, tmpJobSpec.status))
mainLog.debug('getting workers to propagate')
workSpecs = self.dbProxy.get_workers_to_propagate(
harvester_config.propagator.maxWorkers,
harvester_config.propagator.updateInterval)
mainLog.debug('got {0} workers'.format(len(workSpecs)))
# update workers in central database
iWorkers = 0
nWorkers = harvester_config.propagator.nWorkersInBulk
while iWorkers < len(workSpecs):
workList = workSpecs[iWorkers:iWorkers + nJobs]
iWorkers += nWorkers
retList, tmpErrStr = self.communicator.update_workers(workList)
# logging
if retList is None:
mainLog.error(
'failed to update workers with {0}'.format(tmpErrStr))
else:
for tmpWorkSpec, tmpRet in zip(workList, retList):
if tmpRet:
mainLog.debug(
'updated workerID={0} status={1}'.format(
tmpWorkSpec.workerID, tmpWorkSpec.status))
# update logs
for logFilePath, logOffset, logSize, logRemoteName in \
tmpWorkSpec.get_log_files_to_upload():
with open(logFilePath, 'rb') as logFileObj:
tmpStat, tmpErr = self.communicator.upload_file(
logRemoteName, logFileObj, logOffset,
logSize)
if tmpStat:
tmpWorkSpec.update_log_files_to_upload(
logFilePath, logOffset + logSize)
# disable further update
if tmpWorkSpec.is_final_status():
tmpWorkSpec.disable_propagation()
self.dbProxy.update_worker(
tmpWorkSpec,
{'workerID': tmpWorkSpec.workerID})
else:
mainLog.error(
'failed to update workerID={0} status={1}'.
format(tmpWorkSpec.workerID,
tmpWorkSpec.status))
mainLog.debug('getting commands')
commandSpecs = self.dbProxy.get_commands_for_receiver('propagator')
mainLog.debug('got {0} commands'.format(len(commandSpecs)))
for commandSpec in commandSpecs:
if commandSpec.command.startswith(
CommandSpec.COM_reportWorkerStats):
# get worker stats
siteName = commandSpec.command.split(':')[-1]
workerStats = self.dbProxy.get_worker_stats(siteName)
if len(workerStats) == 0:
mainLog.error(
'failed to get worker stats for {0}'.format(
siteName))
else:
# report worker stats
tmpRet, tmpStr = self.communicator.update_worker_stats(
siteName, workerStats)
if tmpRet:
mainLog.debug(
'updated worker stats (command) for {0}'.
format(siteName))
else:
mainLog.error(
'failed to update worker stats (command) for {0} err={1}'
.format(siteName, tmpStr))
if not self._last_stats_update or time.time(
) - self._last_stats_update > STATS_PERIOD:
# update worker stats for all sites
worker_stats_bulk = self.dbProxy.get_worker_stats_bulk()
if not worker_stats_bulk:
mainLog.error('failed to get worker stats in bulk')
else:
for site_name in worker_stats_bulk:
tmp_ret, tmp_str = self.communicator.update_worker_stats(
site_name, worker_stats_bulk[site_name])
if tmp_ret:
mainLog.debug(
'update of worker stats (bulk) for {0}'.format(
site_name))
self._last_stats_update = time.time()
else:
mainLog.error(
'failed to update worker stats (bulk) for {0} err={1}'
.format(site_name, tmp_str))
mainLog.debug('done' + sw.get_elapsed_time())
# check if being terminated
if self.terminated(harvester_config.propagator.sleepTime):
mainLog.debug('terminated')
return