def run(self):
if self._duration < 0:
self._log.info('Running in continuous mode. Press ^C to exit.')
elif self._duration > 0:
self._log.info('Running for %s', str_time_short(self._duration))
# Prepare work package
self.backend.deploy_task(self.task, transfer_se=self._transfer_se, transfer_sb=self._transfer_sb)
# Job submission loop
backend_timing_info = self.backend.get_interval_info()
t_start = time.time()
while not abort():
did_wait = False
# Check whether backend can submit
if not self.backend.can_submit(self._submit_time, self._submit_flag):
self._submit_flag = False
# Check free disk space
if self._no_disk_space_left():
self._check_space_log.warning('Not enough space left in working directory')
else:
did_wait = self._run_actions(backend_timing_info)
# quit if abort flag is set or not in continuous mode
if abort() or ((self._duration >= 0) and (time.time() - t_start > self._duration)):
break
# idle timeout
if not did_wait:
wait(backend_timing_info.wait_on_idle)
self.job_manager.finish()
def _cancel(self, task, wms, jobnum_list, interactive, show_jobs):
if len(jobnum_list) == 0:
return
if show_jobs:
self._abort_report.show_report(self.job_db, jobnum_list)
if interactive and not self._uii.prompt_bool('Do you really want to cancel these jobs?', True):
return
def _mark_cancelled(jobnum):
job_obj = self.job_db.get_job(jobnum)
if job_obj is not None:
self._update(task, job_obj, jobnum, Job.CANCELLED)
self._local_event_handler.on_job_update(task, wms, job_obj, jobnum, {'reason': 'cancelled'})
jobnum_list.reverse()
map_gc_id2jobnum = self._get_map_gc_id_jobnum(jobnum_list)
gc_id_list = sorted(map_gc_id2jobnum, key=lambda gc_id: -map_gc_id2jobnum[gc_id])
for (gc_id,) in wms.cancel_jobs(gc_id_list):
# Remove cancelledd job from todo list and mark as cancelled
_mark_cancelled(map_gc_id2jobnum.pop(gc_id))
if map_gc_id2jobnum:
jobnum_list = list(map_gc_id2jobnum.values())
self._log.warning('There was a problem with cancelling the following jobs:')
self._abort_report.show_report(self.job_db, jobnum_list)
if (not interactive) or self._uii.prompt_bool('Do you want to mark them as cancelled?', True):
lmap(_mark_cancelled, jobnum_list)
if interactive:
wait(2)
def cancel(self, wms, jobs, interactive, showJobs):
if len(jobs) == 0:
return
if showJobs:
self._reportClass(self.jobDB, self._task, jobs).display()
if interactive and not utils.getUserBool(
'Do you really want to cancel these jobs?', True):
return
def mark_cancelled(jobNum):
jobObj = self.jobDB.get(jobNum)
if jobObj is None:
return
self._update(jobObj, jobNum, Job.CANCELLED)
self._eventhandler.onJobUpdate(wms, jobObj, jobNum,
{'reason': 'cancelled'})
jobs.reverse()
for (jobNum, wmsId) in wms.cancelJobs(self._wmsArgs(jobs)):
# Remove deleted job from todo list and mark as cancelled
assert (self.jobDB.get(jobNum).wmsId == wmsId)
jobs.remove(jobNum)
mark_cancelled(jobNum)
if len(jobs) > 0:
self._log_user.warning(
'There was a problem with cancelling the following jobs:')
self._reportClass(self.jobDB, self._task, jobs).display()
if (interactive and utils.getUserBool(
'Do you want to mark them as cancelled?',
True)) or not interactive:
lmap(mark_cancelled, jobs)
if interactive:
utils.wait(2)
def cancel(self, wms, jobs, interactive = False, showJobs = True):
if len(jobs) == 0:
return
if showJobs:
self._reportClass(self.jobDB, self._task, jobs).display()
if interactive and not utils.getUserBool('Do you really want to cancel these jobs?', True):
return
def mark_cancelled(jobNum):
jobObj = self.jobDB.get(jobNum)
if jobObj is None:
return
self._update(jobObj, jobNum, Job.CANCELLED)
self._eventhandler.onJobUpdate(wms, jobObj, jobNum, {'reason': 'cancelled'})
jobs.reverse()
for (jobNum, wmsId) in wms.cancelJobs(self._wmsArgs(jobs)):
# Remove deleted job from todo list and mark as cancelled
assert(self.jobDB.get(jobNum).wmsId == wmsId)
jobs.remove(jobNum)
mark_cancelled(jobNum)
if len(jobs) > 0:
self._log_user.warning('There was a problem with cancelling the following jobs:')
self._reportClass(self.jobDB, self._task, jobs).display()
if (interactive and utils.getUserBool('Do you want to mark them as cancelled?', True)) or not interactive:
lmap(mark_cancelled, jobs)
if interactive:
utils.wait(2)
def _cancel(self, task, wms, jobnum_list, interactive, show_jobs):
if len(jobnum_list) == 0:
return
if show_jobs:
self._abort_report.show_report(self.job_db, jobnum_list)
if interactive and not self._uii.prompt_bool(
'Do you really want to cancel these jobs?', True):
return
def _mark_cancelled(jobnum):
job_obj = self.job_db.get_job(jobnum)
if job_obj is not None:
self._update(task, job_obj, jobnum, Job.CANCELLED)
self._local_event_handler.on_job_update(
task, wms, job_obj, jobnum, {'reason': 'cancelled'})
jobnum_list.reverse()
map_gc_id2jobnum = self._get_map_gc_id_jobnum(jobnum_list)
gc_id_list = sorted(map_gc_id2jobnum,
key=lambda gc_id: -map_gc_id2jobnum[gc_id])
for (gc_id, ) in wms.cancel_jobs(gc_id_list):
# Remove cancelledd job from todo list and mark as cancelled
_mark_cancelled(map_gc_id2jobnum.pop(gc_id))
if map_gc_id2jobnum:
jobnum_list = list(map_gc_id2jobnum.values())
self._log.warning(
'There was a problem with cancelling the following jobs:')
self._abort_report.show_report(self.job_db, jobnum_list)
if (not interactive) or self._uii.prompt_bool(
'Do you want to mark them as cancelled?', True):
lmap(_mark_cancelled, jobnum_list)
if interactive:
wait(2)
def _main():
options = _parse_cmd_line()
opts = options.opts
signal.signal(signal.SIGINT, handle_abort_interrupt)
# Disable loop mode if it is pointless
if (opts.loop and not opts.skip_existing) and (opts.mark_ignore_dl or not opts.mark_dl):
log.info('Loop mode was disabled to avoid continuously downloading the same files\n')
(opts.loop, opts.infinite) = (False, False)
while True:
if (process_all(opts, options.args) or not opts.loop) and not opts.infinite:
break
wait(60)
def _run_actions(self, backend_timing_info):
did_wait = False
for action in imap(str.lower, self._action_list):
if not abort():
if action.startswith('c'): # check for jobs
if self.job_manager.check(self.task, self.backend):
did_wait = wait(backend_timing_info.wait_between_steps)
elif action.startswith('r'): # retrieve finished jobs
if self.job_manager.retrieve(self.task, self.backend):
did_wait = wait(backend_timing_info.wait_between_steps)
elif action.startswith('s') and self._submit_flag:
if self.job_manager.submit(self.task, self.backend):
did_wait = wait(backend_timing_info.wait_between_steps)
return did_wait
def cancelJobs(self, allIds):
if len(allIds) == 0:
raise StopIteration
waitFlag = False
for ids in imap(lambda x: allIds[x:x+5], irange(0, len(allIds), 5)):
# Delete jobs in groups of 5 - with 5 seconds between groups
if waitFlag and not utils.wait(5):
break
waitFlag = True
jobNumMap = dict(ids)
jobs = self.writeWMSIds(ids)
activity = utils.ActivityLog('cancelling jobs')
proc = LocalProcess(self._cancelExec, '--noint', '--logfile', '/dev/stderr', '-i', jobs)
retCode = proc.status(timeout = 60, terminate = True)
del activity
# select cancelled jobs
for deletedWMSId in ifilter(lambda x: x.startswith('- '), proc.stdout.iter()):
deletedWMSId = self._createId(deletedWMSId.strip('- \n'))
yield (jobNumMap.get(deletedWMSId), deletedWMSId)
if retCode != 0:
if self.explainError(proc, retCode):
pass
else:
self._log.log_process(proc, files = {'jobs': utils.safeRead(jobs)})
utils.removeFiles([jobs])
def cancelJobs(self, allIds):
if len(allIds) == 0:
raise StopIteration
waitFlag = False
for ids in map(lambda x: allIds[x:x+5], range(0, len(allIds), 5)):
# Delete jobs in groups of 5 - with 5 seconds between groups
if waitFlag and utils.wait(5) == False:
break
waitFlag = True
jobNumMap = dict(ids)
jobs = self.writeWMSIds(ids)
log = tempfile.mktemp('.log')
activity = utils.ActivityLog('cancelling jobs')
proc = utils.LoggedProcess(self._cancelExec, '--noint --logfile "%s" -i "%s"' % (log, jobs))
retCode = proc.wait()
del activity
# select cancelled jobs
for deletedWMSId in filter(lambda x: x.startswith('- '), proc.iter()):
deletedWMSId = self._createId(deletedWMSId.strip('- \n'))
yield (jobNumMap.get(deletedWMSId), deletedWMSId)
if retCode != 0:
if self.explainError(proc, retCode):
pass
else:
proc.logError(self.errorLog, log = log)
utils.removeFiles([log, jobs])
def cancelJobs(self, allIds):
if len(allIds) == 0:
raise StopIteration
waitFlag = False
for ids in imap(lambda x: allIds[x:x + 5], irange(0, len(allIds), 5)):
# Delete jobs in groups of 5 - with 5 seconds between groups
if waitFlag and not utils.wait(5):
break
waitFlag = True
jobNumMap = dict(ids)
jobs = self.writeWMSIds(ids)
activity = utils.ActivityLog('cancelling jobs')
proc = LocalProcess(self._cancelExec, '--noint', '--logfile',
'/dev/stderr', '-i', jobs)
retCode = proc.status(timeout=60, terminate=True)
del activity
# select cancelled jobs
for deletedWMSId in ifilter(lambda x: x.startswith('- '),
proc.stdout.iter()):
deletedWMSId = self._createId(deletedWMSId.strip('- \n'))
yield (jobNumMap.get(deletedWMSId), deletedWMSId)
if retCode != 0:
if self.explainError(proc, retCode):
pass
else:
self._log.log_process(proc,
files={'jobs': utils.safeRead(jobs)})
utils.removeFiles([jobs])
def execute(self, wmsIDs, *args, **kwargs): do_wait = False for wmsIDChunk in imap(lambda x: wmsIDs[x:x + self._chunk_size], irange(0, len(wmsIDs), self._chunk_size)): if do_wait and not utils.wait(self._chunk_time): break do_wait = True for result in self._executor.execute(wmsIDChunk, *args, **kwargs): yield result
def execute(self, wms_id_list, *args, **kwargs): do_wait = False chunk_pos_iter = irange(0, len(wms_id_list), self._chunk_size) for wms_id_chunk in imap(lambda x: wms_id_list[x:x + self._chunk_size], chunk_pos_iter): if do_wait and not wait(self._chunk_interval): break do_wait = True for result in self._executor.execute(wms_id_chunk, *args, **kwargs): yield result
def _main():
options = _parse_cmd_line()
opts = options.opts
signal.signal(signal.SIGINT, handle_abort_interrupt)
# Disable loop mode if it is pointless
if (opts.loop and not opts.skip_existing) and (opts.mark_ignore_dl
or not opts.mark_dl):
log.info(
'Loop mode was disabled to avoid continuously downloading the same files\n'
)
(opts.loop, opts.infinite) = (False, False)
while True:
if (process_all(opts, options.args)
or not opts.loop) and not opts.infinite:
break
wait(60)
def guiWait(timeout):
onResize(None, None)
oldHandler = signal.signal(signal.SIGWINCH, onResize)
result = utils.wait(timeout)
signal.signal(signal.SIGWINCH, oldHandler)
if (time.time() - guiWait.lastwait > 10) and not timeout:
tmp = utils.ActivityLog('') # force display update
tmp.finish()
guiWait.lastwait = time.time()
return result
def cancelJobs(self, allIds):
if len(allIds) == 0:
raise StopIteration
waitFlag = False
for ids in imap(lambda x: allIds[x:x+self._nJobsPerChunk], irange(0, len(allIds), self._nJobsPerChunk)):
# Delete jobs in groups of 5 - with 5 seconds between groups
if waitFlag and not utils.wait(5):
break
waitFlag = True
jobNumMap = dict(ids)
jobs = ' '.join(self._getRawIDs(ids))
log = tempfile.mktemp('.log')
activity = utils.ActivityLog('cancelling jobs')
proc = utils.LoggedProcess(self._cancelExec, '--noint --logfile "%s" %s' % (log, jobs))
retCode = proc.wait()
del activity
# select cancelled jobs
for rawId in self._getRawIDs(ids):
deletedWMSId = self._createId(rawId)
yield (jobNumMap.get(deletedWMSId), deletedWMSId)
if retCode != 0:
if self.explainError(proc, retCode):
pass
else:
proc.logError(self.errorLog, log = log)
purgeLog = tempfile.mktemp('.log')
purgeProc = utils.LoggedProcess(self._purgeExec, '--noint --logfile "%s" %s' % (purgeLog, jobs))
retCode = purgeProc.wait()
if retCode != 0:
if self.explainError(purgeProc, retCode):
pass
else:
proc.logError(self.errorLog, log = purgeLog, jobs = jobs)
utils.removeFiles([log, purgeLog])
def guiWait(timeout): onResize(None, None) oldHandler = signal.signal(signal.SIGWINCH, onResize) result = utils.wait(timeout) signal.signal(signal.SIGWINCH, oldHandler) return result
def _process_queue(self, timeout):
self._counter += 1
wait(timeout)
def processQueue(self, timeout):
self.counter += 1
utils.wait(timeout)
def _wait(self, timeout):
oldHandler = signal.signal(signal.SIGWINCH,
self._schedule_update_layout)
result = utils.wait(timeout)
signal.signal(signal.SIGWINCH, oldHandler)
return result
def onTaskFinish(self, nJobs): utils.wait(5)
def processQueue(self, timeout): self.counter += 1 utils.wait(timeout)
def _wait(self, timeout): oldHandler = signal.signal(signal.SIGWINCH, self._schedule_update_layout) result = utils.wait(timeout) signal.signal(signal.SIGWINCH, oldHandler) return result