def show_status(self) -> None:
logger.info(
'Currently blocked companies: %s' %
(Stream.of(self.locks.items()).map(lambda kv: '%s=%r' % kv).join(
', ') if self.locks else 'none', ))
for (name, queue) in Stream.of(self.control.queued.items()).map(
lambda kv: ('queued prio %d' % kv[0], kv[1])).list() + [
('running', self.control.running)
]:
logger.info('Currently %d %s processes' % (len(queue), name))
for entry in queue:
logger.info(
' %s%s' %
(entry.description,
(' (%d)' % entry.pid if entry.pid is not None else '')))
for job in self.deferred:
logger.info('%s: deferred' % job.description)
if self._schedule.queue:
logger.info('Currently scheduled events:')
for event in self._schedule.queue:
ts = event.time % (24 * 60 * 60)
logger.info('\t%2d:%02d:%02d [Prio %d]: %s' %
(ts // 3600,
(ts // 60) % 60, ts % 60, event.priority,
cast(Schedule.Job, event.action).name))
def pendings (self) -> bool:
while self.control.running:
pending = self.wait ()
if pending is None:
break
logger.debug ('%s: process finished' % pending.description)
while self.control.queued and len (self.control.running) < self.processes:
prio = Stream.of (self.control.queued.keys ()).sorted ().first ()
w = self.control.queued[prio].pop (0)
if not self.control.queued[prio]:
del self.control.queued[prio]
#
if w.prepare is not None:
try:
w.prepare (*w.args, **w.kwargs)
except Exception as e:
logger.exception ('%s: prepare fails: %s' % (w.description, e))
def starter () -> bool:
self.processtitle (w.description)
rc = w.method (*w.args, **w.kwargs)
self.processtitle ()
return rc
w.pid = self.control.control.spawn (starter)
self.control.running.append (w)
logger.debug ('%s: launched' % w.description)
return bool (self.control.running or self.control.queued)
def start(self) -> None:
self.title()
self.readConfiguration()
(Stream.of(self.modules.values()).sorted(
key=lambda task: task.priority).each(lambda task: self.every(
task.name,
self.configuration('immediately',
name=task.name,
default=task.immediately),
self.configuration('interval',
name=task.name,
default=task.interval,
convert=lambda v: Stream.ifelse(
v, alternator=task.interval)), task.
priority, task(self), ())))
def configurator() -> bool:
self.readConfiguration()
return True
def stopper() -> bool:
while self.control.running:
if self.wait() is None:
break
if not self.control.running and not self.deferred:
self.stop()
return False
return True
self.every('reload', False, '1h', 0, configurator, ())
self.every('restart', False, '24h', 0, stopper, ())
super().start()
def __str__(self) -> str:
return '{name} <{parameter}>'.format(
name=self.__class__.__name__,
parameter=Stream.of(
('mailing', self.mailing_id), ('customer', self.customer_id),
('source', self.source),
('selector', self.selector)).filter(lambda kv: kv[
1] is not None).map(lambda kv: '{key}={value!r}'.format(
key=kv[0], value=kv[1])).join(', '))
def configuration (self, key: str, name: Optional[str] = None, default: Any = None, convert: Optional[Callable[[Any], Any]] = None) -> Any:
return (
Stream.of (
('%s:%s[%s]' % (name, key, host)) if name is not None else None,
'%s[%s]' % (key, host),
('%s:%s' % (name, key)) if name is not None else None,
key
)
.filter (lambda k: k is not None and k in self.config)
.map (lambda k: self.config[k])
.first (no = default, finisher = lambda v: v if convert is None or v is None else convert (v))
)
def term (self) -> None:
super ().term ()
if self.control.running:
self.title ('in termination')
(Stream.of (self.control.running)
.peek (lambda p: logger.info ('Sending terminal signal to %s' % p.description))
.each (lambda p: self.control.control.term (p.pid))
)
while self.control.running:
logger.info ('Waiting for %d remaining child processes' % len (self.control.running))
self.wait (True)
if self.deferred:
logger.info ('Schedule final run for %d deferred tasks' % len (self.deferred))
self.defers ()
def joining (self, job: Watchdog.Job, ec: Daemonic.Status) -> None:
if self.output_path is not None and self.is_temp_output and os.path.isfile (self.output_path):
if ec.exitcode is None or ec.exitcode != Watchdog.EC_EXIT:
with open (self.output_path, errors = 'backslashreplace') as fd:
if self.limit > 0:
st = os.fstat (fd.fileno ())
if st.st_size > self.limit * 1024:
fd.seek (-self.limit * 1024, 2)
truncated = True
else:
truncated = False
lines = Stream.of (fd).remain (self.limit + 1).list ()
if len (lines) > self.limit:
truncated = True
if truncated:
lines[0] = '[..]'
output = '\n'.join (lines) + '\n'
else:
output = fd.read ()
if output:
logger.info ('Output of unexpected terminated process:\n%s' % output)
os.unlink (self.output_path)
def wait(self, block: bool = False) -> Optional[ScheduleGenerate.Pending]:
w: Optional[ScheduleGenerate.Pending] = None
while self.control.running and w is None:
rc = self.control.subprocess.join(timeout=None if block else 0)
if not rc.pid:
break
#
w = (Stream.of(self.control.running).filter(
lambda r: bool(r.pid == rc.pid)).first(no=None))
if w is not None:
logger.debug('{desc}: returned with {ec}'.format(
desc=w.description,
ec=f'exit with {rc.exitcode}' if rc.exitcode is not None
else f'died due to signal {rc.signal}'))
if w.finalize is not None:
try:
w.finalize(rc, *w.args, **w.kwargs)
except Exception as e:
logger.exception('%s: finalize fails: %s' %
(w.description, e))
self.control.running.remove(w)
self.lockTitle()
return w
def executor (self) -> bool:
activator = Activator ()
modules = (Stream.of (globals ().values ())
.filter (lambda module: type (module) is type and issubclass (module, Task) and hasattr (module, 'interval'))
.filter (lambda module: activator.check (['%s-%s' % (program, module.name)]))
.map (lambda module: (module.name, module))
.dict ()
)
logger.info ('Active modules: %s' % ', '.join (sorted (modules.keys ())))
schedule = ScheduleGenerate (modules, self.oldest, self.processes)
if self.modules:
schedule.readConfiguration ()
for name in self.modules:
if name not in modules:
print ('** %s not known' % name)
else:
logger.info ('Module found')
module = modules[name] (schedule)
rc = module ()
schedule.show_status ()
logger.info ('Module returns %r' % (rc, ))
if schedule.control.queued or schedule.control.running:
logger.info ('Execute backgound processes')
try:
while schedule.control.queued:
schedule.show_status ()
schedule.pendings ()
if len (schedule.control.running) == self.processes:
logger.info ('Currently %d processes running, wait for at least one to terminate' % len (schedule.control.running))
schedule.show_status ()
schedule.wait (True)
logger.info ('Wait for %d background process to teminate' % len (schedule.control.running))
while schedule.control.running:
schedule.show_status ()
if not schedule.wait (True):
break
except KeyboardInterrupt:
logger.info ('^C, terminate all running processes')
for p in schedule.control.running[:]:
if p.pid is not None:
schedule.control.control.term (p.pid)
schedule.wait ()
else:
schedule.control.running.remove (p)
logger.info ('Waiting for 2 seconds to kill all remaining processes')
time.sleep (2)
for p in schedule.control.running[:]:
if p.pid is not None:
schedule.control.control.term (p.pid, signal.SIGKILL)
else:
schedule.control.running.remove (p)
logger.info ('Waiting for killed processes to terminate')
while schedule.wait (True) is not None:
pass
logger.info ('Background processes done')
if schedule.deferred:
logger.info ('Deferred jobs active, process them')
try:
last = -1
while schedule.defers ():
cur = len (schedule.deferred)
if cur != last:
logger.info ('%d jobs remaining' % cur)
last = cur
time.sleep (1)
except KeyboardInterrupt:
logger.info ('^C, terminating')
else:
jq = JobqueueGenerate (schedule)
jq.start (restart_delay = '1m', termination_delay = '5m')
return True
def lockTitle (self) -> None:
self.title (Stream.of (self.locks.items ())
.map (lambda kv: '%s=%r' % kv)
.join (', ', lambda s: ('active locks %s' % s) if s else s)
)