class Task: #{{{
name = 'task'
immediately = False
def __init__(self, ref: ScheduleGenerate) -> None:
self.ref = ref
self.db = DB()
self.unit = Unit()
def __del__(self) -> None:
self.close()
def __call__(self) -> bool:
rc = False
if self.db.isopen():
self.ref.processtitle.push(self.name)
try:
self.execute()
self.ref.pendings()
except Exception as e:
logger.exception('failed due to: %s' % e)
else:
rc = True
finally:
self.close()
self.ref.processtitle.pop()
self.ref.lockTitle()
return rc
def close(self) -> None:
self.db.close()
def configuration(self, key: str, default: Any = None) -> Any:
return self.ref.configuration(key,
name=self.name,
default=self.default_for(key, default))
def title(self, title: Optional[str] = None) -> None:
self.ref.processtitle('%s%s' % (self.name,
(' %s' % title) if title else ''))
def default_for(self, key: str, default: Any) -> Any:
return default
#
# to overwrite
def execute(self) -> None:
pass
class Recovery(CLI): #{{{
@dataclass
class MailingInfo:
company_id: int
name: str
exists: bool
deleted: bool
def add_arguments(self, parser: argparse.ArgumentParser) -> None:
parser.add_argument('-n',
'--dryrun',
action='store_true',
help='just show')
parser.add_argument(
'-A',
'--age',
action="store",
type=int,
default=1,
help='set maximum age for a mailing to be recovered in days')
parser.add_argument(
'-D',
'--startup-delay',
action="store",
default="1m",
help=
'set delay to wait for startup of backend in seconds or an offset (e.g. "1m")',
dest='startup_delay')
parser.add_argument(
'parameter',
nargs='*',
help='optional list of mailing_ids to restrict recovery to')
def use_arguments(self, args: argparse.Namespace) -> None:
unit = Unit()
self.dryrun = args.dryrun
self.max_age = args.age
self.startup_delay = unit.parse(args.startup_delay)
self.restrict_to_mailings = set(
int(_p) for _p in args.parameter) if args.parameter else None
def prepare(self) -> None:
self.db = DB()
self.mailings: List[Mailing] = []
self.mailing_info: Dict[int, Recovery.MailingInfo] = {}
self.report: List[str] = []
self.db.check_open()
def cleanup(self, success: bool) -> None:
for m in self.mailings:
m.done()
self.db.close()
def executor(self) -> bool:
log.set_loglevel('debug')
try:
with Lock():
with log('collect'):
self.collect_mailings()
with log('recover'):
self.recover_mailings()
with log('report'):
self.report_mailings()
return True
except error as e:
logger.exception('Failed recovery: %s' % e)
return False
def __make_range(self, start: datetime, end: datetime) -> List[str]: #{{{
rc: List[str] = []
current_day = start.toordinal()
end_day = end.toordinal()
while current_day <= end_day:
day = datetime.fromordinal(current_day)
rc.append(f'{day.year:04d}{day.month:02d}{day.day:02d}')
current_day += 1
return rc
#}}}
def __mail(self, mailing_id: int) -> Recovery.MailingInfo: #{{{
with Ignore(KeyError):
return self.mailing_info[mailing_id]
#
rq = self.db.querys(
'SELECT company_id, shortname, deleted FROM mailing_tbl WHERE mailing_id = :mid',
{'mid': mailing_id})
self.mailing_info[mailing_id] = rc = Recovery.MailingInfo(
company_id=rq.company_id if rq is not None else 0,
name=rq.shortname
if rq is not None else f'#{mailing_id} not found',
exists=rq is not None,
deleted=bool(rq.deleted) if rq is not None else False)
return rc
#}}}
def __mailing_name(self, mailing_id: int) -> str: #{{{
return self.__mail(mailing_id).name
#}}}
def __mailing_exists(self, mailing_id: int) -> bool: #{{{
return self.__mail(mailing_id).exists
#}}}
def __mailing_deleted(self, mailing_id: int) -> bool: #{{{
return self.__mail(mailing_id).deleted
#}}}
def __mailing_valid(self, mailing_id: int) -> bool: #{{{
return self.__mailing_exists(
mailing_id) and not self.__mailing_deleted(mailing_id)
#}}}
def collect_mailings(self) -> None: #{{{
now = datetime.now()
expire = now - timedelta(days=self.max_age)
yesterday = now - timedelta(days=1)
query = (
'SELECT status_id, mailing_id '
'FROM maildrop_status_tbl '
'WHERE genstatus = 2 AND status_field = \'R\' AND genchange > :expire AND genchange < current_timestamp'
)
check_query = self.db.qselect(
oracle=
'SELECT count(*) FROM rulebased_sent_tbl WHERE mailing_id = :mid AND to_char (lastsent, \'YYYY-MM-DD\') = to_char (sysdate - 1, \'YYYY-MM-DD\')',
mysql=
'SELECT count(*) FROM rulebased_sent_tbl WHERE mailing_id = :mid AND date_format(lastsent, \'%%Y-%%m-%%d\') = \'%04d-%02d-%02d\''
% (yesterday.year, yesterday.month, yesterday.day))
update = ('UPDATE maildrop_status_tbl '
'SET genstatus = 1, genchange = current_timestamp '
'WHERE status_id = :sid')
for (status_id, mailing_id) in self.db.queryc(query,
{'expire': expire}):
if (self.restrict_to_mailings is None
or mailing_id in self.restrict_to_mailings
) and self.__mailing_valid(mailing_id):
count = self.db.querys(check_query, {'mid': mailing_id})
if count is not None and count[0] == 1:
logger.info('Reactivate rule based mailing %d: %s' %
(mailing_id, self.__mailing_name(mailing_id)))
if not self.dryrun:
self.db.update(update, {'sid': status_id})
self.report.append(
'%s [%d]: Reactivate rule based mailing' %
(self.__mailing_name(mailing_id), mailing_id))
else:
logger.warning(
'Rule based mailing %d (%s) not reactivated as it had not been sent out yesterday'
% (mailing_id, self.__mailing_name(mailing_id)))
self.report.append(
'%s [%d]: Not reactivating rule based mailing as it had not been sent out yesterday'
% (self.__mailing_name(mailing_id), mailing_id))
if not self.dryrun:
self.db.sync()
query = (
'SELECT status_id, mailing_id, company_id, status_field, senddate '
'FROM maildrop_status_tbl '
'WHERE genstatus IN (1, 2) AND genchange > :expire AND genchange < current_timestamp AND status_field = \'W\''
)
for (status_id, mailing_id, company_id, status_field,
senddate) in self.db.queryc(query, {'expire': expire}):
if (self.restrict_to_mailings is None
or mailing_id in self.restrict_to_mailings
) and self.__mailing_valid(mailing_id):
check = self.__make_range(senddate, now)
self.mailings.append(
Mailing(status_id, status_field, mailing_id, company_id,
check))
logger.info('Mark mailing %d (%s) for recovery' %
(mailing_id, self.__mailing_name(mailing_id)))
self.mailings.sort(key=lambda m: m.status_id)
logger.info('Found %d mailing(s) to recover' % len(self.mailings))
#}}}
def recover_mailings(self) -> None: #{{{
if not self.dryrun and self.mailings and self.startup_delay > 0:
logger.info('Wait for backend to start up')
n = self.startup_delay
while n > 0:
time.sleep(1)
n -= 1
if not self.running:
raise error('abort due to process termination')
for m in self.mailings:
m.collect_seen()
if self.dryrun:
print('%s: %d recipients already seen' %
(self.__mailing_name(m.mailing_id), len(m.seen)))
else:
m.create_filelist()
count = 0
for (total_mails, ) in self.db.query(
'SELECT total_mails FROM mailing_backend_log_tbl WHERE status_id = :sid',
{'sid': m.status_id}):
if not total_mails is None and total_mails > count:
count = total_mails
m.set_generated_count(count)
self.db.update(
'DELETE FROM mailing_backend_log_tbl WHERE status_id = :sid',
{'sid': m.status_id})
self.db.update(
'DELETE FROM world_mailing_backend_log_tbl WHERE mailing_id = :mid',
{'mid': m.mailing_id})
self.db.update(
'UPDATE maildrop_status_tbl SET genstatus = 1 WHERE status_id = :sid',
{'sid': m.status_id})
self.db.sync()
logger.info('Start backend using status_id %d for %s' %
(m.status_id, self.__mailing_name(m.mailing_id)))
starter = agn3.emm.mailing.Mailing()
if not starter.fire(status_id=m.status_id,
cursor=self.db.cursor):
logger.error('Failed to trigger mailing %d')
self.report.append(
'%s [%d]: Failed to trigger mailing' %
(self.__mailing_name(m.mailing_id), m.mailing_id))
break
self.db.sync()
self.report.append(
'%s [%d]: Start recovery using status_id %d' %
(self.__mailing_name(m.mailing_id), m.mailing_id, m.status_id))
if not self.dryrun:
query = 'SELECT genstatus FROM maildrop_status_tbl WHERE status_id = :status_id'
start = int(time.time())
ok = True
last_generation_status = 1
while self.running and m.active and ok:
now = int(time.time())
self.db.sync(False)
row = self.db.querys(query, {'status_id': m.status_id})
if row is None or row[0] is None:
logger.info('Failed to query status for mailing %d' %
m.mailing_id)
self.report.append(
'%s [%d]: Recovery failed due to missing status' %
(self.__mailing_name(m.mailing_id), m.mailing_id))
ok = False
else:
generation_status = row[0]
if generation_status != last_generation_status:
logger.info(
f'Mailings {m.mailing_id} generation status has changed from {last_generation_status} to {generation_status}'
)
last_generation_status = generation_status
if generation_status == 3:
logger.info('Mailing %d terminated as expected' %
m.mailing_id)
self.report.append(
'%s [%d]: Recovery finished' %
(self.__mailing_name(
m.mailing_id), m.mailing_id))
m.active = False
elif generation_status == 2:
if m.last:
current = 0
for (currentMails, ) in self.db.query(
'SELECT current_mails FROM mailing_backend_log_tbl WHERE status_id = :sid',
{'sid': m.status_id}):
if not currentMails is None:
current = currentMails
if current != m.current:
logger.debug(
f'Mailing {m.mailing_id} has created {current:,d} vs. {m.current:,d} when last checked'
)
m.current = current
m.last = now
else:
if (current > 0 and m.last + 1200 < now
) or (current == 0
and m.last + 3600 < now):
logger.info(
'Mailing %d terminated due to inactivity after %d mails'
% (m.mailing_id, current))
self.report.append(
'%s [%d]: Recovery timed out' %
(self.__mailing_name(
m.mailing_id), m.mailing_id))
ok = False
else:
m.last = now
elif generation_status == 1:
if start + 1800 < now:
logger.info(
'Mailing %d terminated while not starting up'
% m.mailing_id)
self.report.append(
'%s [%d]: Recovery not started' %
(self.__mailing_name(
m.mailing_id), m.mailing_id))
ok = False
elif generation_status > 3:
logger.info(
'Mailing %d terminated with status %d' %
(m.mailing_id, generation_status))
self.report.append(
'%s [%d]: Recovery ended with unexpected status %d'
% (self.__mailing_name(m.mailing_id),
m.mailing_id, generation_status))
m.active = False
if m.active and ok:
if start + 30 * 60 < now:
logger.info(
'Failed due to global timeout to recover %d' %
m.mailing_id)
self.report.append(
'%s [%d]: Recovery ended due to global timeout'
% (self.__mailing_name(
m.mailing_id), m.mailing_id))
ok = False
else:
time.sleep(1)
if not m.active:
count = 0
for (total_mails, ) in self.db.query(
'SELECT total_mails FROM mailing_backend_log_tbl WHERE status_id = :sid',
{'sid': m.status_id}):
if not total_mails is None:
count = total_mails
count += len(m.seen)
self.db.update(
'UPDATE mailing_backend_log_tbl SET total_mails = :cnt, current_mails = :cnt WHERE status_id = :sid',
{
'sid': m.status_id,
'cnt': count
})
self.db.update(
'UPDATE world_mailing_backend_log_tbl SET total_mails = :cnt, current_mails = :cnt WHERE mailing_id = :mid',
{
'mid': m.mailing_id,
'cnt': count
})
self.db.sync()
if not self.running or not ok:
break
#}}}
def report_mailings(self) -> None: #{{{
class MailInfo(NamedTuple):
status_id: int
status_field: str
mailing_id: int
mailing_name: str
company_id: int
deleted: bool
genchange: datetime
senddate: datetime
mails = []
query = 'SELECT status_id, mailing_id, genstatus, genchange, status_field, senddate FROM maildrop_status_tbl WHERE '
query += 'genstatus IN (1, 2) AND status_field IN (\'W\', \'R\', \'D\')'
for (status_id, mailing_id, genstatus, genchange, status_field,
senddate) in self.db.queryc(query):
if status_field in ('R', 'D') and genstatus == 1:
continue
info = self.__mail(mailing_id)
mails.append(
MailInfo(status_id=status_id,
status_field=status_field,
mailing_id=mailing_id,
mailing_name=info.name,
company_id=info.company_id,
deleted=info.deleted,
genchange=genchange,
senddate=senddate))
if self.report or mails:
template = os.path.join(base, 'scripts', 'recovery3.tmpl')
try:
with open(template, 'r') as fd:
content = fd.read()
ns = {'host': fqdn, 'report': self.report, 'mails': mails}
tmpl = Template(content)
try:
body = tmpl.fill(ns)
charset = tmpl.property('charset', default='UTF-8')
subject = tmpl.property('subject')
if not subject:
subject = tmpl['subject']
if not subject:
subject = 'Recovery report for %s' % ns['host']
else:
subject = Template(subject).fill(ns)
sender = tmpl.property('sender', f'{user}@{fqdn}')
receiver = tmpl.property('receiver')
if receiver:
receiver = Template(receiver).fill(ns)
if self.dryrun:
print('From: %s' % sender)
print('To: %s' % receiver)
print('Subject: %s' % subject)
print('')
print(body)
else:
EMail.force_encoding(charset, 'qp')
mail = EMail()
if sender:
mail.set_sender(sender)
for recv in [
_r.strip() for _r in receiver.split(',')
]:
if recv:
mail.add_to(recv)
if charset:
mail.set_charset(charset)
mail.set_subject(subject)
mail.set_text(body)
mail.send_mail()
except error as e:
logger.error('Failed to fill template "%s": %s' %
(template, e))
except IOError as e:
logger.error('Unable to find template "%s": %s' %
(template, e))