def use_arguments(self, args: argparse.Namespace) -> None: unit = Unit() self.dryrun = args.dryrun self.max_age = args.age self.startup_delay = unit.parse(args.startup_delay) self.restrict_to_mailings = set( int(_p) for _p in args.parameter) if args.parameter else None
def add_arguments (self, parser: argparse.ArgumentParser) -> None: unit = Unit () parser.add_argument ('-v', '--verbose', action = 'store_true') parser.add_argument ('-i', '--instance', action = 'store') parser.add_argument ('-b', '--background', action = 'store_true') parser.add_argument ('-r', '--restart-delay', action = 'store', type = unit, default = 60, dest = 'restart_delay') parser.add_argument ('-t', '--termination-delay', action = 'store', type = unit, default = 10, dest = 'termination_delay') parser.add_argument ('-o', '--output', action = 'store') parser.add_argument ('-p', '--prior', action = 'store') parser.add_argument ('-l', '--limit', action = 'store', type = int, default = 0) parser.add_argument ('command', nargs = '*')
def add_arguments (self, parser: argparse.ArgumentParser) -> None: unit = Unit () parser.add_argument ( '-O', '--oldest', action = 'store', type = unit, default = unit ('3d'), help = 'oldest entry to process' ) parser.add_argument ( '-P', '--processes', '--parallel', action = 'store', type = int, default = 4, help = 'specifiy the number of parallel generation processes' )
class BavUpdate(Runtime): unit = Unit() config_filename = os.path.join(base, 'var', 'lib', 'bav.conf') local_filename = os.path.join(base, 'var', 'lib', 'bav.conf-local') rule_directory = os.path.join(base, 'var', 'lib') rule_file = os.path.join(base, 'lib', 'bav.rule') rule_pattern = re.compile('bav_([0-9]+).rule') rule_format = os.path.join(rule_directory, 'bav_%d.rule') control_sendmail = os.path.join(base, 'bin', 'smctrl') restart_sendmail = os.path.join(base, 'var', 'run', 'sendmail-control.sh') sendmail_base = '/etc/mail' def supports(self, option: str) -> bool: return option not in ('dryrun', 'parameter') def setup(self) -> None: EMail.force_encoding('UTF-8', 'quoted-printable') self.delay = self.unit.parse('3m') self.fqdn = socket.getfqdn().lower() if not self.fqdn: self.fqdn = fqdn self.fixdomain = syscfg.get_str('filter-name', 'localhost') self.mta = MTA() self.domains: List[str] = [] self.mtdom: Dict[str, int] = {} self.prefix = 'aml_' self.last = '' self.autoresponder: List[Autoresponder] = [] self.read_mailertable() try: files = os.listdir(Autoresponder.directory) for fname in files: if len(fname ) > 8 and fname[:3] == 'ar_' and fname[-5:] == '.mail': with Ignore(ValueError, OSError): rid = int(fname[3:-5]) st = os.stat( os.path.join(Autoresponder.directory, fname)) self.autoresponder.append( Autoresponder(rid, datetime.fromtimestamp(st.st_ctime), None, None)) except OSError as e: logger.error('Unable to read directory %s %s' % (Autoresponder.directory, e)) def add_arguments(self, parser: argparse.ArgumentParser) -> None: parser.add_argument( '-D', '--delay', action='store', type=self.unit.parse, default=self.delay, help='delay between each reread of database content') def use_arguments(self, args: argparse.Namespace) -> None: self.delay = args.delay def executor(self) -> bool: while self.running: self.update() n = self.delay while n > 0 and self.running: time.sleep(1) n -= 1 return True def file_reader(self, fname: str) -> List[str]: with open(fname, errors='backslashreplace') as fd: return [line.rstrip('\r\n') for line in fd if not line[0] in '\n#'] class Domain(NamedTuple): name: str value: str @dataclass class Filecontent: path: Optional[str] content: List[BavUpdate.Domain] modified: bool hash: Optional[str] @dataclass class RID: rid: int domain: str def read_mailertable( self, new_domains: Optional[Dict[str, BavUpdate.RID]] = None) -> None: self.domains.clear() self.mtdom.clear() if self.mta.mta == 'postfix': def find(key: str, default_value: str) -> BavUpdate.Filecontent: rc = BavUpdate.Filecontent(path=None, content=[], modified=False, hash=None) with Ignore(KeyError): for element in self.mta.getlist(key): hash: Optional[str] path: str try: (hash, path) = element.split(':', 1) except ValueError: (hash, path) = (None, element) if path.startswith(base): if rc.path is None: rc.path = path rc.hash = hash if not os.path.isfile(path): create_path(os.path.dirname(path)) open(path, 'w').close() if hash is not None: self.mta.postfix_make(path) if rc.path is not None: try: with open(rc.path) as fd: for line in (_l.strip() for _l in fd): try: (var, val) = [ _v.strip() for _v in line.split(None, 1) ] except ValueError: var = line val = default_value rc.modified = True if var not in [ _c.name for _c in rc.content ]: rc.content.append( BavUpdate.Domain(name=var, value=val)) else: rc.modified = True logger.debug('Read %d lines from %s' % (len(rc.content), rc.path)) except OSError as e: logger.error('Failed to read %s: %s' % (rc.path, e)) else: logger.warning( 'No path for postfix parameter %s found' % key) return rc def save(ct: BavUpdate.Filecontent) -> None: if ct.path is not None and (ct.modified or not os.path.isfile(ct.path)): try: with open(ct.path, 'w') as fd: if ct.content: fd.write('\n'.join( ['%s\t%s' % _c for _c in ct.content]) + '\n') logger.info('Written %d lines to %s' % (len(ct.content), ct.path)) if ct.hash is not None: self.mta.postfix_make(ct.path) except OSError as e: logger.error('Failed to save %s: %s' % (ct.path, e)) # relay_default_value = 'dummy' relays = find('relay_domains', relay_default_value) for d in relays.content: self.mtdom[d.name] = 0 # def add_relay_domain(domain_to_add: str) -> None: if domain_to_add and domain_to_add not in self.mtdom: relays.content.append( BavUpdate.Domain(name=domain_to_add, value=relay_default_value)) relays.modified = True self.mtdom[domain_to_add] = 0 # if new_domains: for domain in new_domains: add_relay_domain(domain) transport_default_value = 'mailloop:' transports = find('transport_maps', transport_default_value) with DB() as db: for row in db.query( 'SELECT mailloop_domain FROM company_tbl WHERE mailloop_domain IS NOT NULL AND status = :status', {'status': 'active'}): if row.mailloop_domain: add_relay_domain(row.mailloop_domain.strip().lower()) transport_domains = set([_c[0] for _c in transports.content]) for d in relays.content[:]: if d.name not in transport_domains: transports.content.append( BavUpdate.Domain(name=d.name, value=transport_default_value)) transports.modified = True self.mtdom[d.name] += 1 save(relays) save(transports) if relays.modified or transports.modified: cmd = which('smctrl') if cmd is not None: n = silent_call(cmd, 'service', 'reload') if n == 0: logger.info('Reloaded') else: logger.error('Reloading failed: %d' % n) self.domains = [_c.name for _c in relays.content] else: try: for line in self.file_reader( os.path.join(self.sendmail_base, 'mailertable')): parts = line.split() if len(parts) > 1 and not parts[0].startswith( '.') and parts[1].startswith('procmail:'): self.domains.append(parts[0]) self.mtdom[parts[0]] = 0 except IOError as e: logger.error('Unable to read mailertable %s' % e) try: for line in self.file_reader( os.path.join(self.sendmail_base, 'relay-domains')): if line in self.mtdom: self.mtdom[line] += 1 else: logger.debug( 'We relay domain "%s" without catching it in mailertable' % line) for key in self.mtdom.keys(): if self.mtdom[key] == 0: logger.debug( 'We define domain "%s" in mailertable, but do not relay it' % key) except IOError as e: logger.error('Unable to read relay-domains %s' % e) def read_mail_files(self) -> List[str]: rc: List[str] = [] if self.fqdn: rc.append('@%s\taccept:rid=local' % self.fqdn) if self.mta.mta == 'sendmail': try: for line in self.file_reader( os.path.join(self.sendmail_base, 'local-host-names')): rc.append('@%s\taccept:rid=local' % line) except IOError as e: logger.error(f'Unable to read local-host-names: {e}') try: for line in self.file_reader( os.path.join(self.sendmail_base, 'virtusertable')): parts = line.split() if len(parts) == 2: rc.append('%s\taccept:rid=virt,fwd=%s' % (parts[0], parts[1])) except IOError as e: logger.error(f'Unable to read virtusertable: {e}') return rc def update_rules(self, rules: Dict[int, Dict[str, List[str]]]) -> None: inuse: Set[int] = set() global_rule = None for (rid, rule) in rules.items(): if global_rule is None: try: with open(self.rule_file) as fd: global_rule = fd.read() if not global_rule.endswith('\n'): global_rule += '\n' except IOError as e: logger.error('Failed to open "%s" for reading: %s' % (self.rule_file, e)) inuse.add(rid) fname = self.rule_format % rid try: with open(fname, 'w') as fd: if global_rule: fd.write(global_rule) for sect in sorted(rule): fd.write('[%s]\n' % sect) for line in rule[sect]: fd.write('%s\n' % line) except IOError as e: logger.error('Failed to open "%s" for writing: %s' % (fname, e)) todel: List[str] = [] try: for fname in os.listdir(self.rule_directory): m = self.rule_pattern.match(fname) if m is not None: rid = int(m.group(1)) if rid not in inuse: todel.append(fname) for fname in todel: path = os.path.join(self.rule_directory, fname) try: os.unlink(path) except OSError as e: logger.error('Failed to remove "%s": %s' % (fname, e)) except OSError as e: logger.error('Failed to access rule_directory "%s": %s' % (self.rule_directory, e)) class Forward(NamedTuple): rid: int address: str def read_database(self, auto: List[Autoresponder]) -> List[str]: rc: List[str] = [] with DB() as db: company_list: List[int] = [] new_domains: Dict[str, BavUpdate.RID] = {} forwards: List[BavUpdate.Forward] = [] seen_domains: Set[str] = set() accepted_forwards: Set[str] = set() ctab: Dict[int, str] = {} # rc.append('fbl@%s\taccept:rid=unsubscribe' % self.fixdomain) for domain in self.domains: if domain not in seen_domains: rc.append('fbl@%s\talias:fbl@%s' % (domain, self.fixdomain)) seen_domains.add(domain) if self.fixdomain not in seen_domains: new_domains[self.fixdomain] = BavUpdate.RID( rid=0, domain=self.fixdomain) seen_domains.add(self.fixdomain) # missing = [] for row in db.query( 'SELECT company_id, mailloop_domain FROM company_tbl WHERE status = :status', {'status': 'active'}): if row.mailloop_domain: ctab[row.company_id] = row.mailloop_domain if row.mailloop_domain not in seen_domains: rc.append('fbl@%s\talias:fbl@%s' % (row.mailloop_domain, self.fixdomain)) if row.mailloop_domain not in self.mtdom and row.mailloop_domain.lower( ) != self.fqdn: new_domains[row.mailloop_domain] = BavUpdate.RID( rid=0, domain=row.mailloop_domain) seen_domains.add(row.mailloop_domain) else: missing.append(row.company_id) company_list.append(row.company_id) if missing: missing.sort() logger.debug('Missing mailloop_domain for %s' % ', '.join([str(m) for m in missing])) # seen_rids: Set[int] = set() for row in db.query( 'SELECT rid, shortname, company_id, filter_address, ' ' forward_enable, forward, ar_enable, ' ' subscribe_enable, mailinglist_id, form_id, timestamp, ' ' spam_email, spam_required, spam_forward, ' ' autoresponder_mailing_id, security_token ' 'FROM mailloop_tbl'): if row.company_id not in company_list or row.rid is None: continue seen_rids.add(row.rid) domains: List[str] = [self.fixdomain] aliases: List[str] = [] if row.filter_address is not None: for alias in listsplit(row.filter_address): if not alias.startswith(self.prefix): with Ignore(ValueError): domain_part = alias.split('@', 1)[-1] if domain_part not in domains: domains.append(domain_part) if domain_part not in self.mtdom and domain_part not in new_domains: new_domains[ domain_part] = BavUpdate.RID( rid=row.rid, domain=domain_part) aliases.append(alias) # ar_enable = False if row.ar_enable and row.autoresponder_mailing_id: if not row.security_token: logger.error( '%s: Autoresponder has mailing id, but no security token' % row.rid) else: auto.append( Autoresponder( row.rid, row.timestamp if row.timestamp is not None else datetime.now(), row.autoresponder_mailing_id, row.security_token)) ar_enable = True # try: cdomain = ctab[row.company_id] if cdomain not in domains: if cdomain in self.domains: domains.append(cdomain) else: logger.debug( 'Company\'s domain "%s" not found in mailertable' % cdomain) except KeyError: logger.debug( 'No domain for company found, further processing') extra = ['rid=%s' % row.rid] if row.company_id: extra.append('cid=%d' % row.company_id) if row.forward_enable and row.forward: extra.append('fwd=%s' % row.forward) forwards.append( BavUpdate.Forward(rid=row.rid, address=row.forward)) if row.spam_email: extra.append('spam_email=%s' % row.spam_email) if row.spam_forward: extra.append('spam_fwd=%d' % row.spam_forward) if row.spam_required: extra.append('spam_req=%d' % row.spam_required) if ar_enable: extra.append('ar=%s' % row.rid) if row.autoresponder_mailing_id: extra.append('armid=%d' % row.autoresponder_mailing_id) if row.subscribe_enable and row.mailinglist_id and row.form_id: extra.append('sub=%d:%d' % (row.mailinglist_id, row.form_id)) for domain in domains: line = '%s%s@%s\taccept:%s' % (self.prefix, row.rid, domain, ','.join(extra)) logger.debug(f'Add line: {line}') rc.append(line) if aliases and domains: for alias in aliases: rc.append('%s\talias:%s%s@%s' % (alias, self.prefix, row.rid, domains[0])) accepted_forwards.add(alias) # if seen_rids: rules: Dict[int, Dict[str, List[str]]] = {} for row in db.query( 'SELECT rid, section, pattern FROM mailloop_rule_tbl'): if row.rid in seen_rids: try: rule = rules[row.rid] except KeyError: rule = rules[row.rid] = {} try: sect = rule[row.section] except KeyError: sect = rule[row.section] = [] sect.append(row.pattern) self.update_rules(rules) # for forward in forwards: with Ignore(ValueError): fdomain = (forward.address.split('@', 1)[-1]).lower() for domain in self.mtdom: if domain == fdomain and forward.address not in accepted_forwards: logger.warning( '%s: using address "%s" with local handled domain "%s"' % (forward.rid, forward.address, domain)) refuse = [] for (domain, new_domain) in ((_d, _n) for (_d, _n) in new_domains.items() if _d == fdomain): logger.warning( '%s: try to add new domain for already existing forward address "%s" in %s, refused' % (new_domain.rid, forward.address, forward.rid)) refuse.append(domain) for domain in refuse: del new_domains[domain] # if new_domains: if self.mta.mta == 'sendmail': cmd = [self.control_sendmail, 'add'] for domain in new_domains: cmd.append(domain) logger.info(f'Found new domains, add them using {cmd}') silent_call(*cmd) logger.info('Restarting sendmail due to domain update') silent_call(self.restart_sendmail) self.read_mailertable(new_domains) return rc def read_local_files(self) -> List[str]: try: return self.file_reader(self.local_filename) except IOError as e: logger.debug('Unable to read local file %s %s' % (self.local_filename, e)) return [] def update_autoresponder(self, auto: List[Autoresponder]) -> None: newlist: List[Autoresponder] = [] for new in auto: found = None for old in self.autoresponder: if new.rid == old.rid: found = old break if not found or new.timestamp > found.timestamp: new.write_file() newlist.append(new) else: newlist.append(found) for old in self.autoresponder: for new in newlist: if old.rid == new.rid: break else: old.remove_file() self.autoresponder = newlist def rename_file(self, old_file: str, new_file: str) -> None: try: os.rename(old_file, new_file) except OSError as e: logger.error('Unable to rename %s to %s %s' % (old_file, new_file, e)) try: os.unlink(old_file) except OSError as e: logger.warning('Failed to remove temp. file %s %s' % (old_file, e)) raise error('rename_file') def update_config_file(self, new: str) -> None: if new != self.last: temp = '%s.%d' % (self.config_filename, os.getpid()) try: with open(temp, 'w') as fd: fd.write(new) self.rename_file(temp, self.config_filename) self.last = new except IOError as e: logger.error('Unable to write %s %s' % (temp, e)) raise error('update_config_file.open', e) def update(self) -> None: try: auto: List[Autoresponder] = [] new = self.read_mail_files() new += self.read_database(auto) new += self.read_local_files() self.update_autoresponder(auto) self.update_config_file('\n'.join(new) + '\n') except error as e: logger.exception('Update failed: %s' % e)
def __init__ (self, ref: ScheduleGenerate) -> None: self.ref = ref self.db = DB () self.unit = Unit ()