def _save_namespace():
mkdirp(CONF["nameserver.root.dir"])
f = open(path.join(CONF["nameserver.root.dir"], "fsimage"), 'w')
ns_copy = copy.deepcopy(namespace)
# purge the block mappings
for blockmapping in ns_copy.values():
for set_hosts in blockmapping.values():
set_hosts.clear()
pickle.dump(ns_copy, f)
pickle.dump(block_to_file, f)
f.close()
logging.info("Saved namespace metadata to %s" % f.name)
def start_data_service(config, port):
data_dir = path.join(path.abspath(config['dataserver.root.dir']), 'storage')
id_file = path.join(path.abspath(config['dataserver.root.dir']), 'id')
nameserver_address = config['nameserver.address']
dataserver_address = cat_host(gethostname(), port)
mkdirp(data_dir)
if os.path.exists(id_file):
id = open(id_file, 'r').read()
_refresh_blocks(data_dir)
else:
nameserver_conn = connect(nameserver_address)
id = nameserver_conn.root.new_ds_id(dataserver_address)
open(id_file, 'w').write(id)
logging.info("ID is %s" % id)
DataServer._id = id
DataServer._data_dir = data_dir
DataServer._config = config
t = threading.Thread(target=start_rpyc_dataserver, args=[port])
t.daemon = True
t.start()
nameserver_conn = connect(nameserver_address)
nameserver_conn.root.register(id, dataserver_address)
nameserver_conn.close()
dataserver_conn = connect(dataserver_address)
try:
while t.isAlive():
dataserver_conn.root.send_heartbeat()
dataserver_conn.root.send_block_report()
t.join(3)
except:
logging.info("Caught exception, unregistering")
nameserver_conn = connect(nameserver_address)
nameserver_conn.root.unregister(id, dataserver_address)
def main():
DISCLAIMER = """WARNING: the policy herein built is based on a set
of common features found among the current layout and
configuration of the MX hostnames associated to input
mail domains. There is no warranty that the current
settings will be kept by mail servers' owners in the
future nor that these settings are the correct ones that
really identify the recipient domain's mail servers.
A bad policy could result in messages delivery failures.
USE THIS POLICY DEFINITIONS FILE AT YOUR OWN RISK."""
parser = argparse.ArgumentParser(
description="Guess STARTTLS policies on the basis of current MX "
"hostnames settings.",
epilog="""Consume a target list of mail domains and output a \
policy definitions file for those domains. %s""" % DISCLAIMER)
parser.add_argument("-c", "--cfg", default=Config.default_cfg_path,
help="general configuration file path", metavar="file",
dest="cfg_path")
parser.add_argument("inputfile", type=argparse.FileType("r"),
default=sys.stdin, metavar="domains_list_file",
help="""file containing the list of domains to consume;
one domain on each line;
use "-" to read from stdin""")
parser.add_argument("-o", metavar="file", type=argparse.FileType("w"),
help="path where policy definitions file will be "
"written to; default: stdout", dest="outputfile")
parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
help="print some explanatory messages")
parser.add_argument("--hash-alg", default="sha256",
choices=["sha1","sha256","sha512"],
help="hash algorithm used for fingerprints matching",
dest="hash_alg")
parser.add_argument("--no-cache", dest="nocache", action="store_true",
help="ignore any cached data")
parser.add_argument("--expires", dest="expires", type=int,
metavar="minutes",
help="policy expiration time, in minutes "
"(default: 10080, 1 week)", default=10080)
avoid_choices = ["ta", "ee_pubkey", "ee_certificate", "valid"]
parser.add_argument("--avoid-cert-matching", metavar="policy_type",
choices=avoid_choices,
dest="avoid",
help="do not use these policy types for certificate "
"matching; allowed values: " + \
", ".join(avoid_choices), nargs="*")
args = parser.parse_args()
Config.read(args.cfg_path)
global CACHE_DIR
CACHE_DIR = Config.get("general", "guessstarttlspolicy_cache_dir")
if not os.path.isdir(CACHE_DIR):
mkdirp(CACHE_DIR)
if not os.access(CACHE_DIR, os.W_OK):
raise InsufficientPermissionError("Insufficient permissions to write "
"into GuessSTARTTLSPolicies cache dir "
"(guessstarttlspolicy_cache_dir): %s" %
CACHE_DIR)
hash_alg = args.hash_alg
if args.avoid:
avoid_cert_matching = args.avoid
else:
avoid_cert_matching = []
check_ko = []
expires = datetime.datetime.utcnow() + \
datetime.timedelta(minutes=args.expires)
output = {
"version": "0.1",
"timestamp": datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S"),
"author": "GuessSTARTTLSPolicies on %s - "
"USE THIS POLICY AT YOUR OWN RISK" % platform.node(),
"expires": expires.strftime("%Y-%m-%dT%H:%M:%S"),
"tls-policies": {}
}
def verbose(s):
if args.verbose:
print(s)
for domain in args.inputfile.readlines():
mail_domain = domain.strip()
verbose("Analysing domain %s..." % mail_domain)
verbose("")
check_ko, domain_data = collect(mail_domain, args.nocache)
if len(check_ko) > 0:
verbose(" One or more MX hosts can't be analysed:")
for mx_hostname, failure in check_ko:
verbose(" %s: %s" % (mx_hostname,failure))
verbose("")
continue
if not domain_data:
verbose(" Can't get information about any MX host for %s" % mail_domain)
verbose("")
continue
common = {}
# ----------------------------------------------
verbose(" Highest common TLS version...")
common["min-tls-version"] = None
for mx_hostname in domain_data["mx-hostnames"]:
mx_host = domain_data["mx-hostnames"][mx_hostname]
tls_ver = mx_host["tls-version"]
verbose(" %s supports %s" % (mx_hostname,tls_ver))
if not common["min-tls-version"]:
common["min-tls-version"] = tls_ver
else:
if not tls_ver in tls_protocols_higher_than(common["min-tls-version"]):
common["min-tls-version"] = tls_ver
verbose(" min-tls-version: %s" % common["min-tls-version"])
verbose("")
# ----------------------------------------------
common["ta_certificate"] = None
common["ta_pubkey"] = None
common["ee_certificate"] = None
common["ee_pubkey"] = None
for descr, ee_ta, dest, pem, fp in [("trust anchor certificate",
"ta",
"ta_certificate",
"certificate_pem",
"certificate_fingerprints"),
("trust anchor public key",
"ta",
"ta_pubkey",
"pubkey_pem",
"pubkey_fingerprints"),
("leaf certificate",
"ee",
"ee_certificate",
"certificate_pem",
"certificate_fingerprints"),
("leaf certificate public key",
"ee",
"ee_pubkey",
"pubkey_pem",
"pubkey_fingerprints")]:
verbose(" Common %s..." % descr)
for mx_hostname in domain_data["mx-hostnames"]:
mx_host = domain_data["mx-hostnames"][mx_hostname]
if not ee_ta in mx_host["certificates"]:
verbose(" no %s certificate found for %s" % (ee_ta.upper(),
mx_hostname))
common[dest] = None
break
cert = mx_host["certificates"][ee_ta]
verbose(" %s %s's fingerprint: %s" % (mx_hostname, descr,
cert[fp][hash_alg]))
if not common[dest]:
common[dest] = {}
common[dest][pem] = cert[pem]
common[dest][fp] = cert[fp]
elif common[dest][pem] != cert[pem]:
common[dest] = None
break
if common[dest]:
verbose(" Common %s found: fingerprint %s" %
(descr,common[dest][fp][hash_alg]))
else:
verbose(" No common %s found" % descr)
verbose("")
# ----------------------------------------------
verbose(" Any invalid EE certificates...")
common["any-invalid-EE-cert"] = False
for mx_hostname in domain_data["mx-hostnames"]:
mx_host = domain_data["mx-hostnames"][mx_hostname]
if not mx_host["certificates"]["ee"]["verify_ok"]:
verbose(" %s: not valid (%s)" %
(mx_hostname, mx_host["certificates"]["ee"]["verify_res"]))
common["any-invalid-EE-cert"] = True
else:
verbose(" %s: valid" % mx_hostname)
if common["any-invalid-EE-cert"]:
verbose(" Invalid EE certificates found")
else:
verbose(" No invalid EE certificates found")
verbose("")
# ----------------------------------------------
verbose(" Common names in EE certificates...")
common["shortest_names"] = []
pdoms = {}
for mx_hostname in domain_data["mx-hostnames"]:
mx_host = domain_data["mx-hostnames"][mx_hostname]
verbose(" %s: %s" % (mx_hostname,
", ".join(mx_host["certificates"]["ee"]["names"])))
for name in mx_host["certificates"]["ee"]["names"]:
lbls = name.split(".")
for dom_len in range(2, len(lbls)+1):
pdom = ".".join(lbls[-dom_len:])
if dom_len != len(lbls):
pdom = "." + pdom
if not str(dom_len) in pdoms:
pdoms[str(dom_len)] = {}
if not pdom in pdoms[str(dom_len)]:
pdoms[str(dom_len)][pdom] = [mx_hostname]
elif not mx_hostname in pdoms[str(dom_len)][pdom]:
pdoms[str(dom_len)][pdom].append(mx_hostname)
common_names = {}
for dom_len in pdoms.keys():
for name in pdoms[dom_len].keys():
if len(pdoms[dom_len][name]) == len(domain_data["mx-hostnames"]):
if not dom_len in common_names:
common_names[dom_len] = []
common_names[dom_len].append(name)
if len(common_names.keys()) > 0:
min_len = sorted([int(x) for x in common_names.keys()])[0]
common["shortest_names"] = common_names[str(min_len)]
verbose(" Common shortest names: " + ", ".join(common["shortest_names"]))
else:
verbose(" No common names found in EE certificates")
# ----------------------------------------------
# Decisions follow
policy = {}
def add_tlsas(ee_ta,entity):
assert(ee_ta in [ "ee", "ta" ])
assert(entity in [ "pubkey", "certificate" ])
# add both full entity (base64 PEM) and it's fingerprint
policy["%s-tlsa" % ee_ta].append({
"entity": entity,
"data_format": "b64",
"data": common["%s_%s" %
(ee_ta,entity)]["%s_pem" % entity]
})
policy["%s-tlsa" % ee_ta].append({
"entity": entity,
"hash_alg": hash_alg,
"data_format": "hex",
"data": common["%s_%s" %
(ee_ta,entity)]["%s_fingerprints" % entity][hash_alg]
})
verbose("")
if common["ta_certificate"] or common["ta_pubkey"]:
if len(common["shortest_names"]) > 0:
if "ta" in avoid_cert_matching:
verbose(" Common trust anchor found "
"but forbidden by user's choice: "
"--avoid-cert-matching ta")
else:
verbose(" Certificate matching based on common trust anchor.")
policy["certificate-matching"] = "TA"
policy["ta-tlsa"] = []
if common["ta_certificate"]:
add_tlsas("ta", "certificate")
if common["ta_pubkey"]:
add_tlsas("ta", "pubkey")
else:
verbose(" WARNING: even if domain's MX hosts share a common "
"trust anchor it can't be used for certificate "
"matching because no common EE certificate names have "
"be found. ")
if "certificate-matching" not in policy and common["ee_pubkey"]:
if "ee_pubkey" in avoid_cert_matching:
verbose(" Common EE certificates' public keys found "
"but forbidden by user's choice: "
"--avoid-cert-matching ee_pubkey")
else:
verbose(" Certificate matching based on the common EE certificates' "
"public key.")
policy["certificate-matching"] = "EE"
policy["ee-tlsa"] = []
add_tlsas("ee", "pubkey")
if "certificate-matching" not in policy and common["ee_certificate"]:
if "ee_certificate" in avoid_cert_matching:
verbose(" Common EE certificates found "
"but forbidden by user's choice: "
"--avoid-cert-matching ee_certificate")
else:
verbose(" Certificate matching based on common EE certificates.")
policy["certificate-matching"] = "EE"
policy["ee-tlsa"] = []
add_tlsas("ee", "certificate")
if "certificate-matching" not in policy and \
common["shortest_names"] != [] and not common["any-invalid-EE-cert"]:
verbose(" No common TA or EE certificate have been found among domain's "
"MX hosts.")
if "valid" in avoid_cert_matching:
verbose(" Certificate matching based on any valid certificate "
"would be used "
"but it's forbidden by user's choice: "
"--avoid-cert-matching valid")
else:
verbose(" Certificate matching based on any valid certificate "
"with a matching name.")
policy["certificate-matching"] = "valid"
if "certificate-matching" not in policy:
verbose(" WARNING: no common certificates' trust anchors nor common "
"EE valid certificates have been found. TLS will be enforced "
"but no authentication will be provided.")
if "certificate-matching" in policy:
if policy["certificate-matching"] in [ "TA", "valid"]:
policy["allowed-cert-names"] = copy.copy(common["shortest_names"])
if mail_domain in policy["allowed-cert-names"]:
policy["allowed-cert-names"].remove(mail_domain)
if policy["allowed-cert-names"] == []:
policy.pop("allowed-cert-names", None)
if common["min-tls-version"]:
policy["min-tls-version"] = common["min-tls-version"]
output["tls-policies"][mail_domain] = copy.deepcopy(policy)
verbose("")
# print(json.dumps(common,indent=2))
# print json.dumps(domain_data,indent=2)
# print(json.dumps(policy,indent=2))
if args.outputfile:
args.outputfile.write(json.dumps(output,indent=2))
verbose("Policy definitions written to the output file.")
args.outputfile.close()
else:
verbose("Policy definitions follow:")
verbose("")
print(json.dumps(output,indent=2))
verbose("")
def collect(mail_domain, ignore_cache):
"""
Attempt to connect to each MX hostname for mail_doman and negotiate STARTTLS.
Store the output in a directory with the same name as mail_domain to make
subsequent analysis faster.
Return a set:
list of failed MX hosts
dictionary with data
"""
mkdirp(os.path.join(CACHE_DIR, mail_domain))
cache_file = os.path.join(CACHE_DIR, mail_domain, "data.json")
if not ignore_cache and os.path.exists(cache_file):
if os.path.getmtime(cache_file) >= time.time() - CACHE_TIME:
with open(cache_file, "r") as f:
res = json.loads(f.read())
return [], res
res = { "mx-hostnames": {} }
check_ko = []
try:
answers = dns.resolver.query(mail_domain, "MX")
except:
return [], None
for rdata in answers:
mx_host = str(rdata.exchange).rstrip(".")
mx_host_data = {}
try:
openssl_output = tls_connect(mx_host, mail_domain, ignore_cache)
except ( CheckSTARTTLSSupportError, SSLCertificatesError ) as e:
check_ko.append( ( mx_host, str(e) ) )
continue
if not openssl_output:
check_ko.append( ( mx_host, "STARTTLS not supported" ) )
continue
# TLS protocol version
protocol = re.findall("Protocol\s+:\s+(.*)", openssl_output)[0]
if not protocol in [ "TLSv1", "TLSv1.1", "TLSv1.2" ]:
raise ValueError("Unknown TLS protocol version for %s: %s" %
(mx_host,protocol))
mx_host_data["tls-version"] = protocol
mx_host_data["certificates"] = get_certs_info(openssl_output)
res["mx-hostnames"][mx_host] = mx_host_data
if len(check_ko) == 0:
with open(cache_file, "w") as f:
f.write(json.dumps(res,indent=2))
else:
if os.path.exists(cache_file):
os.remove(cache_file)
return check_ko, res
def main():
parser = argparse.ArgumentParser(
description="""MTA log watcher""",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Incremental reading is not available when logfile = "-" (stdin).
If a policy definitions file is supplied (-p argument) the output counters are
incremented only for logfile lines that match one of the mail domains covered
by the policy.
Output type:
- matched-lines: only the lines that have been analysed will be shown.
- unmatched-lines: only the lines that have not been included in the analysis
will be shown; this option can be useful to evaluate the effectiveness of
log parsing patterns and to display log lines that have been ignored.
- domains: all the domains that have been analysed are shown, with counters
of successful and failed delivery attempts.
- warnings: like for 'domains', but only mail domains with a failure rate that
is higher than the configured threshold are shown.""")
parser.add_argument("-c", "--cfg", default=Config.default_cfg_path,
help="general configuration file path", metavar="file",
dest="cfg_path")
parser.add_argument("-m", default="Postfix",
help="MTA flavor", choices=["Postfix"],
dest="mta_flavor")
parser.add_argument("logfile", help="MTA's log file to analyze; "
"a dash ('-') means read from stdin")
parser.add_argument("-i", "--incremental", action="store_true",
dest="incremental", help="read file incrementally")
parser.add_argument("--remove-cursor", action="store_true",
dest="remove_cursor",
help="remove the file containing the cursor used for "
"incrementally reading the logfile")
parser.add_argument("--show-cursor", action="store_true",
dest="show_cursor",
help="show the file containing the cursor used for "
"incrementally reading the logfile")
output_choices = ["warnings", "domains", "summary",
"matched-lines", "unmatched-lines"]
parser.add_argument("-o", default="warnings", dest="output",
choices=output_choices,
metavar="output-type",
help="requested output: " +
" | ".join("'" + c + "'" for c in output_choices))
parser.add_argument("-p",
help="JSON policy definitions file",
dest="policy_defs",
metavar="policy_defs.json")
args = parser.parse_args()
Config.read(args.cfg_path)
if args.output == "warnings":
# Reporting facilities initialization
reports_dir = Config.get("general","logwatcher_reports_dir")
if not os.path.isdir(reports_dir):
mkdirp(reports_dir)
#raise MissingFileError("Logwatcher's reports directory "
# "(logwatcher_reports_dir) not found: %s" %
# reports_dir)
if not os.access(reports_dir, os.W_OK):
raise InsufficientPermissionError("Insufficient permissions to write "
"into logwatcher's reports "
"directory "
"(logwatcher_reports_dir): %s" %
reports_dir)
Config.get_logger()
# failure_threshold = failure_threshold_percent / 100
# 1 = 100%
# 0.001 = 0.1%
failure_threshold = Config.get("general","failure_threshold_percent")
try:
failure_threshold = float(failure_threshold)/100
except:
raise TypeError("Invalid failure threshold: %s" % failure_threshold)
if failure_threshold < 0 or failure_threshold > 1:
raise ValueError("Failure threshold must be between 0 and 100: %s" %
failure_threshold)
if args.logfile == "-":
if args.incremental:
print("Can't use incremental reading on stdin.")
return
if args.remove_cursor or args.show_cursor:
print("Can't manage cursors for stdin.")
return
if args.policy_defs:
policy_defs = DefsParser.Defs(args.policy_defs)
else:
policy_defs = None
if args.mta_flavor == "Postfix":
logwatcher = PostfixLogWatcher(args.logfile,args.incremental,policy_defs)
else:
print("Unexpected MTA flavor: {}".format(args.mta_flavor))
return
if args.remove_cursor:
print(logwatcher.remove_cursor())
return
if args.show_cursor:
print(logwatcher.show_cursor())
return
res = logwatcher.analyze_lines(logwatcher.get_newlines())
if args.output == "summary":
print("Displaying the summary accordingly to logfile parsing results")
print("")
for s in logwatcher.status_tags:
if s in res:
print("%s:" % s)
print(json.dumps(res[s],indent=2))
print("Domains:")
print(json.dumps(res["domains"],indent=2))
elif args.output == "matched-lines":
print("Displaying the logfile's lines that matched configured patterns")
print("")
for l in res["matched_lines"]:
print(l.rstrip("\n"))
elif args.output == "unmatched-lines":
print("Displaying the logfile's lines that did not match "
"configured patterns")
print("")
for l in res["unmatched_lines"]:
print(l.rstrip("\n"))
elif args.output in [ "domains", "warnings" ]:
print("Displaying successful/failed delivery attempts for %s" %
("every domain" if args.output == "domains" else
"domains with an high failure rate (%s%%)" %
(failure_threshold*100)))
print("")
warning_domains = []
for domainname in res["domains"]:
domain = res["domains"][domainname]
if not "attempted" in domain:
continue
#TODO: implement results for "log-only = true" status.
if "sent_ko" in domain and domain["attempted"] > 0:
failure_rate = domain["sent_ko"] / domain["attempted"]
else:
failure_rate = None
if args.output == "domains" or \
( args.output == "warnings" and failure_rate >= failure_threshold ):
succeeded = domain["sent_ok"] if "sent_ok" in domain else "none"
failed = domain["sent_ko"] if "sent_ko" in domain else "none"
s = "{d}: {t} delivery attempts, {s} succeeded, {f} failed"
if failure_rate:
s = s + ", {r:.2%} failure rate"
if failure_rate >= failure_threshold:
s = s + " - WARNING"
warning_domains.append(domainname)
print(s.format(d=domainname, r=failure_rate,
t=domain["attempted"], s=succeeded,
f=failed))
if args.output == "warnings" and len(warning_domains) > 0:
report_format= Config.get("general","logwatcher_reports_fmt")
report_filename = datetime.datetime.now().strftime(report_format)
report_file = "%s/%s" % (reports_dir,report_filename)
with open(report_file, "w") as r:
r.write("domainname,attempts,ko,ok\n")
for domainname in warning_domains:
r.write("{domainname},{attempted},{sent_ko},{sent_ok}\n".format(
domainname=domainname,
attempted=res["domains"][domainname]["attempted"],
sent_ko=res["domains"][domainname].get("sent_ko",0),
sent_ok=res["domains"][domainname].get("sent_ok",0)))
notification_t = "Delivery errors found for {domains} for a " + \
"total of {fail} failures over {tot} total " + \
"attempts. More details on {report_file}"
if len(warning_domains) > 3:
notification_domains = ", ".join(warning_domains[:3]) + \
"and " + str(len(warning_domains)-3) + \
" more domains"
else:
notification_domains = ", ".join(warning_domains)
fail = 0
tot = 0
for domainname in warning_domains:
fail = fail + res["domains"][domainname]["sent_ko"]
tot = tot + res["domains"][domainname]["attempted"]
notification = notification_t.format(domains=notification_domains,
fail=fail,
tot=tot,
report_file=report_file)
Config.get_logger().error(notification)
