示例#1
0
def _save_namespace():
    mkdirp(CONF["nameserver.root.dir"])
    f = open(path.join(CONF["nameserver.root.dir"], "fsimage"), 'w')

    ns_copy = copy.deepcopy(namespace)

    # purge the block mappings
    for blockmapping in ns_copy.values():
        for set_hosts in blockmapping.values():
            set_hosts.clear()

    pickle.dump(ns_copy, f)
    pickle.dump(block_to_file, f)
    f.close()
    logging.info("Saved namespace metadata to %s" % f.name)
示例#2
0
def start_data_service(config, port):
    data_dir = path.join(path.abspath(config['dataserver.root.dir']), 'storage')
    id_file = path.join(path.abspath(config['dataserver.root.dir']), 'id')
    nameserver_address = config['nameserver.address']
    dataserver_address = cat_host(gethostname(), port)
    mkdirp(data_dir)

    if os.path.exists(id_file):
        id = open(id_file, 'r').read()
        _refresh_blocks(data_dir)
    else:
        nameserver_conn = connect(nameserver_address)
        id = nameserver_conn.root.new_ds_id(dataserver_address)
        open(id_file, 'w').write(id)

    logging.info("ID is %s" % id)

    DataServer._id = id
    DataServer._data_dir = data_dir
    DataServer._config = config

    t = threading.Thread(target=start_rpyc_dataserver, args=[port])
    t.daemon = True
    t.start()

    nameserver_conn = connect(nameserver_address)
    nameserver_conn.root.register(id, dataserver_address)
    nameserver_conn.close()

    dataserver_conn = connect(dataserver_address)

    try:
        while t.isAlive():
            dataserver_conn.root.send_heartbeat()
            dataserver_conn.root.send_block_report()
            t.join(3)

    except:
        logging.info("Caught exception, unregistering")
        nameserver_conn = connect(nameserver_address)
        nameserver_conn.root.unregister(id, dataserver_address)
def main():
  DISCLAIMER = """WARNING: the policy herein built is based on a set
                  of common features found among the current layout and
                  configuration of the MX hostnames associated to input
                  mail domains. There is no warranty that the current
                  settings will be kept by mail servers' owners in the
                  future nor that these settings are the correct ones that
                  really identify the recipient domain's mail servers.
                  A bad policy could result in messages delivery failures.
                  USE THIS POLICY DEFINITIONS FILE AT YOUR OWN RISK."""

  parser = argparse.ArgumentParser(
    description="Guess STARTTLS policies on the basis of current MX "
      "hostnames settings.",
    epilog="""Consume a target list of mail domains and output a \
    policy definitions file for those domains. %s""" % DISCLAIMER)

  parser.add_argument("-c", "--cfg", default=Config.default_cfg_path,
                      help="general configuration file path", metavar="file",
                      dest="cfg_path")

  parser.add_argument("inputfile", type=argparse.FileType("r"),
                      default=sys.stdin, metavar="domains_list_file",
                      help="""file containing the list of domains to consume;
                      one domain on each line;
                      use "-" to read from stdin""")

  parser.add_argument("-o", metavar="file", type=argparse.FileType("w"),
                      help="path where policy definitions file will be "
                      "written to; default: stdout", dest="outputfile")

  parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
                      help="print some explanatory messages")

  parser.add_argument("--hash-alg", default="sha256",
                      choices=["sha1","sha256","sha512"],
                      help="hash algorithm used for fingerprints matching",
                      dest="hash_alg")

  parser.add_argument("--no-cache", dest="nocache", action="store_true",
                      help="ignore any cached data")

  parser.add_argument("--expires", dest="expires", type=int,
                      metavar="minutes",
                      help="policy expiration time, in minutes "
                      "(default: 10080, 1 week)", default=10080)

  avoid_choices = ["ta", "ee_pubkey", "ee_certificate", "valid"]
  parser.add_argument("--avoid-cert-matching", metavar="policy_type",
                      choices=avoid_choices,
                      dest="avoid",
                      help="do not use these policy types for certificate "
                      "matching; allowed values: " + \
                      ", ".join(avoid_choices), nargs="*")
  args = parser.parse_args()

  Config.read(args.cfg_path)

  global CACHE_DIR
  CACHE_DIR = Config.get("general", "guessstarttlspolicy_cache_dir")

  if not os.path.isdir(CACHE_DIR):
    mkdirp(CACHE_DIR)
  if not os.access(CACHE_DIR, os.W_OK):
    raise InsufficientPermissionError("Insufficient permissions to write "
                                      "into GuessSTARTTLSPolicies cache dir "
                                      "(guessstarttlspolicy_cache_dir): %s" %
                                      CACHE_DIR)

  hash_alg = args.hash_alg

  if args.avoid:
    avoid_cert_matching = args.avoid
  else:
    avoid_cert_matching = []

  check_ko = []

  expires = datetime.datetime.utcnow() + \
            datetime.timedelta(minutes=args.expires)

  output = {
    "version": "0.1",
    "timestamp": datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%S"),
    "author": "GuessSTARTTLSPolicies on %s - "
              "USE THIS POLICY AT YOUR OWN RISK" % platform.node(),
    "expires": expires.strftime("%Y-%m-%dT%H:%M:%S"), 
    "tls-policies": {}
  }

  def verbose(s):
    if args.verbose:
      print(s)

  for domain in args.inputfile.readlines():
    mail_domain = domain.strip()

    verbose("Analysing domain %s..." % mail_domain)
    verbose("")

    check_ko, domain_data = collect(mail_domain, args.nocache)

    if len(check_ko) > 0:
      verbose(" One or more MX hosts can't be analysed:")
      for mx_hostname, failure in check_ko:
        verbose("  %s: %s" % (mx_hostname,failure))
      verbose("")
      continue

    if not domain_data:
      verbose(" Can't get information about any MX host for %s" % mail_domain)
      verbose("")
      continue

    common = {}

    # ----------------------------------------------

    verbose(" Highest common TLS version...")
    common["min-tls-version"] = None
    for mx_hostname in domain_data["mx-hostnames"]:
      mx_host = domain_data["mx-hostnames"][mx_hostname]
      tls_ver = mx_host["tls-version"]
      verbose("  %s supports %s" % (mx_hostname,tls_ver))
      if not common["min-tls-version"]:
        common["min-tls-version"] = tls_ver
      else:
        if not tls_ver in tls_protocols_higher_than(common["min-tls-version"]):
          common["min-tls-version"] = tls_ver

    verbose(" min-tls-version: %s" % common["min-tls-version"])
    verbose("")

    # ----------------------------------------------

    common["ta_certificate"] = None
    common["ta_pubkey"] = None
    common["ee_certificate"] = None
    common["ee_pubkey"] = None

    for descr, ee_ta, dest, pem, fp in [("trust anchor certificate",
                                         "ta",
                                         "ta_certificate",
                                         "certificate_pem",
                                         "certificate_fingerprints"),
                                        ("trust anchor public key",
                                         "ta",
                                         "ta_pubkey",
                                         "pubkey_pem",
                                         "pubkey_fingerprints"),
                                        ("leaf certificate",
                                         "ee",
                                         "ee_certificate",
                                         "certificate_pem",
                                         "certificate_fingerprints"),
                                        ("leaf certificate public key",
                                         "ee",
                                         "ee_pubkey",
                                         "pubkey_pem",
                                         "pubkey_fingerprints")]:

      verbose(" Common %s..." % descr)
      for mx_hostname in domain_data["mx-hostnames"]:
        mx_host = domain_data["mx-hostnames"][mx_hostname]
        if not ee_ta in mx_host["certificates"]:
          verbose("  no %s certificate found for %s" % (ee_ta.upper(),
                                                      mx_hostname))
          common[dest] = None
          break

        cert = mx_host["certificates"][ee_ta]
        verbose("  %s %s's fingerprint: %s" % (mx_hostname, descr,
                                             cert[fp][hash_alg]))
        if not common[dest]:
          common[dest] = {}
          common[dest][pem] = cert[pem]
          common[dest][fp] = cert[fp]
        elif common[dest][pem] != cert[pem]:
          common[dest] = None
          break

      if common[dest]:
        verbose(" Common %s found: fingerprint %s" %
              (descr,common[dest][fp][hash_alg]))
      else:
        verbose(" No common %s found" % descr)
      verbose("")

    # ----------------------------------------------

    verbose(" Any invalid EE certificates...")

    common["any-invalid-EE-cert"] = False

    for mx_hostname in domain_data["mx-hostnames"]:
      mx_host = domain_data["mx-hostnames"][mx_hostname]

      if not mx_host["certificates"]["ee"]["verify_ok"]:
        verbose("  %s: not valid (%s)" %
                (mx_hostname, mx_host["certificates"]["ee"]["verify_res"]))
        common["any-invalid-EE-cert"] = True
      else:
        verbose("  %s: valid" % mx_hostname)

    if common["any-invalid-EE-cert"]:
      verbose(" Invalid EE certificates found")
    else:
      verbose(" No invalid EE certificates found")
    verbose("")

    # ----------------------------------------------

    verbose(" Common names in EE certificates...")

    common["shortest_names"] = []

    pdoms = {}

    for mx_hostname in domain_data["mx-hostnames"]:
      mx_host = domain_data["mx-hostnames"][mx_hostname]
      verbose("  %s: %s" % (mx_hostname,
                          ", ".join(mx_host["certificates"]["ee"]["names"])))
      for name in mx_host["certificates"]["ee"]["names"]:
        lbls = name.split(".")

        for dom_len in range(2, len(lbls)+1):
          pdom = ".".join(lbls[-dom_len:])
          if dom_len != len(lbls):
            pdom = "." + pdom

          if not str(dom_len) in pdoms:
            pdoms[str(dom_len)] = {}

          if not pdom in pdoms[str(dom_len)]:
            pdoms[str(dom_len)][pdom] = [mx_hostname]
          elif not mx_hostname in pdoms[str(dom_len)][pdom]:
            pdoms[str(dom_len)][pdom].append(mx_hostname)

    common_names = {}
    for dom_len in pdoms.keys():
      for name in pdoms[dom_len].keys():
        if len(pdoms[dom_len][name]) == len(domain_data["mx-hostnames"]):
          if not dom_len in common_names:
            common_names[dom_len] = []
          common_names[dom_len].append(name)

    if len(common_names.keys()) > 0:
      min_len = sorted([int(x) for x in common_names.keys()])[0]
      common["shortest_names"] = common_names[str(min_len)]
      verbose(" Common shortest names: " + ", ".join(common["shortest_names"]))
    else:
      verbose(" No common names found in EE certificates")

    # ----------------------------------------------
    # Decisions follow

    policy = {}

    def add_tlsas(ee_ta,entity):
      assert(ee_ta in [ "ee", "ta" ])
      assert(entity in [ "pubkey", "certificate" ])

      # add both full entity (base64 PEM) and it's fingerprint
      policy["%s-tlsa" % ee_ta].append({
        "entity": entity,
        "data_format": "b64",
        "data": common["%s_%s" %
                       (ee_ta,entity)]["%s_pem" % entity]
      })
      policy["%s-tlsa" % ee_ta].append({
        "entity": entity,
        "hash_alg": hash_alg,
        "data_format": "hex",
        "data": common["%s_%s" %
                       (ee_ta,entity)]["%s_fingerprints" % entity][hash_alg]
      })

    verbose("")

    if common["ta_certificate"] or common["ta_pubkey"]:

      if len(common["shortest_names"]) > 0:
        if "ta" in avoid_cert_matching:
          verbose(" Common trust anchor found "
                  "but forbidden by user's choice: "
                  "--avoid-cert-matching ta")
        else:
          verbose(" Certificate matching based on common trust anchor.")
          policy["certificate-matching"] = "TA"
          policy["ta-tlsa"] = []

          if common["ta_certificate"]:
            add_tlsas("ta", "certificate")
          if common["ta_pubkey"]:
            add_tlsas("ta", "pubkey")
      else:
        verbose(" WARNING: even if domain's MX hosts share a common "
                "trust anchor it can't be used for certificate "
                "matching because no common EE certificate names have "
                "be found. ")

    if "certificate-matching" not in policy and common["ee_pubkey"]:

      if "ee_pubkey" in avoid_cert_matching:
        verbose(" Common EE certificates' public keys found "
                "but forbidden by user's choice: "
                "--avoid-cert-matching ee_pubkey")
      else:
        verbose(" Certificate matching based on the common EE certificates' "
                "public key.")
        policy["certificate-matching"] = "EE"
        policy["ee-tlsa"] = []

        add_tlsas("ee", "pubkey")

    if "certificate-matching" not in policy and common["ee_certificate"]:

      if "ee_certificate" in avoid_cert_matching:
        verbose(" Common EE certificates found "
                "but forbidden by user's choice: "
                "--avoid-cert-matching ee_certificate")
      else:
        verbose(" Certificate matching based on common EE certificates.")
        policy["certificate-matching"] = "EE"
        policy["ee-tlsa"] = []

        add_tlsas("ee", "certificate")

    if "certificate-matching" not in policy and \
      common["shortest_names"] != [] and not common["any-invalid-EE-cert"]:

      verbose(" No common TA or EE certificate have been found among domain's "
              "MX hosts.")

      if "valid" in avoid_cert_matching:
        verbose(" Certificate matching based on any valid certificate "
                "would be used "
                "but it's forbidden by user's choice: "
                "--avoid-cert-matching valid")
      else:
        verbose(" Certificate matching based on any valid certificate "
                "with a matching name.")
        policy["certificate-matching"] = "valid"
    
    if "certificate-matching" not in policy:
      verbose(" WARNING: no common certificates' trust anchors nor common "
              "EE valid certificates have been found. TLS will be enforced "
              "but no authentication will be provided.")

    if "certificate-matching" in policy:
      if policy["certificate-matching"] in [ "TA", "valid"]:
        policy["allowed-cert-names"] = copy.copy(common["shortest_names"])
        if mail_domain in policy["allowed-cert-names"]:
          policy["allowed-cert-names"].remove(mail_domain)
          if policy["allowed-cert-names"] == []:
            policy.pop("allowed-cert-names", None)

    if common["min-tls-version"]:
      policy["min-tls-version"] = common["min-tls-version"]

    output["tls-policies"][mail_domain] = copy.deepcopy(policy)
    verbose("")

#    print(json.dumps(common,indent=2))
#    print json.dumps(domain_data,indent=2)
#    print(json.dumps(policy,indent=2))

  if args.outputfile:
    args.outputfile.write(json.dumps(output,indent=2))
    verbose("Policy definitions written to the output file.")
    args.outputfile.close()
  else:
    verbose("Policy definitions follow:")
    verbose("")
    print(json.dumps(output,indent=2))
    verbose("")
def collect(mail_domain, ignore_cache):
  """
  Attempt to connect to each MX hostname for mail_doman and negotiate STARTTLS.
  Store the output in a directory with the same name as mail_domain to make
  subsequent analysis faster.

  Return a set:
    list of failed MX hosts
    dictionary with data
  """

  mkdirp(os.path.join(CACHE_DIR, mail_domain))
  cache_file = os.path.join(CACHE_DIR, mail_domain, "data.json")

  if not ignore_cache and os.path.exists(cache_file):
    if os.path.getmtime(cache_file) >= time.time() - CACHE_TIME:
      with open(cache_file, "r") as f:
        res = json.loads(f.read())
      return [], res

  res = { "mx-hostnames": {} }
  check_ko = []

  try:
    answers = dns.resolver.query(mail_domain, "MX")
  except:
    return [], None

  for rdata in answers:
    mx_host = str(rdata.exchange).rstrip(".")

    mx_host_data = {}

    try:
      openssl_output = tls_connect(mx_host, mail_domain, ignore_cache)
    except ( CheckSTARTTLSSupportError, SSLCertificatesError ) as e:
      check_ko.append( ( mx_host, str(e) ) )
      continue

    if not openssl_output:
      check_ko.append( ( mx_host, "STARTTLS not supported" ) )
      continue

    # TLS protocol version
    protocol = re.findall("Protocol\s+:\s+(.*)", openssl_output)[0]

    if not protocol in [ "TLSv1", "TLSv1.1", "TLSv1.2" ]:
      raise ValueError("Unknown TLS protocol version for %s: %s" %
                       (mx_host,protocol))

    mx_host_data["tls-version"] = protocol
    mx_host_data["certificates"] = get_certs_info(openssl_output)

    res["mx-hostnames"][mx_host] = mx_host_data

  if len(check_ko) == 0:
    with open(cache_file, "w") as f:
      f.write(json.dumps(res,indent=2))
  else:
    if os.path.exists(cache_file):
      os.remove(cache_file)
 
  return check_ko, res
  def main():
    parser = argparse.ArgumentParser(
      description="""MTA log watcher""",
      formatter_class=argparse.RawDescriptionHelpFormatter,
      epilog="""
Incremental reading is not available when logfile = "-" (stdin).

If a policy definitions file is supplied (-p argument) the output counters are
incremented only for logfile lines that match one of the mail domains covered
by the policy.

Output type:
- matched-lines: only the lines that have been analysed will be shown.
- unmatched-lines: only the lines that have not been included in the analysis
  will be shown; this option can be useful to evaluate the effectiveness of 
  log parsing patterns and to display log lines that have been ignored.
- domains: all the domains that have been analysed are shown, with counters
  of successful and failed delivery attempts.
- warnings: like for 'domains', but only mail domains with a failure rate that
  is higher than the configured threshold are shown.""")

    parser.add_argument("-c", "--cfg", default=Config.default_cfg_path,
                        help="general configuration file path", metavar="file",
                        dest="cfg_path")

    parser.add_argument("-m", default="Postfix",
                        help="MTA flavor", choices=["Postfix"],
                        dest="mta_flavor")

    parser.add_argument("logfile", help="MTA's log file to analyze; "
                        "a dash ('-') means read from stdin")

    parser.add_argument("-i", "--incremental", action="store_true",
                        dest="incremental", help="read file incrementally")

    parser.add_argument("--remove-cursor", action="store_true",
                        dest="remove_cursor",
                        help="remove the file containing the cursor used for "
                             "incrementally reading the logfile")

    parser.add_argument("--show-cursor", action="store_true",
                        dest="show_cursor",
                        help="show the file containing the cursor used for "
                             "incrementally reading the logfile")

    output_choices = ["warnings", "domains", "summary",
                              "matched-lines", "unmatched-lines"]

    parser.add_argument("-o", default="warnings", dest="output",
                        choices=output_choices,
                        metavar="output-type",
                        help="requested output: " + 
                        " | ".join("'" + c + "'" for c in output_choices))

    parser.add_argument("-p",
                        help="JSON policy definitions file",
                        dest="policy_defs",
                        metavar="policy_defs.json")

    args = parser.parse_args()

    Config.read(args.cfg_path)

    if args.output == "warnings":
      # Reporting facilities initialization

      reports_dir = Config.get("general","logwatcher_reports_dir")
      if not os.path.isdir(reports_dir):
        mkdirp(reports_dir)
        #raise MissingFileError("Logwatcher's reports directory "
        #                        "(logwatcher_reports_dir) not found: %s" %
        #                        reports_dir)
      if not os.access(reports_dir, os.W_OK):
        raise InsufficientPermissionError("Insufficient permissions to write "
                                          "into logwatcher's reports "
                                          "directory "
                                          "(logwatcher_reports_dir): %s" %
                                          reports_dir)

      Config.get_logger()

    # failure_threshold = failure_threshold_percent / 100
    #   1 = 100%
    #   0.001 = 0.1%
    failure_threshold = Config.get("general","failure_threshold_percent")
    try:
      failure_threshold = float(failure_threshold)/100
    except:
      raise TypeError("Invalid failure threshold: %s" % failure_threshold)

    if failure_threshold < 0 or failure_threshold > 1:
      raise ValueError("Failure threshold must be between 0 and 100: %s" %
                        failure_threshold)

    if args.logfile == "-":
      if args.incremental:
        print("Can't use incremental reading on stdin.")
        return
      if args.remove_cursor or args.show_cursor:
        print("Can't manage cursors for stdin.")
        return

    if args.policy_defs:
      policy_defs = DefsParser.Defs(args.policy_defs)
    else:
      policy_defs = None

    if args.mta_flavor == "Postfix":
      logwatcher = PostfixLogWatcher(args.logfile,args.incremental,policy_defs)
    else:
      print("Unexpected MTA flavor: {}".format(args.mta_flavor))
      return

    if args.remove_cursor:
      print(logwatcher.remove_cursor())
      return
    if args.show_cursor:
      print(logwatcher.show_cursor())
      return

    res = logwatcher.analyze_lines(logwatcher.get_newlines())

    if args.output == "summary":
      print("Displaying the summary accordingly to logfile parsing results")
      print("")

      for s in logwatcher.status_tags:
        if s in res:
          print("%s:" % s)
          print(json.dumps(res[s],indent=2))
      print("Domains:")
      print(json.dumps(res["domains"],indent=2))

    elif args.output == "matched-lines":
      print("Displaying the logfile's lines that matched configured patterns")
      print("")

      for l in res["matched_lines"]:
        print(l.rstrip("\n"))

    elif args.output == "unmatched-lines":
      print("Displaying the logfile's lines that did not match "
            "configured patterns")
      print("")

      for l in res["unmatched_lines"]:
        print(l.rstrip("\n"))

    elif args.output in [ "domains", "warnings" ]:
      print("Displaying successful/failed delivery attempts for %s" %
            ("every domain" if args.output == "domains" else
             "domains with an high failure rate (%s%%)" %
             (failure_threshold*100)))
      print("")

      warning_domains = []

      for domainname in res["domains"]:
        domain = res["domains"][domainname]
        
        if not "attempted" in domain:
          continue

        #TODO: implement results for "log-only = true" status.
        if "sent_ko" in domain and domain["attempted"] > 0:
          failure_rate = domain["sent_ko"] / domain["attempted"]
        else:
          failure_rate = None

        if args.output == "domains" or \
          ( args.output == "warnings" and failure_rate >= failure_threshold ):
          succeeded = domain["sent_ok"] if "sent_ok" in domain else "none"
          failed = domain["sent_ko"] if "sent_ko" in domain else "none"

          s = "{d}: {t} delivery attempts, {s} succeeded, {f} failed"
          if failure_rate:
            s = s + ", {r:.2%} failure rate"
            if failure_rate >= failure_threshold:
              s = s + " - WARNING"
              warning_domains.append(domainname)

          print(s.format(d=domainname, r=failure_rate,
                         t=domain["attempted"], s=succeeded,
                         f=failed))

      if args.output == "warnings" and len(warning_domains) > 0:
        report_format= Config.get("general","logwatcher_reports_fmt")
        report_filename = datetime.datetime.now().strftime(report_format)
        report_file = "%s/%s" % (reports_dir,report_filename)
        with open(report_file, "w") as r:
          r.write("domainname,attempts,ko,ok\n")
          for domainname in warning_domains:
            r.write("{domainname},{attempted},{sent_ko},{sent_ok}\n".format(
                    domainname=domainname,
                    attempted=res["domains"][domainname]["attempted"],
                    sent_ko=res["domains"][domainname].get("sent_ko",0),
                    sent_ok=res["domains"][domainname].get("sent_ok",0)))

        notification_t = "Delivery errors found for {domains} for a " + \
                         "total of {fail} failures over {tot} total " + \
                         "attempts. More details on {report_file}"

        if len(warning_domains) > 3:
          notification_domains = ", ".join(warning_domains[:3]) + \
                                 "and " + str(len(warning_domains)-3) + \
                                 " more domains"
        else:
          notification_domains = ", ".join(warning_domains)

        fail = 0
        tot = 0
        for domainname in warning_domains:
          fail = fail + res["domains"][domainname]["sent_ko"]
          tot = tot + res["domains"][domainname]["attempted"]

        notification = notification_t.format(domains=notification_domains,
                                             fail=fail,
                                             tot=tot,
                                             report_file=report_file)

        Config.get_logger().error(notification)