def __init__(self,
marathon_client,
group,
cert_store,
mlb_client,
txacme_client_creator,
reactor,
email=None):
"""
Create the marathon-acme service.
:param marathon_client: The Marathon API client.
:param group: The name of the marathon-lb group.
:param cert_store: The ``ICertificateStore`` instance to use.
:param mlb_clinet: The marathon-lb API client.
:param txacme_client_creator: Callable to create the txacme client.
:param reactor: The reactor to use.
:param email: The ACME registration email.
"""
self.marathon_client = marathon_client
self.group = group
self.reactor = reactor
responder = HTTP01Responder()
self.server = MarathonAcmeServer(responder.resource)
mlb_cert_store = MlbCertificateStore(cert_store, mlb_client)
self.txacme_service = AcmeIssuingService(mlb_cert_store,
txacme_client_creator,
reactor, [responder], email)
self._server_listening = None
def _setUp(self):
self.cert_store = MemoryStore(self._certs)
self.clock = Clock()
self.clock.rightNow = (self.now - datetime(1970, 1, 1)).total_seconds()
if self.acme_client is None:
acme_client = FakeClient(RSA_KEY_512,
clock=self.clock,
ca_key=RSA_KEY_512_RAW,
controller=self.controller)
else:
acme_client = self.acme_client
self.responder = RecordingResponder(set(), u'tls-sni-01')
args = dict(cert_store=self.cert_store,
client_creator=lambda: succeed(acme_client),
clock=self.clock,
responders=[self.responder],
email=self._email,
panic_interval=self._panic_interval,
panic=self._panic,
generate_key=lambda: RSA_KEY_512_RAW)
self.service = AcmeIssuingService(
**{k: v
for k, v in args.items() if v is not None})
self.addCleanup(
lambda: self.service.running and self.service.stopService())
def action(secret):
password = secret
if driver_name == 'gandi':
responders = [
GandiV5Responder(api_key=password, zone_name=zone_name)
]
elif driver_name == 'cloudflare':
responders = [
CloudflareV4Responder(email=user_name,
api_key=password,
zone_name=zone_name)
]
else:
responders = [
LibcloudDNSResponder.create(reactor, driver_name, user_name,
password, zone_name)
]
acme_key = maybe_key(acme_path)
cert_store = DirectoryStore(acme_path)
if staging:
le_url = LETSENCRYPT_STAGING_DIRECTORY
else:
le_url = LETSENCRYPT_DIRECTORY
client_creator = partial(Client.from_url,
reactor=reactor,
url=le_url,
key=acme_key,
alg=RS256)
clock = reactor
service = AcmeIssuingService(cert_store, client_creator, clock,
responders)
service.startService()
forever = Deferred()
return forever
def _got_port(port):
self.service = AcmeIssuingService(
cert_store=self.cert_store,
client_creator=partial(self.client_creator, self.reactor,
self.directory),
clock=self.reactor,
responders=[responder],
check_interval=self.check_interval,
reissue_interval=self.reissue_interval,
panic_interval=self.panic_interval,
panic=self._panic,
generate_key=self._generate_key)
self.service.startService()
return (self.service.when_certs_valid().addCallback(
lambda _: _WrapperPort(port=port, service=self.service)))
def create_issuing_service(
reactor: IReactorTCP,
acme_url: str,
account_key_file: str,
well_known_resource: IResource,
) -> AcmeIssuingService:
"""Create an ACME issuing service, and attach it to a web Resource
Args:
reactor: twisted reactor
acme_url: URL to use to request certificates
account_key_file: where to store the account key
well_known_resource: web resource for .well-known.
we will attach a child resource for "acme-challenge".
Returns:
AcmeIssuingService
"""
responder = HTTP01Responder()
well_known_resource.putChild(b"acme-challenge", responder.resource)
store = ErsatzStore()
return AcmeIssuingService(
cert_store=store,
client_creator=(lambda: Client.from_url(
reactor=reactor,
url=URL.from_text(acme_url),
key=load_or_create_client_key(account_key_file),
alg=RS256,
)),
clock=reactor,
responders=[responder],
)
def _setUp(self):
self.cert_store = MemoryStore(self._certs)
self.clock = Clock()
self.clock.rightNow = (
self.now - datetime(1970, 1, 1)).total_seconds()
if self.acme_client is None:
acme_client = FakeClient(
RSA_KEY_512, clock=self.clock, ca_key=RSA_KEY_512_RAW,
controller=self.controller)
else:
acme_client = self.acme_client
self.responder = RecordingResponder(set(), u'tls-sni-01')
args = dict(
cert_store=self.cert_store,
client_creator=lambda: succeed(acme_client),
clock=self.clock,
responders=[self.responder],
email=self._email,
panic_interval=self._panic_interval,
panic=self._panic,
generate_key=lambda: RSA_KEY_512_RAW)
self.service = AcmeIssuingService(
**{k: v for k, v in args.items() if v is not None})
self.addCleanup(
lambda: self.service.running and self.service.stopService())
def action(secret):
password = secret
responders = [
LibcloudDNSResponder.create(reactor, driver_name, user_name,
password, zone_name)
]
acme_key = maybe_key(acme_path)
cert_store = DirectoryStore(acme_path)
client_creator = partial(Client.from_url,
reactor=reactor,
url=LETSENCRYPT_DIRECTORY,
key=acme_key,
alg=RS256)
clock = reactor
service = AcmeIssuingService(cert_store, client_creator, clock,
responders)
service.startService()
forever = Deferred()
return forever
def makeService(config, reactor=reactor):
parent = MultiService()
basedir = FilePath(os.path.expanduser(config["basedir"]))
basedir.makedirs(ignoreExistingDirectory=True)
basedir.chmod(0o700)
data = Data(basedir.child("config.json"))
certFile = basedir.child("tub.data").path
tub = Tub(certFile=certFile)
tub.setOption("keepaliveTimeout", 60) # ping after 60s of idle
tub.setOption("disconnectTimeout", 5*60) # disconnect/reconnect after 5m
tub.listenOn("tcp:6319:interface=127.0.0.1")
tub.setLocation("tcp:127.0.0.1:6319")
tub.setServiceParent(parent)
acme_path = basedir.asTextMode()
acme_key = maybe_key(acme_path)
cert_store = FlancerCertificateStore(data, basedir)
staging = not config["really"]
if staging:
print("STAGING mode")
le_url = LETSENCRYPT_STAGING_DIRECTORY
else:
print("REAL CERTIFICATE mode")
le_url = LETSENCRYPT_DIRECTORY
client_creator = partial(Client.from_url, reactor=reactor,
url=le_url,
key=acme_key, alg=RS256)
r = FlancerResponder(tub, data)
issuer = AcmeIssuingService(cert_store, client_creator, reactor, [r])
issuer.setServiceParent(parent)
if "dyndns_furl" in data:
start_dyndns_canary(tub, data["dyndns_furl"].encode("ascii"))
c = Controller(tub, data, issuer)
tub.registerReference(c, furlFile=basedir.child("controller.furl").path)
#TimerService(5*60.0, f.timerUpdateStats).setServiceParent(parent)
return parent
def _got_port(port):
self.service = AcmeIssuingService(
cert_store=self.cert_store,
client_creator=partial(
self.client_creator, self.reactor, self.directory),
clock=self.reactor,
responders=[responder],
check_interval=self.check_interval,
reissue_interval=self.reissue_interval,
panic_interval=self.panic_interval,
panic=self._panic,
generate_key=self._generate_key)
self.service.startService()
return (
self.service.when_certs_valid()
.addCallback(
lambda _: _WrapperPort(port=port, service=self.service)))
class AcmeFixture(Fixture):
"""
A fixture for setting up an `~txacme.service.AcmeIssuingService`.
"""
def __init__(self,
now=datetime(2000, 1, 1, 0, 0, 0),
certs=None,
panic_interval=None,
panic=None,
client=None,
email=None):
super(AcmeFixture, self).__init__()
self.now = now
self._certs = certs
self._panic_interval = panic_interval
self._panic = panic
self._email = email
self.acme_client = client
self.controller = FakeClientController()
def _setUp(self):
self.cert_store = MemoryStore(self._certs)
self.clock = Clock()
self.clock.rightNow = (self.now - datetime(1970, 1, 1)).total_seconds()
if self.acme_client is None:
acme_client = FakeClient(RSA_KEY_512,
clock=self.clock,
ca_key=RSA_KEY_512_RAW,
controller=self.controller)
else:
acme_client = self.acme_client
self.responder = RecordingResponder(set(), u'tls-sni-01')
args = dict(cert_store=self.cert_store,
client_creator=lambda: succeed(acme_client),
clock=self.clock,
responders=[self.responder],
email=self._email,
panic_interval=self._panic_interval,
panic=self._panic,
generate_key=lambda: RSA_KEY_512_RAW)
self.service = AcmeIssuingService(
**{k: v
for k, v in args.items() if v is not None})
self.addCleanup(
lambda: self.service.running and self.service.stopService())
class AcmeFixture(Fixture):
"""
A fixture for setting up an `~txacme.service.AcmeIssuingService`.
"""
def __init__(self, now=datetime(2000, 1, 1, 0, 0, 0), certs=None,
panic_interval=None, panic=None, client=None, email=None):
super(AcmeFixture, self).__init__()
self.now = now
self._certs = certs
self._panic_interval = panic_interval
self._panic = panic
self._email = email
self.acme_client = client
self.controller = FakeClientController()
def _setUp(self):
self.cert_store = MemoryStore(self._certs)
self.clock = Clock()
self.clock.rightNow = (
self.now - datetime(1970, 1, 1)).total_seconds()
if self.acme_client is None:
acme_client = FakeClient(
RSA_KEY_512, clock=self.clock, ca_key=RSA_KEY_512_RAW,
controller=self.controller)
else:
acme_client = self.acme_client
self.responder = RecordingResponder(set(), u'tls-sni-01')
args = dict(
cert_store=self.cert_store,
client_creator=lambda: succeed(acme_client),
clock=self.clock,
responders=[self.responder],
email=self._email,
panic_interval=self._panic_interval,
panic=self._panic,
generate_key=lambda: RSA_KEY_512_RAW)
self.service = AcmeIssuingService(
**{k: v for k, v in args.items() if v is not None})
self.addCleanup(
lambda: self.service.running and self.service.stopService())
def get_things_done():
"""
Here is where the service part is setup and action is done.
"""
responders = yield start_responders()
store = MemoryStore()
# We first validate the directory.
account_key = _get_account_key()
try:
client = yield Client.from_url(
reactor,
URL.fromText(acme_url.decode('utf-8')),
key=JWKRSA(key=account_key),
alg=RS256,
)
except Exception as error:
print('\n\nFailed to connect to ACME directory. %s' % (error, ))
yield reactor.stop()
defer.returnValue(None)
service = AcmeIssuingService(
email='[email protected],[email protected]',
cert_store=store,
client=client,
clock=reactor,
responders=responders,
panic=on_panic,
)
# Start the service and wait for it to start.
yield service.start()
# Wait for the existing certificate from the storage to be available.
yield service.when_certs_valid()
# Request a SAN ... if passed via command line.
yield service.issue_cert(','.join(requested_domains))
yield service.stopService()
print('That was all the example.')
class AutoTLSEndpoint(object):
"""
A server endpoint that does TLS SNI, with certificates automatically
(re)issued from an ACME certificate authority.
:param reactor: The Twisted reactor.
:param directory: ``twisted.python.url.URL`` for the ACME directory to use
for issuing certs.
:type client_creator: Callable[[reactor, ``twisted.python.url.URL``],
Deferred[`txacme.client.Client`]]
:param client_creator: A callable called with the reactor and directory URL
for creating the ACME client. For example, ``partial(Client.from_url,
key=acme_key, alg=RS256)``.
:type cert_store: `txacme.interfaces.ICertificateStore`
:param cert_store: The certificate
store containing the certificates to manage. For example,
`txacme.store.DirectoryStore`.
:param dict cert_mapping: The certificate mapping to use for SNI; for
example, ``txsni.snimap.HostDirectoryMap``. Usually this should
correspond to the same underlying storage as ``cert_store``.
:param ~datetime.timedelta check_interval: How often to check for expiring
certificates.
:param ~datetime.timedelta reissue_interval: If a certificate is expiring
in less time than this interval, it will be reissued.
:param ~datetime.timedelta panic_interval: If a certificate is expiring in
less time than this interval, and reissuing fails, the panic callback
will be invoked.
:type panic: Callable[[Failure, `str`], Deferred]
:param panic: A callable invoked with the failure and server name when
reissuing fails for a certificate expiring in the ``panic_interval``.
For example, you could generate a monitoring alert. The default
callback logs a message at *CRITICAL* level.
:param generate_key: A 0-arg callable used to generate a private key for a
new cert. Normally you would not pass this unless you have specialized
key generation requirements.
"""
reactor = attr.ib()
directory = attr.ib(
validator=lambda inst, a, value: check_directory_url_type(value))
client_creator = attr.ib()
cert_store = attr.ib()
cert_mapping = attr.ib()
sub_endpoint = attr.ib()
check_interval = attr.ib(default=timedelta(days=1))
reissue_interval = attr.ib(default=timedelta(days=30))
panic_interval = attr.ib(default=timedelta(days=15))
_panic = attr.ib(default=_default_panic)
_generate_key = attr.ib(default=partial(generate_private_key, u'rsa'))
def listen(self, protocolFactory): # noqa
"""
Start an issuing service, and wait until initial issuing is complete.
"""
def _got_port(port):
self.service = AcmeIssuingService(
cert_store=self.cert_store,
client_creator=partial(
self.client_creator, self.reactor, self.directory),
clock=self.reactor,
responders=[responder],
check_interval=self.check_interval,
reissue_interval=self.reissue_interval,
panic_interval=self.panic_interval,
panic=self._panic,
generate_key=self._generate_key)
self.service.startService()
return (
self.service.when_certs_valid()
.addCallback(
lambda _: _WrapperPort(port=port, service=self.service)))
responder = TLSSNI01Responder()
sni_map = SNIMap(responder.wrap_host_map(self.cert_mapping))
return (
maybeDeferred(
self.sub_endpoint.listen,
TLSMemoryBIOFactory(
contextFactory=sni_map,
isClient=False,
wrappedFactory=protocolFactory))
.addCallback(_got_port))
def makeService(config):
ini = ConfigParser.RawConfigParser()
ini.read(config['config'])
configPath = FilePath(config['config']).parent()
rproxyConf = dict(ini.items("rproxy"))
hostsConf = dict(ini.items("hosts"))
hosts = {}
for k, v in hostsConf.items():
k = k.lower()
hostname, part = k.rsplit("_", 1)
if hostname not in hosts:
hosts[hostname] = {}
hosts[hostname][part] = v
if not hosts:
raise ValueError("No hosts configured.")
for i in hosts:
if "port" not in hosts[i]:
raise ValueError("All hosts need a port.")
if "host" not in hosts[i]:
print("%s does not have a host, making localhost" % (i,))
hosts[i]["host"] = "localhost"
if "wwwtoo" not in hosts[i]:
print("%s does not have an wwwtoo setting, making True" % (i,))
hosts[i]["wwwtoo"] = "True"
if "proxysecure" not in hosts[i]:
print("%s does not have an proxysecure setting, making False" % (i,))
hosts[i]["proxysecure"] = False
hosts[i]["wwwtoo"] = True if hosts[i]["wwwtoo"]=="True" else False
hosts[i]["proxysecure"] = True if hosts[i]["proxysecure"]=="True" else False
hosts[i]["sendhsts"] = True if hosts[i].get("sendhsts")=="True" else False
from twisted.internet import reactor
pool = HTTPConnectionPool(reactor)
resource = EncodingResourceWrapper(
RProxyResource(hosts, rproxyConf.get("clacks"), pool, reactor, {}, False),
[server.GzipEncoderFactory()])
responder = HTTP01Responder()
site = server.Site(EnsureHTTPS(resource, responder.resource),)
multiService = service.MultiService()
certificates = rproxyConf.get("certificates", None)
if certificates:
try:
configPath.child(certificates).makedirs()
except:
pass
certificates = configPath.child(certificates).path
for i in rproxyConf.get("https_ports").split(","):
print("Starting HTTPS on port " + i)
multiService.addService(strports.service('txsni:' + certificates + ':tcp:' + i, site))
for host in hosts.keys():
with open(FilePath(certificates).child(host + ".pem").path, 'r+'):
# Open it so that txacme can find it
pass
if hosts[host]["wwwtoo"]:
with open(FilePath(certificates).child("www." + host + ".pem").path, 'r+'):
# Open it so that txacme can find it
pass
for i in rproxyConf.get("http_ports", "").split(","):
print("Starting HTTP on port " + i)
multiService.addService(strports.service('tcp:' + i, site))
issuingService = AcmeIssuingService(
cert_store=DirectoryStore(FilePath(certificates)),
client_creator=(lambda: Client.from_url(
reactor=reactor,
url=LETSENCRYPT_DIRECTORY,
key=load_or_create_client_key(FilePath(certificates)),
alg=RS256,
)),
clock=reactor,
responders=[responder],
)
issuingService.setServiceParent(multiService)
return multiService
class AcmeHandler(object):
def __init__(self, hs):
self.hs = hs
self.reactor = hs.get_reactor()
self._acme_domain = hs.config.acme_domain
@defer.inlineCallbacks
def start_listening(self):
# Configure logging for txacme, if you need to debug
# from eliot import add_destinations
# from eliot.twisted import TwistedDestination
#
# add_destinations(TwistedDestination())
from txacme.challenges import HTTP01Responder
from txacme.service import AcmeIssuingService
from txacme.endpoint import load_or_create_client_key
from txacme.client import Client
from josepy.jwa import RS256
self._store = ErsatzStore()
responder = HTTP01Responder()
self._issuer = AcmeIssuingService(
cert_store=self._store,
client_creator=(lambda: Client.from_url(
reactor=self.reactor,
url=URL.from_text(self.hs.config.acme_url),
key=load_or_create_client_key(
FilePath(self.hs.config.config_dir_path)),
alg=RS256,
)),
clock=self.reactor,
responders=[responder],
)
well_known = Resource()
well_known.putChild(b'acme-challenge', responder.resource)
responder_resource = Resource()
responder_resource.putChild(b'.well-known', well_known)
responder_resource.putChild(b'check',
static.Data(b'OK', b'text/plain'))
srv = server.Site(responder_resource)
bind_addresses = self.hs.config.acme_bind_addresses
for host in bind_addresses:
logger.info(
"Listening for ACME requests on %s:%i",
host,
self.hs.config.acme_port,
)
try:
self.reactor.listenTCP(
self.hs.config.acme_port,
srv,
interface=host,
)
except twisted.internet.error.CannotListenError as e:
check_bind_error(e, host, bind_addresses)
# Make sure we are registered to the ACME server. There's no public API
# for this, it is usually triggered by startService, but since we don't
# want it to control where we save the certificates, we have to reach in
# and trigger the registration machinery ourselves.
self._issuer._registered = False
yield self._issuer._ensure_registered()
@defer.inlineCallbacks
def provision_certificate(self):
logger.warning("Reprovisioning %s", self._acme_domain)
try:
yield self._issuer.issue_cert(self._acme_domain)
except Exception:
logger.exception("Fail!")
raise
logger.warning("Reprovisioned %s, saving.", self._acme_domain)
cert_chain = self._store.certs[self._acme_domain]
try:
with open(self.hs.config.tls_private_key_file,
"wb") as private_key_file:
for x in cert_chain:
if x.startswith(b"-----BEGIN RSA PRIVATE KEY-----"):
private_key_file.write(x)
with open(self.hs.config.tls_certificate_file,
"wb") as certificate_file:
for x in cert_chain:
if x.startswith(b"-----BEGIN CERTIFICATE-----"):
certificate_file.write(x)
except Exception:
logger.exception("Failed saving!")
raise
defer.returnValue(True)
class MarathonAcme(object):
log = Logger()
def __init__(self,
marathon_client,
group,
cert_store,
mlb_client,
txacme_client_creator,
reactor,
email=None):
"""
Create the marathon-acme service.
:param marathon_client: The Marathon API client.
:param group: The name of the marathon-lb group.
:param cert_store: The ``ICertificateStore`` instance to use.
:param mlb_clinet: The marathon-lb API client.
:param txacme_client_creator: Callable to create the txacme client.
:param reactor: The reactor to use.
:param email: The ACME registration email.
"""
self.marathon_client = marathon_client
self.group = group
self.reactor = reactor
responder = HTTP01Responder()
self.server = MarathonAcmeServer(responder.resource)
mlb_cert_store = MlbCertificateStore(cert_store, mlb_client)
self.txacme_service = AcmeIssuingService(mlb_cert_store,
txacme_client_creator,
reactor, [responder], email)
self._server_listening = None
def run(self, endpoint_description):
self.log.info('Starting marathon-acme...')
# Start the server
d = self.server.listen(self.reactor, endpoint_description)
def on_server_listening(listening_port):
self._server_listening = listening_port
# Start the txacme service and wait for the initial check
self.txacme_service.startService()
return self.txacme_service.when_certs_valid()
d.addCallback(on_server_listening)
# Then listen for events...
d.addCallback(lambda _: self.listen_events())
# If anything goes wrong or listening for events returns, stop
d.addBoth(self._stop)
return d
def _stop(self, result):
if isinstance(result, Failure):
self.log.failure('Unhandle error during operation', result)
self.log.warn('Stopping marathon-acme...')
# If the server failed to start we have nothing to cancel yet
if self._server_listening is not None:
return gatherResults([
self._server_listening.stopListening(),
self.txacme_service.stopService()
],
consumeErrors=True)
def listen_events(self, reconnects=0):
"""
Start listening for events from Marathon, running a sync when we first
successfully subscribe and triggering a sync on API request events.
"""
self.log.info('Listening for events from Marathon...')
self._attached = False
def on_finished(result, reconnects):
# If the callback fires then the HTTP request to the event stream
# went fine, but the persistent connection for the SSE stream was
# dropped. Just reconnect for now- if we can't actually connect
# then the errback will fire rather.
self.log.warn(
'Connection lost listening for events, '
'reconnecting... ({reconnects} so far)',
reconnects=reconnects)
reconnects += 1
return self.listen_events(reconnects)
def log_failure(failure):
self.log.failure('Failed to listen for events', failure)
return failure
return self.marathon_client.get_events({
'event_stream_attached':
self._sync_on_event_stream_attached,
'api_post_event':
self._sync_on_api_post_event
}).addCallbacks(on_finished, log_failure, callbackArgs=[reconnects])
def _sync_on_event_stream_attached(self, event):
if self._attached:
self.log.debug(
'event_stream_attached event received (timestamp: '
'"{timestamp}", remoteAddress: "{remoteAddress}"), but '
'already attached',
timestamp=event['timestamp'],
remoteAddress=event['remoteAddress'])
return
self._attached = True
self.log.info(
'event_stream_attached event received (timestamp: "{timestamp}", '
'remoteAddress: "{remoteAddress}"), running initial sync...',
timestamp=event['timestamp'],
remoteAddress=event['remoteAddress'])
return self.sync()
def _sync_on_api_post_event(self, event):
self.log.info(
'api_post_event event received (timestamp: "{timestamp}", uri: '
'"{uri}"), triggering a sync...',
timestamp=event['timestamp'],
uri=event['uri'])
return self.sync()
def sync(self):
"""
Fetch the list of apps from Marathon, find the domains that require
certificates, and issue certificates for any domains that don't already
have a certificate.
"""
self.log.info('Starting a sync...')
def log_success(result):
self.log.info('Sync completed successfully')
return result
def log_failure(failure):
self.log.failure('Sync failed', failure, LogLevel.error)
return failure
return (self.marathon_client.get_apps().addCallback(
self._apps_acme_domains).addCallback(
self._filter_new_domains).addCallback(
self._issue_certs).addCallbacks(log_success, log_failure))
def _apps_acme_domains(self, apps):
domains = []
for app in apps:
domains.extend(self._app_acme_domains(app))
self.log.debug('Found {len_domains} domains for apps: {domains}',
len_domains=len(domains),
domains=domains)
return domains
def _app_acme_domains(self, app):
app_domains = []
labels = app['labels']
app_group = labels.get('HAPROXY_GROUP')
# Prefer the 'portDefinitions' field added in Marathon 1.0.0 but fall
# back to the deprecated 'ports' array if that's not present.
if 'portDefinitions' in app:
ports = app['portDefinitions']
else:
ports = app['ports']
# Iterate through the ports, checking for corresponding labels
for port_index, _ in enumerate(ports):
# Get the port group label, defaulting to the app group label
port_group = labels.get('HAPROXY_%d_GROUP' % (port_index, ),
app_group)
if port_group == self.group:
domain_label = labels.get(
'MARATHON_ACME_%d_DOMAIN' % (port_index, ), '')
port_domains = parse_domain_label(domain_label)
if port_domains:
# TODO: Support SANs- for now just use the first domain
if len(port_domains) > 1:
self.log.warn(
'Multiple domains found for port {port} of app '
'{app}, only the first will be used',
port=port_index,
app=app['id'])
app_domains.append(port_domains[0])
self.log.debug('Found {len_domains} domains for app {app}: {domains}',
len_domains=len(app_domains),
app=app['id'],
domains=app_domains)
return app_domains
def _filter_new_domains(self, marathon_domains):
def filter_domains(stored_domains):
return set(marathon_domains) - set(stored_domains.keys())
d = self.txacme_service.cert_store.as_dict()
d.addCallback(filter_domains)
return d
def _issue_certs(self, domains):
if domains:
self.log.info(
'Issuing certificates for {len_domains} domains: {domains}',
len_domains=len(domains),
domains=domains)
else:
self.log.debug('No new domains to issue certificates for')
return gatherResults([self._issue_cert(domain) for domain in domains])
def _issue_cert(self, domain):
"""
Issue a certificate for the given domain.
"""
def errback(failure):
# Don't fail on some of the errors we could get from the ACME
# server, rather just log an error so that we can continue with
# other domains.
failure.trap(txacme_ServerError)
acme_error = failure.value.message
# FIXME: The acme error code stuff is a mess pre- the unreleased
# 0.10 version. Update this to use the 'code' attribute when the
# new acme library is released.
acme_error_code = str(acme_error.typ).split(':')[-1]
if acme_error_code in [
'rateLimited', 'serverInternal', 'connection',
'unknownHost'
]:
# TODO: Fire off an error to Sentry or something?
self.log.error(
'Error ({code}) issuing certificate for "{domain}": '
'{detail}',
code=acme_error_code,
domain=domain,
detail=acme_error.detail)
else:
# There are more error codes but if they happen then something
# serious has gone wrong-- carry on error-ing.
return failure
d = self.txacme_service.issue_cert(domain)
return d.addErrback(errback)
def start_listening(self):
# Configure logging for txacme, if you need to debug
# from eliot import add_destinations
# from eliot.twisted import TwistedDestination
#
# add_destinations(TwistedDestination())
from txacme.challenges import HTTP01Responder
from txacme.service import AcmeIssuingService
from txacme.endpoint import load_or_create_client_key
from txacme.client import Client
from josepy.jwa import RS256
self._store = ErsatzStore()
responder = HTTP01Responder()
self._issuer = AcmeIssuingService(
cert_store=self._store,
client_creator=(lambda: Client.from_url(
reactor=self.reactor,
url=URL.from_text(self.hs.config.acme_url),
key=load_or_create_client_key(
FilePath(self.hs.config.config_dir_path)),
alg=RS256,
)),
clock=self.reactor,
responders=[responder],
)
well_known = Resource()
well_known.putChild(b'acme-challenge', responder.resource)
responder_resource = Resource()
responder_resource.putChild(b'.well-known', well_known)
responder_resource.putChild(b'check',
static.Data(b'OK', b'text/plain'))
srv = server.Site(responder_resource)
bind_addresses = self.hs.config.acme_bind_addresses
for host in bind_addresses:
logger.info(
"Listening for ACME requests on %s:%i",
host,
self.hs.config.acme_port,
)
try:
self.reactor.listenTCP(
self.hs.config.acme_port,
srv,
interface=host,
)
except twisted.internet.error.CannotListenError as e:
check_bind_error(e, host, bind_addresses)
# Make sure we are registered to the ACME server. There's no public API
# for this, it is usually triggered by startService, but since we don't
# want it to control where we save the certificates, we have to reach in
# and trigger the registration machinery ourselves.
self._issuer._registered = False
yield self._issuer._ensure_registered()
class AutoTLSEndpoint(object):
"""
A server endpoint that does TLS SNI, with certificates automatically
(re)issued from an ACME certificate authority.
:param reactor: The Twisted reactor.
:param directory: ``twisted.python.url.URL`` for the ACME directory to use
for issuing certs.
:type client_creator: Callable[[reactor, ``twisted.python.url.URL``],
Deferred[`txacme.client.Client`]]
:param client_creator: A callable called with the reactor and directory URL
for creating the ACME client. For example, ``partial(Client.from_url,
key=acme_key, alg=RS256)``.
:param .ICertificateStore cert_store: The certificate store containing the
certificates to manage. For example, `txacme.store.DirectoryStore`.
:param dict cert_mapping: The certificate mapping to use for SNI; for
example, ``txsni.snimap.HostDirectoryMap``. Usually this should
correspond to the same underlying storage as ``cert_store``.
:param ~datetime.timedelta check_interval: How often to check for expiring
certificates.
:param ~datetime.timedelta reissue_interval: If a certificate is expiring
in less time than this interval, it will be reissued.
:param ~datetime.timedelta panic_interval: If a certificate is expiring in
less time than this interval, and reissuing fails, the panic callback
will be invoked.
:type panic: Callable[[Failure, `str`], Deferred]
:param panic: A callable invoked with the failure and server name when
reissuing fails for a certificate expiring in the ``panic_interval``.
For example, you could generate a monitoring alert. The default
callback logs a message at *CRITICAL* level.
:param generate_key: A 0-arg callable used to generate a private key for a
new cert. Normally you would not pass this unless you have specialized
key generation requirements.
"""
reactor = attr.ib()
directory = attr.ib(
validator=lambda inst, a, value: check_directory_url_type(value))
client_creator = attr.ib()
cert_store = attr.ib()
cert_mapping = attr.ib()
sub_endpoint = attr.ib()
check_interval = attr.ib(default=timedelta(days=1))
reissue_interval = attr.ib(default=timedelta(days=30))
panic_interval = attr.ib(default=timedelta(days=15))
_panic = attr.ib(default=_default_panic)
_generate_key = attr.ib(default=partial(generate_private_key, u'rsa'))
def listen(self, protocolFactory): # noqa
"""
Start an issuing service, and wait until initial issuing is complete.
"""
def _got_port(port):
self.service = AcmeIssuingService(
cert_store=self.cert_store,
client_creator=partial(
self.client_creator, self.reactor, self.directory),
clock=self.reactor,
responders=[responder],
check_interval=self.check_interval,
reissue_interval=self.reissue_interval,
panic_interval=self.panic_interval,
panic=self._panic,
generate_key=self._generate_key)
self.service.startService()
return (
self.service.when_certs_valid()
.addCallback(
lambda _: _WrapperPort(port=port, service=self.service)))
responder = TLSSNI01Responder()
sni_map = SNIMap(responder.wrap_host_map(self.cert_mapping))
return (
maybeDeferred(
self.sub_endpoint.listen,
TLSMemoryBIOFactory(
contextFactory=sni_map,
isClient=False,
wrappedFactory=protocolFactory))
.addCallback(_got_port))
def makeService(config):
ini = ConfigParser.RawConfigParser()
ini.read(config['config'])
configPath = FilePath(config['config']).parent()
rproxyConf = dict(ini.items("rproxy"))
hostsConf = dict(ini.items("hosts"))
hosts = {}
for k, v in hostsConf.items():
k = k.lower()
hostname, part = k.rsplit("_", 1)
if hostname not in hosts:
hosts[hostname] = {}
hosts[hostname][part] = v
if not hosts:
raise ValueError("No hosts configured.")
for i in hosts:
if "port" not in hosts[i]:
raise ValueError("All hosts need a port.")
if "host" not in hosts[i]:
print("%s does not have a host, making localhost" % (i, ))
hosts[i]["host"] = "localhost"
if "wwwtoo" not in hosts[i]:
print("%s does not have an wwwtoo setting, making True" % (i, ))
hosts[i]["wwwtoo"] = "True"
if "proxysecure" not in hosts[i]:
print("%s does not have an proxysecure setting, making False" %
(i, ))
hosts[i]["proxysecure"] = False
hosts[i]["wwwtoo"] = True if hosts[i]["wwwtoo"] == "True" else False
hosts[i]["proxysecure"] = True if hosts[i][
"proxysecure"] == "True" else False
hosts[i]["sendhsts"] = True if hosts[i].get(
"sendhsts") == "True" else False
from twisted.internet import reactor
pool = HTTPConnectionPool(reactor)
resource = EncodingResourceWrapper(
RProxyResource(hosts, rproxyConf.get("clacks"), pool, reactor, {},
False), [server.GzipEncoderFactory()])
responder = HTTP01Responder()
site = server.Site(EnsureHTTPS(resource, responder.resource), )
multiService = service.MultiService()
certificates = rproxyConf.get("certificates", None)
if certificates:
try:
configPath.child(certificates).makedirs()
except:
pass
certificates = configPath.child(certificates).path
for i in rproxyConf.get("https_ports").split(","):
print("Starting HTTPS on port " + i)
multiService.addService(
strports.service('txsni:' + certificates + ':tcp:' + i, site))
for host in hosts.keys():
with open(FilePath(certificates).child(host + ".pem").path, 'w'):
# Open it so that txacme can find it
pass
if hosts[host]["wwwtoo"]:
with open(
FilePath(certificates).child("www." + host +
".pem").path, 'w'):
# Open it so that txacme can find it
pass
for i in rproxyConf.get("http_ports", "").split(","):
print("Starting HTTP on port " + i)
multiService.addService(strports.service('tcp:' + i, site))
issuingService = AcmeIssuingService(
cert_store=DirectoryStore(FilePath(certificates)),
client_creator=(lambda: Client.from_url(
reactor=reactor,
url=LETSENCRYPT_DIRECTORY,
key=load_or_create_client_key(FilePath(certificates)),
alg=RS256,
)),
clock=reactor,
responders=[responder],
)
issuingService.setServiceParent(multiService)
return multiService
