def _prepare_pebble_server(self):
"""Configure and launch the Pebble server"""
print('=> Starting pebble instance deployment...')
pebble_path, challtestsrv_path, pebble_config_path = pebble_artifacts.fetch(
self._workspace)
# Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid
# nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment.
environ = os.environ.copy()
environ['PEBBLE_VA_NOSLEEP'] = '1'
environ['PEBBLE_WFE_NONCEREJECT'] = '0'
self._launch_process([
pebble_path, '-config', pebble_config_path, '-dnsserver',
'127.0.0.1:8053'
],
env=environ)
self._launch_process([
challtestsrv_path, '-management', ':{0}'.format(CHALLTESTSRV_PORT),
'-defaultIPv6', '""', '-defaultIPv4', '127.0.0.1', '-http01', '""',
'-tlsalpn01', '""', '-https01', '""'
])
# Wait for the ACME CA server to be up.
print('=> Waiting for pebble instance to respond...')
misc.check_until_timeout(self.acme_xdist['directory_url'])
print('=> Finished pebble instance deployment.')
def _prepare_boulder_server(self):
"""Configure and launch the Boulder server"""
print('=> Starting boulder instance deployment...')
instance_path = join(self._workspace, 'boulder')
# Load Boulder from git, that includes a docker-compose.yml ready for production.
process = self._launch_process([
'git', 'clone', 'https://github.com/letsencrypt/boulder',
'--single-branch', '--depth=1', instance_path
])
process.wait()
# Allow Boulder to ignore usual limit rate policies, useful for tests.
os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'),
join(instance_path, 'test/rate-limit-policies.yml'))
# Launch the Boulder server
self._launch_process(['docker-compose', 'up', '--force-recreate'],
cwd=instance_path)
# Wait for the ACME CA server to be up.
print('=> Waiting for boulder instance to respond...')
misc.check_until_timeout(self.acme_xdist['directory_url'],
attempts=240)
# Configure challtestsrv to answer any A record request with ip of the docker host.
response = requests.post(
'http://localhost:{0}/set-default-ipv4'.format(CHALLTESTSRV_PORT),
json={'ip': '10.77.77.1'})
response.raise_for_status()
print('=> Finished boulder instance deployment.')
def _prepare_pebble_server(self):
"""Configure and launch the Pebble server"""
print('=> Starting pebble instance deployment...')
pebble_path, challtestsrv_path, pebble_config_path = pebble_artifacts.fetch(self._workspace)
# Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid
# nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment.
environ = os.environ.copy()
environ['PEBBLE_VA_NOSLEEP'] = '1'
environ['PEBBLE_WFE_NONCEREJECT'] = '0'
self._launch_process(
[pebble_path, '-config', pebble_config_path, '-dnsserver', '127.0.0.1:8053'],
env=environ)
self._launch_process(
[challtestsrv_path, '-management', ':{0}'.format(CHALLTESTSRV_PORT), '-defaultIPv6', '""',
'-defaultIPv4', '127.0.0.1', '-http01', '""', '-tlsalpn01', '""', '-https01', '""'])
# pebble_ocsp_server is imported here and not at the top of module in order to avoid a useless
# ImportError, in the case where cryptography dependency is too old to support ocsp, but
# Boulder is used instead of Pebble, so pebble_ocsp_server is not used. This is the typical
# situation of integration-certbot-oldest tox testenv.
from certbot_integration_tests.utils import pebble_ocsp_server
self._launch_process([sys.executable, pebble_ocsp_server.__file__])
# Wait for the ACME CA server to be up.
print('=> Waiting for pebble instance to respond...')
misc.check_until_timeout(self.acme_xdist['directory_url'])
print('=> Finished pebble instance deployment.')
def _prepare_boulder_server(self):
"""Configure and launch the Boulder server"""
print('=> Starting boulder instance deployment...')
instance_path = join(self._workspace, 'boulder')
# Load Boulder from git, that includes a docker-compose.yml ready for production.
process = self._launch_process([
'git', 'clone', 'https://github.com/letsencrypt/boulder',
'--single-branch', '--depth=1', instance_path
])
process.wait()
# Allow Boulder to ignore usual limit rate policies, useful for tests.
os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'),
join(instance_path, 'test/rate-limit-policies.yml'))
if self._dns_server:
# Change Boulder config to use the provided DNS server
for suffix in ["", "-remote-a", "-remote-b"]:
with open(
join(instance_path,
'test/config/va{}.json'.format(suffix)),
'r') as f:
config = json.loads(f.read())
config['va']['dnsResolvers'] = [self._dns_server]
with open(
join(instance_path,
'test/config/va{}.json'.format(suffix)),
'w') as f:
f.write(
json.dumps(config, indent=2, separators=(',', ': ')))
try:
# Launch the Boulder server
self._launch_process(['docker-compose', 'up', '--force-recreate'],
cwd=instance_path)
# Wait for the ACME CA server to be up.
print('=> Waiting for boulder instance to respond...')
misc.check_until_timeout(self.acme_xdist['directory_url'],
attempts=300)
if not self._dns_server:
# Configure challtestsrv to answer any A record request with ip of the docker host.
response = requests.post(
'http://localhost:{0}/set-default-ipv4'.format(
CHALLTESTSRV_PORT),
json={'ip': '10.77.77.1'})
response.raise_for_status()
except BaseException:
# If we failed to set up boulder, print its logs.
print('=> Boulder setup failed. Boulder logs are:')
process = self._launch_process(['docker-compose', 'logs'],
cwd=instance_path,
force_stderr=True)
process.wait()
raise
print('=> Finished boulder instance deployment.')
def _prepare_acme_server(workspace, acme_type, acme_xdist):
"""Configure and launch the ACME server, Boulder or Pebble"""
print('=> Starting {0} instance deployment...'.format(acme_type))
instance_path = join(workspace, acme_type)
try:
# Load Boulder/Pebble from git, that includes a docker-compose.yml ready for production.
_launch_command([
'git', 'clone',
'https://github.com/letsencrypt/{0}'.format(acme_type),
'--single-branch', '--depth=1', instance_path
])
if acme_type == 'boulder':
# Allow Boulder to ignore usual limit rate policies, useful for tests.
os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'),
join(instance_path, 'test/rate-limit-policies.yml'))
if acme_type == 'pebble':
with open(os.path.join(instance_path, 'docker-compose.yml'),
'r') as file_handler:
config = yaml.load(file_handler.read())
# Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid
# nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment.
config['services']['pebble'].setdefault('environment', [])\
.extend(['PEBBLE_VA_NOSLEEP=1', 'PEBBLE_WFE_NONCEREJECT=0'])
# Also disable strict mode for now, since Pebble v2.1.0 added specs in
# strict mode for which Certbot is not compliant for now.
# See https://github.com/certbot/certbot/pull/7175
# TODO: Add back -strict mode once Certbot is compliant with Pebble v2.1.0+
config['services']['pebble']['command'] = config['services']['pebble']['command']\
.replace('-strict', '')
with open(os.path.join(instance_path, 'docker-compose.yml'),
'w') as file_handler:
file_handler.write(yaml.dump(config))
# Launch the ACME CA server.
_launch_command(['docker-compose', 'up', '--force-recreate', '-d'],
cwd=instance_path)
# Wait for the ACME CA server to be up.
print('=> Waiting for {0} instance to respond...'.format(acme_type))
misc.check_until_timeout(acme_xdist['directory_url'])
# Configure challtestsrv to answer any A record request with ip of the docker host.
acme_subnet = '10.77.77' if acme_type == 'boulder' else '10.30.50'
response = requests.post(
'http://localhost:{0}/set-default-ipv4'.format(
acme_xdist['challtestsrv_port']),
json={'ip': '{0}.1'.format(acme_subnet)})
response.raise_for_status()
print('=> Finished {0} instance deployment.'.format(acme_type))
except BaseException:
print('Error while setting up {0} instance.'.format(acme_type))
raise
def _prepare_traefik_proxy(workspace, acme_xdist):
"""Configure and launch Traefik, the HTTP reverse proxy"""
print('=> Starting traefik instance deployment...')
instance_path = join(workspace, 'traefik')
traefik_subnet = '10.33.33'
traefik_api_port = 8056
try:
os.mkdir(instance_path)
with open(join(instance_path, 'docker-compose.yml'), 'w') as file_h:
file_h.write('''\
version: '3'
services:
traefik:
image: traefik
command: --api --rest
ports:
- {http_01_port}:80
- {traefik_api_port}:8080
networks:
traefiknet:
ipv4_address: {traefik_subnet}.2
networks:
traefiknet:
ipam:
config:
- subnet: {traefik_subnet}.0/24
'''.format(traefik_subnet=traefik_subnet,
traefik_api_port=traefik_api_port,
http_01_port=HTTP_01_PORT))
_launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path)
misc.check_until_timeout('http://localhost:{0}/api'.format(traefik_api_port))
config = {
'backends': {
node: {
'servers': {node: {'url': 'http://{0}.1:{1}'.format(traefik_subnet, port)}}
} for node, port in acme_xdist['http_port'].items()
},
'frontends': {
node: {
'backend': node, 'passHostHeader': True,
'routes': {node: {'rule': 'HostRegexp: {{subdomain:.+}}.{0}.wtf'.format(node)}}
} for node in acme_xdist['http_port'].keys()
}
}
response = requests.put('http://localhost:{0}/api/providers/rest'.format(traefik_api_port),
data=json.dumps(config))
response.raise_for_status()
print('=> Finished traefik instance deployment.')
except BaseException:
print('Error while setting up traefik instance.')
raise
def _prepare_traefik_proxy(workspace, acme_xdist):
"""Configure and launch Traefik, the HTTP reverse proxy"""
print('=> Starting traefik instance deployment...')
instance_path = join(workspace, 'traefik')
traefik_subnet = '10.33.33'
traefik_api_port = 8056
try:
os.mkdir(instance_path)
with open(join(instance_path, 'docker-compose.yml'), 'w') as file_h:
file_h.write('''\
version: '3'
services:
traefik:
image: traefik
command: --api --rest
ports:
- {http_01_port}:80
- {traefik_api_port}:8080
networks:
traefiknet:
ipv4_address: {traefik_subnet}.2
networks:
traefiknet:
ipam:
config:
- subnet: {traefik_subnet}.0/24
'''.format(traefik_subnet=traefik_subnet,
traefik_api_port=traefik_api_port,
http_01_port=HTTP_01_PORT))
_launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path)
misc.check_until_timeout('http://localhost:{0}/api'.format(traefik_api_port))
config = {
'backends': {
node: {
'servers': {node: {'url': 'http://{0}.1:{1}'.format(traefik_subnet, port)}}
} for node, port in acme_xdist['http_port'].items()
},
'frontends': {
node: {
'backend': node, 'passHostHeader': True,
'routes': {node: {'rule': 'HostRegexp: {{subdomain:.+}}.{0}.wtf'.format(node)}}
} for node in acme_xdist['http_port'].keys()
}
}
response = requests.put('http://localhost:{0}/api/providers/rest'.format(traefik_api_port),
data=json.dumps(config))
response.raise_for_status()
print('=> Finished traefik instance deployment.')
except BaseException:
print('Error while setting up traefik instance.')
raise
def _start_nginx(self, default_server):
self.nginx_config = config.construct_nginx_config(
self.nginx_root, self.webroot, self.http_01_port, self.tls_alpn_01_port,
self.other_port, default_server, wtf_prefix=self.worker_id)
with open(self.nginx_config_path, 'w') as file:
file.write(self.nginx_config)
process = subprocess.Popen(['nginx', '-c', self.nginx_config_path, '-g', 'daemon off;'])
assert process.poll() is None
misc.check_until_timeout('http://localhost:{0}'.format(self.http_01_port))
return process
def _start_nginx(self, default_server):
self.nginx_config = config.construct_nginx_config(
self.nginx_root, self.webroot, self.http_01_port, self.tls_alpn_01_port,
self.other_port, default_server, wtf_prefix=self.worker_id)
with open(self.nginx_config_path, 'w') as file:
file.write(self.nginx_config)
process = subprocess.Popen(['nginx', '-c', self.nginx_config_path, '-g', 'daemon off;'])
assert process.poll() is None
misc.check_until_timeout('http://localhost:{0}'.format(self.http_01_port))
return process
def _prepare_acme_server(workspace, acme_type, acme_xdist):
"""Configure and launch the ACME server, Boulder or Pebble"""
print('=> Starting {0} instance deployment...'.format(acme_type))
instance_path = join(workspace, acme_type)
try:
# Load Boulder/Pebble from git, that includes a docker-compose.yml ready for production.
_launch_command(['git', 'clone', 'https://github.com/letsencrypt/{0}'.format(acme_type),
'--single-branch', '--depth=1', instance_path])
if acme_type == 'boulder':
# Allow Boulder to ignore usual limit rate policies, useful for tests.
os.rename(join(instance_path, 'test/rate-limit-policies-b.yml'),
join(instance_path, 'test/rate-limit-policies.yml'))
if acme_type == 'pebble':
# Configure Pebble at full speed (PEBBLE_VA_NOSLEEP=1) and not randomly refusing valid
# nonce (PEBBLE_WFE_NONCEREJECT=0) to have a stable test environment.
with open(os.path.join(instance_path, 'docker-compose.yml'), 'r') as file_handler:
config = yaml.load(file_handler.read())
config['services']['pebble'].setdefault('environment', [])\
.extend(['PEBBLE_VA_NOSLEEP=1', 'PEBBLE_WFE_NONCEREJECT=0'])
with open(os.path.join(instance_path, 'docker-compose.yml'), 'w') as file_handler:
file_handler.write(yaml.dump(config))
# Launch the ACME CA server.
_launch_command(['docker-compose', 'up', '--force-recreate', '-d'], cwd=instance_path)
# Wait for the ACME CA server to be up.
print('=> Waiting for {0} instance to respond...'.format(acme_type))
misc.check_until_timeout(acme_xdist['directory_url'])
# Configure challtestsrv to answer any A record request with ip of the docker host.
acme_subnet = '10.77.77' if acme_type == 'boulder' else '10.30.50'
response = requests.post('http://localhost:{0}/set-default-ipv4'
.format(acme_xdist['challtestsrv_port']),
json={'ip': '{0}.1'.format(acme_subnet)})
response.raise_for_status()
print('=> Finished {0} instance deployment.'.format(acme_type))
except BaseException:
print('Error while setting up {0} instance.'.format(acme_type))
raise
