def configure_host(self, host, config):
# Extract provisioning-specific config from host
# Establish host-specific list of aliases and their public FQDNs.
host.aliases.update({
alias.strip(): f'{alias}.{host.name}.{self.target_host}'
for alias in config.get('provision-aliases', '').strip().split()
})
# Development containers have a different internal address for the
# external aliases so that we need to explicitly override the resolver
# so that services like nginx can bind to the correct local address.
try:
addr = batou.utils.resolve(host.name)
for alias_fqdn in host.aliases.values():
output.annotate(f' alias override v4 {alias_fqdn} -> {addr}',
debug=True)
batou.utils.resolve_override[alias_fqdn] = addr
except (socket.gaierror, ValueError):
pass
try:
addr = batou.utils.resolve_v6(host.name)
for alias_fqdn in host.aliases.values():
output.annotate(f' alias override v6 {alias_fqdn} -> {addr}',
debug=True)
batou.utils.resolve_v6_override[alias_fqdn] = addr
except (socket.gaierror, ValueError):
pass
host.provision_channel = config.get('provision-channel', self.channel)
def verify(self):
if self._delayed:
self._render()
try:
with open(self.path, 'rb') as target:
current = target.read()
if current == self.content:
return
except FileNotFoundError:
current = b''
except Exception:
output.annotate('Unknown content - can\'t predict diff.')
raise batou.UpdateNeeded()
encoding = self.encoding or 'ascii'
current_text = current.decode(encoding, errors='replace')
wanted_text = self.content.decode(encoding)
for line in difflib.unified_diff(current_text.splitlines(),
wanted_text.splitlines()):
line = line.replace('\n', '')
if not line.strip():
continue
output.annotate('\t{} {}'.format(os.path.basename(self.path),
line),
red=line.startswith('-'),
green=line.startswith('+'))
raise batou.UpdateNeeded()
def report(self):
output.error(self.cmd)
output.tabular("Return code", str(self.returncode), red=True)
output.line('STDOUT', red=True)
output.annotate(self.stdout)
output.line('STDERR', red=True)
output.annotate(self.stderr)
def deploy(self, predict_only=False):
# Remember: this is a tight loop - we need to keep this code fast.
# Reset changed flag here to support triggering deploy() multiple
# times. This is mostly helpful for testing, but who knows.
self.changed = False
for sub_component in self.sub_components:
sub_component.deploy(predict_only)
if sub_component.changed:
self.changed = True
if not os.path.exists(self.workdir):
os.makedirs(self.workdir)
with self.chdir(self.workdir), self:
try:
with batou.utils.Timer('{} verify()'.format(
self._breadcrumbs)):
self.verify()
except AssertionError:
self.__trigger_event__('before-update',
predict_only=predict_only)
output.annotate(self.host.name + ' > ' + self._breadcrumbs)
if not predict_only:
self.update()
self.changed = True
def main(destination, **kw):
develop = os.environ['BATOU_DEVELOP']
if develop:
output.annotate(
'Initializing with a development copy of batou will cause your '
'project to have a reference outside its repository. '
'Use at your own risk. ')
develop = os.path.abspath(develop)
print((
'Bootstrapping new batou project in {}. This can take a while.'.format(
os.path.abspath(destination))))
if os.path.exists(destination):
print(('{} exists already. Not copying template structure.'.format(
destination)))
os.chdir(destination)
else:
source = os.path.dirname(__file__) + '/init-template'
shutil.copytree(source, destination)
os.chdir(destination)
cmd('hg -y init .')
update_bootstrap(os.environ['BATOU_VERSION'], develop)
# Need to clean up to avoid inheriting info that we're bootstrapped
# already.
for key in list(os.environ):
if key.startswith('BATOU_'):
del os.environ[key]
cmd('./batou --help')
def _ship(self, host):
head = host.rpc.git_current_head()
if head is None:
bundle_range = self.branch
else:
head = head.decode("ascii")
bundle_range = "{head}..{branch}".format(
head=head, branch=self.branch)
fd, bundle_file = tempfile.mkstemp()
os.close(fd)
out, err = cmd(
"git bundle create {file} {range}".format(
file=bundle_file, range=bundle_range),
acceptable_returncodes=[0, 128])
if "create empty bundle" in err:
return
change_size = os.stat(bundle_file).st_size
output.annotate(
"Sending {} bytes of changes".format(change_size), debug=True)
rsync = execnet.RSync(bundle_file, verbose=False)
rsync.add_target(host.gateway,
host.remote_repository + "/batou-bundle.git")
rsync.send()
os.unlink(bundle_file)
output.annotate("Unbundling changes", debug=True)
host.rpc.git_unbundle_code()
def verify(self):
with self.chdir(self.target):
if not os.path.exists(".hg"):
raise UpdateNeeded()
if not self.vcs_update:
return
if self.has_outgoing_changesets():
output.annotate(
"Hg clone at {} has outgoing changesets.".format(
self.target),
red=True,
)
if self.has_changes():
output.annotate(
"Hg clone at {} is dirty, going to lose changes.".format(
self.target),
red=True,
)
raise UpdateNeeded()
if self.revision:
long_rev = len(self.revision) == 40
if self.current_revision(long_rev) != self.revision:
raise UpdateNeeded()
if self.branch and (self.current_branch() != self.branch
or self.has_incoming_changesets()):
raise UpdateNeeded()
def call(*args, **kw):
output.annotate("rpc {}: {}(*{}, **{})".format(
self.host.fqdn, name, args, kw),
debug=True)
self.host.channel.send((name, args, kw))
while True:
message = self.host.channel.receive()
output.annotate("{}: message: {}".format(
self.host.fqdn, message),
debug=True)
type = message[0]
if type == "batou-result":
return message[1]
elif type == "batou-output":
_, output_cmd, args, kw = message
getattr(output, output_cmd)(*args, **kw)
elif type == "batou-configuration-error":
raise SilentConfigurationError()
elif type == "batou-deployment-error":
raise DeploymentError()
elif type == "batou-unknown-error":
output.error(message[1])
raise RuntimeError(
"{}: Remote exception encountered.".format(
self.host.fqdn))
elif type == "batou-error":
# Remote put out the details already.
raise RuntimeError(
"{}: Remote exception encountered.".format(
self.host.fqdn))
else:
raise RuntimeError("{}: Unknown message type {}".format(
self.host.fqdn, type))
def update(self, host):
source, target = self.root, host.remote_repository
output.annotate("rsync: {} -> {}".format(source, target), debug=True)
rsync = FilteredRSync(source, verbose=False)
# We really want to use `delete=True` here but there's an execnet issue
# preventing us to use it. See
# https://github.com/flyingcircusio/batou/issues/107
rsync.add_target(host.gateway, target)
rsync.send()
def verify(self):
stdout, stderr = self.cmd('rsync {} {}{}/ {}'.format(
self.verify_opts, self.exclude_arg, self.source, self.path))
# In case of we see non-convergent rsync runs
output.annotate('rsync result:', debug=True)
output.annotate(stdout, debug=True)
if len(stdout.strip().splitlines()) - 4 > 0:
raise batou.UpdateNeeded()
def expand(self, templatestr, args, identifier="<template>"):
if len(templatestr) > 100 * 1024:
output.error(
"You are trying to render a template that is bigger than "
"100KiB we've seen that Jinja can crash at large templates "
"and suggest you find alternatives for this. The affected "
"template starts with:")
output.annotate(templatestr[:100])
tmpl = self.env.from_string(templatestr)
tmpl.filename = identifier
return tmpl.render(**args)
def __trigger_event__(self, event, predict_only):
# We notify all components that belong to the same root.
for target in self.root.component.recursive_sub_components:
for handler in target._event_handlers.get(event, []):
if not check_event_scope(handler._event['scope'], self,
target):
continue
if predict_only:
output.annotate('Trigger {}: {}.{}'.format(
event, handler.__self__, handler.__name__))
continue
handler(self)
def evade(self, component):
if self.deployment == 'hot':
return
if self._evaded:
return
# Only try once. Keep going anyway.
self._evaded = True
output.annotate(
"\u2623 Stopping {} for cold deployment".format(self.name))
try:
self.ctl('stop {}'.format(self.name))
except Exception:
pass
def verify(self):
if self._delayed:
self._render()
with open(self.path, 'rb') as target:
current = target.read()
if current == self.content:
return
if self.encoding:
current_text = current.decode(self.encoding, errors='replace')
wanted_text = self.content.decode(self.encoding)
for line in difflib.unified_diff(current_text.splitlines(),
wanted_text.splitlines()):
output.annotate(line, debug=True)
raise batou.UpdateNeeded()
def assert_component_is_current(self, requirements=[], **kw):
"""Assert that this component has been updated more recently
than the components specified in the ``requirements``,
raise :py:class:`UpdateNeeded` otherwise.
:param list requirements: The list of components you want to
check against.
:return: ``None``, if this component is as new or newer as all
``requirements``.
:param dict kw: Arguments that are passed through to
each ``last_update`` call. The semantics depend on the components'
implementations.
:raises UpdateNeeded: if this component is older than any of the
``requirements``.
The age of a component is determined by calling ``last_updated``
on this and each requirement component.
"""
if isinstance(requirements, Component):
requirements = [requirements]
reference = self.last_updated(**kw)
if reference is None:
output.annotate(
"assert_component_is_current({}, ...): No reference".format(
self._breadcrumb),
debug=True,
)
raise batou.UpdateNeeded()
for requirement in requirements:
self |= requirement
required = requirement.last_updated(**kw)
if required is None:
continue
if reference < required:
output.annotate(
"assert_component_is_current({}, {}): {} < {}".format(
self._breadcrumb,
requirement._breadcrumb,
reference,
required,
),
debug=True,
)
raise batou.UpdateNeeded()
def update(self, host):
env = self.environment
blacklist = ['.batou', 'work', '.git', '.hg', '.vagrant', '.kitchen',
'.batou-lock']
for candidate in os.listdir(env.base_dir):
if candidate in blacklist:
continue
source = os.path.join(env.base_dir, candidate)
target = os.path.join(host.remote_base, candidate)
output.annotate("rsync: {} -> {}".format(source, target),
debug=True)
rsync = execnet.RSync(source, verbose=False)
rsync.add_target(host.gateway, target, delete=True)
rsync.send()
def resolve(host, port=0, resolve_override=resolve_override):
if host in resolve_override:
address = resolve_override[host]
output.annotate('resolved `{}` to {} (override)'.format(host, address))
else:
output.annotate('resolving `{}` (v4)'.format(host))
responses = socket.getaddrinfo(host, int(port), socket.AF_INET)
output.annotate('resolved `{}` to {}'.format(host, responses))
address = responses[0][4][0]
output.annotate('selected '.format(host, address))
return address
def cmd(
cmd,
silent=False,
ignore_returncode=False,
communicate=True,
env=None,
acceptable_returncodes=[0],
encoding="utf-8",
):
if not isinstance(cmd, str):
# We use `shell=True`, so the command needs to be a single string and
# we need to pay attention to shell quoting.
quoted_args = []
for arg in cmd:
arg = arg.replace("'", "\\'")
if " " in arg:
arg = "'{}'".format(arg)
quoted_args.append(arg)
cmd = " ".join(quoted_args)
if env is not None:
add_to_env = env
env = os.environ.copy()
env.update(add_to_env)
output.annotate("cmd: {}".format(cmd), debug=True)
process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
stdin=subprocess.PIPE,
shell=True,
env=env,
)
if not communicate:
# XXX See #12550
return process
stdout, stderr = process.communicate()
if encoding is not None:
stdout = stdout.decode(encoding, errors="replace")
stderr = stderr.decode(encoding, errors="replace")
if process.returncode not in acceptable_returncodes:
if not ignore_returncode:
raise CmdExecutionError(cmd, process.returncode, stdout, stderr)
return stdout, stderr
def verify(self):
self._force_clone = False
if not os.path.exists(self.target):
self._force_clone = True
raise UpdateNeeded()
# if the path does exist but isn't a directory, just let the error
# bubble for now, the message will at least tell something useful
with self.chdir(self.target):
if not os.path.exists(".git"):
self._force_clone = True
raise UpdateNeeded()
if self.remote_url() != self.url:
self._force_clone = True
raise UpdateNeeded()
if not self.vcs_update:
return
if self.has_outgoing_changesets():
output.annotate(
"Git clone at {} has outgoing changesets.".format(
self.target
)
)
if self.has_changes():
output.annotate(
"Git clone at {} is dirty, going to lose changes.".format(
self.target
),
red=True,
)
raise UpdateNeeded()
if self.revision and self.current_revision() != self.revision:
raise UpdateNeeded()
if self.branch and (
self.current_branch() != self.branch
or self.has_incoming_changesets()
):
raise UpdateNeeded()
def connect(self, interpreter="python3"):
if self.gateway:
output.annotate("Disconnecting ...", debug=True)
self.disconnect()
output.annotate("Connecting ...", debug=True)
# Call sudo, ensuring:
# - no password will ever be asked (fail instead)
# - we ensure a consistent set of environment variables
# irregardless of the local configuration of env_reset, etc.
CONDITIONAL_SUDO = """\
if [ -n "$ZSH_VERSION" ]; then setopt SH_WORD_SPLIT; fi;
if [ \"$USER\" = \"{user}\" ]; then \
pre=\"\"; else pre=\"sudo -ni -u {user}\"; fi; $pre\
""".format(user=self.service_user)
spec = "ssh={fqdn}//python={sudo} {interpreter}//type={method}".format(
fqdn=self.fqdn,
sudo=CONDITIONAL_SUDO,
interpreter=interpreter,
method=self.environment.connect_method,
)
if os.path.exists("ssh_config"):
spec += "//ssh_config=ssh_config"
self.gateway = execnet.makegateway(spec)
try:
self.channel = self.gateway.remote_exec(remote_core)
except IOError:
raise RuntimeError(
"Could not start batou on host `{}`. "
"The output above may contain more information. ".format(
self.fqdn))
output.annotate("Connected ...", debug=True)
def connect(self, interpreter='python3'):
if self.gateway:
output.annotate('Reconnecting ...', debug=True)
self.disconnect()
try_sudo = False
try:
self.gateway = execnet.makegateway(
"ssh={}//python={}//type={}".format(
self.fqdn, interpreter,
self.environment.connect_method))
self.channel = self.gateway.remote_exec(remote_core)
if not self.environment.service_user:
self.environment.service_user = self.rpc.whoami()
elif self.rpc.whoami() != self.environment.service_user:
try_sudo = True
self.disconnect()
except IOError:
try_sudo = True
if try_sudo:
output.annotate('Trying to switch to sudo ...', debug=True)
# Call sudo, ensuring:
# - no password will ever be asked (fail instead)
# - we ensure a consistent set of environment variables
# irregardless of the local configuration of env_reset, etc.
self.gateway = execnet.makegateway(
"ssh={}//python=sudo -ni -u {} {}//type={}".format(
self.fqdn,
self.environment.service_user,
interpreter,
self.environment.connect_method))
self.channel = self.gateway.remote_exec(remote_core)
output.annotate('Connected ...', debug=True)
def verify(self):
# Safety belt that we're acting on a clean repository.
if self.environment.deployment.dirty:
output.annotate(
"You are running a dirty deployment. This can cause "
"inconsistencies -- continuing on your own risk!", red=True)
return
try:
status, _ = cmd('hg -q stat')
except CmdExecutionError:
output.error('Unable to check repository status. '
'Is there an HG repository here?')
raise
else:
status = status.strip()
if status.strip():
output.error("Your repository has uncommitted changes.")
output.annotate("""\
I am refusing to deploy in this situation as the results will be unpredictable.
Please commit and push first.
""", red=True)
output.annotate(status, red=True)
raise DeploymentError()
try:
cmd('hg -q outgoing -l 1', acceptable_returncodes=[1])
except CmdExecutionError:
output.error("""\
Your repository has outgoing changes.
I am refusing to deploy in this situation as the results will be unpredictable.
Please push first.
""")
raise DeploymentError()
def _ship(self, host):
heads = host.rpc.hg_current_heads()
if not heads:
raise ValueError("Remote repository did not find any heads. "
"Can not continue creating a bundle.")
fd, bundle_file = tempfile.mkstemp()
os.close(fd)
bases = " ".join("--base {}".format(x) for x in heads)
cmd("hg -qy bundle {} {}".format(bases, bundle_file),
acceptable_returncodes=[0, 1])
change_size = os.stat(bundle_file).st_size
if not change_size:
return
output.annotate(
"Sending {} bytes of changes".format(change_size), debug=True)
rsync = execnet.RSync(bundle_file, verbose=False)
rsync.add_target(host.gateway,
host.remote_repository + "/batou-bundle.hg")
rsync.send()
os.unlink(bundle_file)
output.annotate("Unbundling changes", debug=True)
host.rpc.hg_unbundle_code()
def resolve_v6(host, port=0, resolve_override=resolve_v6_override):
if host in resolve_override:
address = resolve_override[host]
output.annotate('resolved `{}` to {} (override)'.format(host, address))
else:
output.annotate('resolving (v6) `{}` (getaddrinfo)'.format(host))
responses = socket.getaddrinfo(host, int(port), socket.AF_INET6)
output.annotate('resolved (v6) `{}` to {}'.format(host, responses))
address = None
for _, _, _, _, sockaddr in responses:
addr, _, _, _ = sockaddr
if addr.startswith('fe80:'):
continue
address = addr
break
if not address:
raise ValueError('No valid address found for `{}`.'.format(host))
output.annotate('selected {}'.format(address))
return address
def verify(self):
# Safety belt that we're acting on a clean repository.
if self.environment.deployment.dirty:
output.annotate(
"You are running a dirty deployment. This can cause "
"inconsistencies -- continuing on your own risk!",
red=True,
)
return
try:
status, _ = cmd("git status --porcelain")
except CmdExecutionError:
output.error("Unable to check repository status. "
"Is there a Git repository here?")
raise
else:
status = status.strip()
if status.strip():
output.error("Your repository has uncommitted changes.")
output.annotate(
"""\
I am refusing to deploy in this situation as the results will be unpredictable.
Please commit and push first.
""",
red=True,
)
output.annotate(status, red=True)
raise DeploymentError()
outgoing, _ = cmd(
"git log {remote}/{branch}..{branch} --pretty=oneline".format(
remote=self.remote, branch=self.branch),
acceptable_returncodes=[0, 128],
)
if outgoing.strip():
output.error("""\
Your repository has outgoing changes.
I am refusing to deploy in this situation as the results will be unpredictable.
Please push first.
""")
raise DeploymentError()
def connect(self, interpreter='python3'):
if self.gateway:
output.annotate('Disconnecting ...', debug=True)
self.disconnect()
output.annotate('Connecting ...', debug=True)
# Call sudo, ensuring:
# - no password will ever be asked (fail instead)
# - we ensure a consistent set of environment variables
# irregardless of the local configuration of env_reset, etc.
spec = "ssh={}//python=sudo -ni -u {} {}//type={}".format(
self.fqdn, self.environment.service_user, interpreter,
self.environment.connect_method)
if os.path.exists('ssh_config'):
spec += '//ssh_config=ssh_config'
self.gateway = execnet.makegateway(spec)
self.channel = self.gateway.remote_exec(remote_core)
output.annotate('Connected ...', debug=True)
def verify(self):
# Safety belt that we're acting on a clean repository.
if self.environment.deployment.dirty:
output.annotate(
"You are running a dirty deployment. This can cause "
"inconsistencies -- continuing on your own risk!",
red=True)
return
try:
status = hg_cmd("hg stat")
except CmdExecutionError:
output.error("Unable to check repository status. "
"Is there an HG repository here?")
raise
else:
if status:
output.error("Your repository has uncommitted changes.")
output.annotate(
"""\
I am refusing to deploy in this situation as the results will be unpredictable.
Please commit and push first.
""",
red=True)
for item in status:
output.annotate(
"{} {}".format(item['status'], item['path']), red=True)
raise DeploymentError("Uncommitted changes")
try:
cmd("hg -q outgoing -l 1", acceptable_returncodes=[1])
except CmdExecutionError:
output.error("""\
Your repository has outgoing changes.
I am refusing to deploy in this situation as the results will be unpredictable.
Please push first.
""")
raise DeploymentError("Outgoing changes")
def log_finish_configure(self):
for msg, args in self._logs:
output.annotate(msg % args)
self._logs = None
def log(self, msg, *args):
if self._logs is None:
msg = '%s: %s' % (self.host.fqdn, msg)
output.annotate(msg % args)
else:
self._logs.append((msg, args))
def __exit__(self, exc1, exc2, exc3):
self.duration = time.time() - self.started
output.annotate(self.note + ' took %fs' % self.duration, debug=True)