def filter_files_for_archival(conn, config, extension):
"""
Scan the working tree for files that end with `extension`.
Add the file in the same folder with the same name sans
the extension to the archival branch, and remove the file
with the extension from the archival branch.
Excludes descending into dotfile folders.
"""
wt = config.get_working_tree()
for dirpath, dirnames, filenames in os.walk(wt):
parts = os.path.split(dirpath)
has_dotfile = False
for part in parts:
if part.startswith("."):
has_dotfile = True
break
if has_dotfile:
continue
for fname in filenames:
if fname.endswith(extension):
transformed = os.path.splitext(fname)[0]
if os.path.exists(os.path.join(dirpath, transformed)):
conn.run("cd {} && git add -f {}".format(
shellquote(dirpath), shellquote(transformed)))
else:
warn("Could not find file '{}' for archival.".format(
transformed))
conn.run("cd {} && git rm -f {}".format(
shellquote(dirpath), shellquote(fname)))
def install_local_rpm(conn, path):
"""
Copy a local package to the target host and then install it.
"""
remote_path = conn.run("mktemp").stdout.rstrip()
conn.put(path, remote_path)
install_rpm(conn, remote_path)
conn.run("rm -f {}".format(shellquote(remote_path)))
def docker_rm(conn, container):
"""
Stop a docker container from a deployed image.
"""
args = ['docker', 'rm']
args.append(container)
args = [shellquote(arg) for arg in args]
cmd = ' '.join(args)
conn.sudo(cmd)
def deploy_config(args):
"""
Deploy a configuration.
"""
cfg = load_config(args.config, args.stage, args.sudo_passwd, args.pty)
invoker = cfg.invoker
archive_path = config_deployer.create_local_archive(invoker, cfg, args.commit)
if not args.archive is None:
invoker.run("mv {} {}".format(shellquote(archive_path), shellquote(args.archive)))
try:
pool = filter_conn_pool(cfg.conn_pool, set(args.exclude_host))
for conn in pool:
print_host_banner(conn)
config_deployer.deploy_config(
conn,
cfg,
archive_path,
move_etc=(not args.no_etc) and (args.archive is None))
finally:
invoker.run("rm {}".format(shellquote(archive_path)))
def execute_shell(args):
"""
Execute arbitrary remote commands.
"""
cfg = load_config(args.config, args.stage, args.sudo_passwd, args.pty)
cmd = ' '.join([shellquote(arg) for arg in args.arg])
pool = filter_conn_pool(cfg.conn_pool, set(args.exclude_host))
for conn in pool:
print_host_banner(conn)
if args.sudo:
conn.sudo(cmd)
else:
conn.run(cmd)
def docker_run(conn, config, stop_and_remove=None):
"""
Run a docker container from a deployed image.
"""
if stop_and_remove is not None:
docker_stop(conn, stop_and_remove)
docker_rm(conn, stop_and_remove)
build_name = config.get_docker_build_name()
args = ['docker', 'run']
args.extend(config.get_docker_run_args())
args.append(build_name)
args = [shellquote(arg) for arg in args]
cmd = ' '.join(args)
conn.sudo(cmd)
def build_docker_target(conn, config, remote_stagedir):
"""
Build a docker image from the configuration in `remote_stagedir`.
"""
with conn.cd(remote_stagedir):
build_name = config.get_docker_build_name()
build_path = config.get_docker_build_path()
rm_flag = config.get_docker_build_rm()
build_args = config.get_docker_build_args()
build_options = config.get_docker_build_options()
args = ['docker', 'build']
if rm_flag:
args.append("--rm")
if not build_name is None:
args.append("-t")
args.append(shellquote(build_name))
for k, v in build_args.items():
args.append("--build-arg")
args.append(shellquote("{}={}".format(k, v)))
args.extend(build_options)
args.append(shellquote(build_path))
command = ' '.join(args)
conn.sudo('''bash -c "cd {} && {}"'''.format(shellquote(remote_stagedir),
command))
def create_local_archive(conn, config, src_commit):
"""
Create local archive and return its path.
"""
wt = config.get_working_tree()
if src_commit is None:
src_branch = config.get_config_branch()
else:
src_branch = src_commit
if not os.path.exists(wt):
raise Exit("Working tree '{}' does not exist!".format(wt))
has_secrets = os.path.exists(os.path.join(wt, ".gitsecret"))
with conn.cd(wt):
result = conn.run("git diff-index --quiet HEAD --", warn=True)
if result.failed:
if confirm(
"There are uncommited changes in the working tree. Reset to HEAD?"
):
conn.run("git reset --hard HEAD")
else:
raise Exit(
"Can't use working tree with uncommitted changes. Stash, commit, or reset."
)
conn.run("git checkout {}".format(shellquote(src_branch)))
if has_secrets:
conn.run("git secret reveal")
ttools.fill_templates(config)
archive_branch = "{}-archive".format(src_branch)
conn.run("git branch -D {}".format(shellquote(archive_branch)),
warn=True)
conn.run("git checkout -b {}".format(shellquote(archive_branch)))
filter_files_for_archival(conn, config, ".secret")
filter_files_for_archival(conn, config, ".template")
if has_secrets:
secrets_file_name = config.get_secrets_file_name()
if secrets_file_name is not None:
conn.run("git rm -f {}".format(shellquote(secrets_file_name)))
if os.path.exists(os.path.join(wt, '.gitignore')):
conn.run("git rm -f .gitignore")
if os.path.exists(os.path.join(wt, '.gitsecret')):
conn.run("git rm -rf .gitsecret")
conn.run("git commit -m 'Decrypted for deployment.'", warn=True)
archive_path = conn.run("mktemp").stdout.rstrip()
conn.run("git archive --format tgz -o {} HEAD".format(
shellquote(archive_path)))
conn.run("git checkout {}".format(shellquote(src_branch)))
conn.run("git branch -D {}".format(shellquote(archive_branch)))
return archive_path
def parse_fields(conn, perm_file):
"""
Parse the fields of a permissions file.
Yield each permission.
"""
result = conn.sudo("cat {}".format(shellquote(perm_file)))
lines = result.stdout.splitlines()
for line in lines:
if line.strip() == "":
continue
if line.strip().startswith("#"):
continue
fields = line.split(":")
if len(fields) != 4:
warn("Permission '{}' in file {} is mal-formed.".format(
line, perm_file))
continue
fname = fields[0]
user = fields[1]
group = fields[2]
perms = fields[3]
yield (fname, user, group, perms)
def apply_permissions(conn, config, folder, perm_file='__perms__'):
"""
Descend recursively into each folder starting with `folder`.
Look for a file with a name that matches the value of `perm_file`.
Parse the permission file and apply the permissions to files in
the folder as applicable.
"""
result = conn.sudo("find {} -name {} -print".format(
shellquote(folder), shellquote(perm_file)))
lines = result.stdout.splitlines()
for line in lines:
if line == '[sudo] password: ':
continue
if line.strip() == "":
continue
dirpth = os.path.dirname(line)
for fname, user, group, perms in parse_fields(conn, line):
pth = os.path.join(dirpth, fname)
conn.sudo("chown {}:{} {}".format(shellquote(user),
shellquote(group),
shellquote(pth)))
conn.sudo("chmod {} {}".format(shellquote(perms), shellquote(pth)))
conn.sudo("rm -f {}".format(line))
def deploy_config(conn, config, archive_path, move_etc=True):
"""
Deploy a configuration.
:param move_etc:`(True)/False - Move the embedded 'etc' config to the '/etc' root.`
:param local_archive:`Don't deploy-- instead create a local archive at this path.`
"""
remote_config_folder = config.get_remote_config_folder()
remote_archive = conn.run("mktemp").stdout.rstrip()
paths = conn.put(archive_path, remote_archive)
remote_stagedir = conn.run("mktemp -d").stdout.rstrip()
with conn.cd(remote_stagedir):
conn.run("tar xzvf {}".format(shellquote(remote_archive)))
conn.run("rm {}".format(shellquote(remote_archive)))
config_owner = config.get_config_owner()
config_group = config.get_config_group()
conn.sudo("chown -R {}:{} {}".format(shellquote(config_owner),
shellquote(config_group),
shellquote(remote_stagedir)))
folder_perms = config.get_config_folder_perms()
file_perms = config.get_config_file_perms()
ad_hoc_perms = config.get_ad_hoc_perms()
if folder_perms.lower() != "skip":
conn.sudo("chmod {} -R {}".format(folder_perms,
shellquote(remote_stagedir)))
if file_perms.lower() != "skip":
conn.sudo("find {} -type f -exec chmod {} {{}} \;".format(
shellquote(remote_stagedir), file_perms))
for path, perm in ad_hoc_perms.items():
conn.sudo("chmod {} {}".format(shellquote(perm), shellquote(path)))
apply_permissions(conn, config, remote_stagedir)
if move_etc:
remote_staged_etc = os.path.join(remote_stagedir, "etc")
_copy_etc(conn, remote_stagedir, 'etc', '/etc')
conn.sudo("rm -Rf {}".format(shellquote(remote_staged_etc)))
is_docker_build_target = config.is_docker_build_target()
if is_docker_build_target:
build_docker_target(conn, config, remote_stagedir)
if not remote_config_folder is None:
conn.sudo("rm -Rf {}".format(remote_config_folder))
conn.sudo("mv {} {}".format(remote_stagedir, remote_config_folder))
result = conn.sudo("which restorecon", warn=True)
if not result.failed:
conn.sudo("restorecon -R {}".format(
shellquote(remote_config_folder)))
else:
conn.sudo("rm -Rf {}".format(shellquote(remote_stagedir)))