def download_dependencies(prefer_offline=False, redownload=False):
print("\nModules:")
counter = 1
definition = get_definition()
max_length = longest_module_name()
for module in definition["build"]:
name = module["name"]
if name.startswith("./"):
local_module_copy(module, counter, max_length)
counter += 1
continue
commit = module["commit"]
url = strip_right(module["repo"], ".git")
commit_dir = get_download_path(module)
if redownload:
rm(commit_dir, missing_ok=True)
if not os.path.exists(commit_dir):
sh(f"git clone {url} {commit_dir}")
sh(f"(cd {commit_dir} && git checkout {commit})")
target = f"out/steps/{counter:03d}_{module['name']}_{commit}/"
module["_directory"] = target
module["_counter"] = counter
subdirectory = module.get("subdirectory", None)
if not subdirectory:
cp(commit_dir, target)
else:
cp(os.path.join(commit_dir, subdirectory), target)
print(
f"{counter:03d} {pad_right(name, max_length)} @ {commit} (Downloaded)"
)
counter += 1
def _download_dependencies(config, prefer_offline=False, redownload=False):
# TODO: This function should be split in 2:
# 1. Code for downloading things into ~/.cfengine
# 2. Code for copying things into ./out
print("\nModules:")
counter = 1
max_length = config.longest_module_name()
downloads = os.path.join(cfbs_dir(), "downloads")
for module in config["build"]:
name = module["name"]
if name.startswith("./"):
local_module_copy(module, counter, max_length)
counter += 1
continue
if "commit" not in module:
user_error("module %s must have a commit property" % name)
commit = module["commit"]
if not is_a_commit_hash(commit):
user_error("'%s' is not a commit reference" % commit)
url = module.get("url") or module["repo"]
url = strip_right(url, ".git")
commit_dir = get_download_path(module)
if redownload:
rm(commit_dir, missing_ok=True)
if "subdirectory" in module:
module_dir = os.path.join(commit_dir, module["subdirectory"])
else:
module_dir = commit_dir
if not os.path.exists(module_dir):
if url.endswith(SUPPORTED_ARCHIVES):
fetch_archive(url, commit)
elif "index" in module:
sh("git clone %s %s" % (url, commit_dir))
sh("(cd %s && git checkout %s)" % (commit_dir, commit))
else:
versions = get_json(_VERSION_INDEX)
try:
checksum = versions[name][
module["version"]]["archive_sha256"]
except KeyError:
user_error("Cannot verify checksum of the '%s' module" %
name)
module_archive_url = os.path.join(_MODULES_URL, name,
commit + ".tar.gz")
fetch_archive(module_archive_url,
checksum,
directory=commit_dir,
with_index=False)
target = "out/steps/%03d_%s_%s/" % (counter, module["name"], commit)
module["_directory"] = target
module["_counter"] = counter
subdirectory = module.get("subdirectory", None)
if not subdirectory:
cp(commit_dir, target)
else:
cp(os.path.join(commit_dir, subdirectory), target)
print("%03d %s @ %s (Downloaded)" %
(counter, pad_right(name, max_length), commit))
counter += 1
def build_step(module, step, max_length):
step = step.split(" ")
operation, args = step[0], step[1:]
source = module["_directory"]
counter = module["_counter"]
destination = "out/masterfiles"
prefix = f"{counter:03d} {pad_right(module['name'], max_length)} :"
if operation == "copy":
src, dst = args
if dst in [".", "./"]:
dst = ""
print(f"{prefix} copy '{src}' 'masterfiles/{dst}'")
src, dst = os.path.join(source, src), os.path.join(destination, dst)
cp(src, dst)
elif operation == "run":
shell_command = " ".join(args)
print(f"{prefix} run '{shell_command}'")
sh(shell_command, source)
elif operation == "delete":
files = [args] if type(args) is str else args
assert len(files) > 0
as_string = " ".join([f"'{f}'" for f in files])
print(f"{prefix} delete {as_string}")
for file in files:
rm(os.path.join(source, file))
elif operation == "json":
src, dst = args
if dst in [".", "./"]:
dst = ""
print(f"{prefix} json '{src}' 'masterfiles/{dst}'")
src, dst = os.path.join(source, src), os.path.join(destination, dst)
extras, original = read_json(src), read_json(dst)
assert extras is not None
if not extras:
print(
f"Warning: '{os.path.basename(src)}' looks empty, adding nothing"
)
if original:
merged = merge_json(original, extras)
else:
merged = extras
write_json(dst, merged)
elif operation == "append":
src, dst = args
if dst in [".", "./"]:
dst = ""
print(f"{prefix} append '{src}' 'masterfiles/{dst}'")
src, dst = os.path.join(source, src), os.path.join(destination, dst)
if not os.path.exists(dst):
touch(dst)
assert os.path.isfile(dst)
sh(f"cat '{src}' >> '{dst}'")
else:
user_error(f"Unknown build step operation: {operation}")
def install_command(destination=None) -> int:
if not os.path.exists("out/masterfiles"):
r = build_command()
if r != 0:
return r
if not destination:
destination = "/var/cfengine/masterfiles"
rm(destination, missing_ok=True)
cp("out/masterfiles", destination)
return 0
def local_module_copy(module, counter, max_length):
name = module["name"]
assert name.startswith("./")
assert os.path.isfile(name) or os.path.isdir(name)
pretty_name = prettify_name(name)
target = f"out/steps/{counter:03d}_{pretty_name}_local/"
module["_directory"] = target
module["_counter"] = counter
cp(name, target + name)
print(
f"{counter:03d} {pad_right(name, max_length)} @ local (Copied)"
)
def local_module_copy(module, counter, max_length):
name = module["name"]
if not name.startswith("./"):
user_error("module %s must start with ./" % name)
if not os.path.isfile(name) and not os.path.isdir(name):
user_error("module %s does not exist" % name)
pretty_name = _prettify_name(name)
target = "out/steps/%03d_%s_local/" % (counter, pretty_name)
module["_directory"] = target
module["_counter"] = counter
if name.endswith(("/", "/.")):
# If this is a local folder, the target should be a copy of the folder
# (Don't create an extra unnecessary subfolder)
cp(name, target)
else:
# If this is not a folder it is a file
# create a copy of that file in the target folder
cp(name, target + name)
print("%03d %s @ local (Copied)" %
(counter, pad_right(name, max_length)))
def install_command(args) -> int:
if len(args) > 1:
user_error(
"Only one destination is allowed for command: cfbs install [destination]"
)
if not os.path.exists("out/masterfiles"):
r = build_command()
if r != 0:
return r
if os.getuid() == 0:
destination = "/var/cfengine/masterfiles"
if len(args) > 0:
destination = args[0]
elif os.getuid() == 0:
destination = "/var/cfengine/masterfiles"
else:
destination = os.path.join(os.environ["HOME"], ".cfagent/inputs")
if not destination.startswith("/") and not destination.startswith("./"):
destination = "./" + destination
rm(destination, missing_ok=True)
cp("out/masterfiles", destination)
print("Installed to %s" % destination)
return 0
def _perform_build_step(module, step, max_length):
step = step.split(" ")
operation, args = step[0], step[1:]
source = module["_directory"]
counter = module["_counter"]
destination = "out/masterfiles"
prefix = "%03d %s :" % (counter, pad_right(module["name"], max_length))
if operation == "copy":
src, dst = args
if dst in [".", "./"]:
dst = ""
print("%s copy '%s' 'masterfiles/%s'" % (prefix, src, dst))
src, dst = os.path.join(source, src), os.path.join(destination, dst)
cp(src, dst)
elif operation == "run":
shell_command = " ".join(args)
print("%s run '%s'" % (prefix, shell_command))
sh(shell_command, source)
elif operation == "delete":
files = [args] if type(args) is str else args
assert len(files) > 0
as_string = " ".join(["'%s'" % f for f in files])
print("%s delete %s" % (prefix, as_string))
for file in files:
rm(os.path.join(source, file))
elif operation == "json":
src, dst = args
if dst in [".", "./"]:
dst = ""
print("%s json '%s' 'masterfiles/%s'" % (prefix, src, dst))
if not os.path.isfile(os.path.join(source, src)):
user_error("'%s' is not a file" % src)
src, dst = os.path.join(source, src), os.path.join(destination, dst)
extras, original = read_json(src), read_json(dst)
if not extras:
print("Warning: '%s' looks empty, adding nothing" %
os.path.basename(src))
if original:
merged = merge_json(original, extras)
else:
merged = extras
write_json(dst, merged)
elif operation == "append":
src, dst = args
if dst in [".", "./"]:
dst = ""
print("%s append '%s' 'masterfiles/%s'" % (prefix, src, dst))
src, dst = os.path.join(source, src), os.path.join(destination, dst)
if not os.path.exists(dst):
touch(dst)
assert os.path.isfile(dst)
sh("cat '%s' >> '%s'" % (src, dst))
elif operation == "directory":
src, dst = args
if dst in [".", "./"]:
dst = ""
print("{} directory '{}' 'masterfiles/{}'".format(prefix, src, dst))
dstarg = dst # save this for adding .cf files to inputs
src, dst = os.path.join(source, src), os.path.join(destination, dst)
defjson = os.path.join(destination, "def.json")
merged = read_json(defjson)
if not merged:
merged = {}
if "classes" not in merged:
merged["classes"] = {}
if "services_autorun_bundles" not in merged["classes"]:
merged["classes"]["services_autorun_bundles"] = ["any"]
inputs = []
for root, dirs, files in os.walk(src):
for f in files:
if f.endswith(".cf"):
inputs.append(os.path.join(dstarg, f))
cp(os.path.join(root, f),
os.path.join(destination, dstarg, f))
elif f == "def.json":
extra = read_json(os.path.join(root, f))
if extra:
merged = merge_json(merged, extra)
else:
cp(os.path.join(root, f),
os.path.join(destination, dstarg, f))
if "inputs" in merged:
merged["inputs"].extend(inputs)
else:
merged["inputs"] = inputs
write_json(defjson, merged)
else:
user_error("Unknown build step operation: %s" % operation)