def test_relwalk_prefix_preserve(self):
cwd = os.path.realpath(os.getcwd())
a = list(relwalk(cwd))
b = list(relwalk(cwd + os.path.sep))
self.assertListEqual(
a, b,
"should return the same paths with or without a trailing slash")
def run(self, args):
# Ignores case sensitivity. If you're on Windows, name your files right.
conf_file_re = re.compile("([a-z]+\.conf|(default|local)\.meta)$")
if args.target is None:
self.stderr.write("Must provide the '--target' directory.\n")
return EXIT_CODE_MISSING_ARG
self.stderr.write("Combining conf files into {}\n".format(args.target))
args.source = list(_expand_glob_list(args.source))
for src in args.source:
self.stderr.write("Reading conf files from {}\n".format(src))
marker_file = os.path.join(args.target, CONTROLLED_DIR_MARKER)
if os.path.isdir(args.target):
if not os.path.isfile(os.path.join(args.target, CONTROLLED_DIR_MARKER)):
self.stderr.write("Target directory already exists, but it appears to have been"
"created by some other means. Marker file missing.\n")
return EXIT_CODE_COMBINE_MARKER_MISSING
elif args.dry_run:
self.stderr.write(
"Skipping creating destination folder {0} (dry-run)\n".format(args.target))
else:
self.stderr.write("Creating destination folder {0}\n".format(args.target))
os.mkdir(args.target)
open(marker_file, "w").write("This directory is managed by KSCONF. Don't touch\n")
# Build a common tree of all src files.
src_file_index = defaultdict(list)
for src_root in args.source:
for (root, dirs, files) in relwalk(src_root):
for fn in files:
# Todo: Add blacklist CLI support: defaults to consider: *sw[po], .git*, .bak, .~
if fn.endswith(".swp") or fn.endswith("*.bak"):
continue # pragma: no cover (peephole optimization)
src_file = os.path.join(root, fn)
src_path = os.path.join(src_root, root, fn)
src_file_index[src_file].append(src_path)
# Find a set of files that exist in the target folder, but in NO source folder (for cleanup)
target_extra_files = set()
for (root, dirs, files) in relwalk(args.target):
for fn in files:
tgt_file = os.path.join(root, fn)
if tgt_file not in src_file_index:
# Todo: Add support for additional blacklist wildcards (using fnmatch)
if fn == CONTROLLED_DIR_MARKER or fn.endswith(".bak"):
continue # pragma: no cover (peephole optimization)
target_extra_files.add(tgt_file)
for (dest_fn, src_files) in sorted(src_file_index.items()):
dest_path = os.path.join(args.target, dest_fn)
# Make missing destination folder, if missing
dest_dir = os.path.dirname(dest_path)
if not os.path.isdir(dest_dir) and not args.dry_run:
os.makedirs(dest_dir)
# Handle conf files and non-conf files separately
if not conf_file_re.search(dest_fn):
# self.stderr.write("Considering {0:50} NON-CONF Copy from source: {1!r}\n".format(dest_fn, src_files[-1]))
# Always use the last file in the list (since last directory always wins)
src_file = src_files[-1]
if args.dry_run:
if os.path.isfile(dest_path):
if file_compare(src_file, dest_path):
smart_rc = SMART_NOCHANGE
else:
if (_is_binary_file(src_file) or _is_binary_file(dest_path)):
# Binary files. Can't compare...
smart_rc = "DRY-RUN (NO-DIFF=BIN)"
else:
show_text_diff(self.stdout, dest_path, src_file)
smart_rc = "DRY-RUN (DIFF)"
else:
smart_rc = "DRY-RUN (NEW)"
else:
smart_rc = smart_copy(src_file, dest_path)
if smart_rc != SMART_NOCHANGE:
self.stderr.write(
"Copy <{0}> {1:50} from {2}\n".format(smart_rc, dest_path, src_file))
else:
# Handle merging conf files
dest = ConfFileProxy(os.path.join(args.target, dest_fn), "r+",
parse_profile=PARSECONF_MID)
srcs = [ConfFileProxy(sf, "r", parse_profile=PARSECONF_STRICT) for sf in src_files]
# self.stderr.write("Considering {0:50} CONF MERGE from source: {1!r}\n".format(dest_fn, src_files[0]))
smart_rc = merge_conf_files(dest, srcs, dry_run=args.dry_run,
banner_comment=args.banner)
if smart_rc != SMART_NOCHANGE:
self.stderr.write(
"Merge <{0}> {1:50} from {2!r}\n".format(smart_rc, dest_path,
src_files))
if True and target_extra_files: # Todo: Allow for cleanup to be disabled via CLI
self.stderr.write("Cleaning up extra files not part of source tree(s): {0} files.\n".
format(len(target_extra_files)))
for dest_fn in target_extra_files:
self.stderr.write("Remove unwanted file {0}\n".format(dest_fn))
os.unlink(os.path.join(args.target, dest_fn))
def run(self, args):
""" Install / upgrade a Splunk app from an archive file """
# Handle ignored files by preserving them as much as possible.
# Add --dry-run mode? j/k - that's what git is for!
DEBUG = KSCONF_DEBUG in os.environ
if not os.path.isfile(args.tarball):
self.stderr.write("No such file or directory {}\n".format(
args.tarball))
return EXIT_CODE_FAILED_SAFETY_CHECK
if not os.path.isdir(args.dest):
self.stderr.write(
"Destination directory does not exist: {}\n".format(args.dest))
return EXIT_CODE_FAILED_SAFETY_CHECK
f_hash = file_hash(args.tarball)
self.stdout.write("Inspecting archive: {}\n".format(
args.tarball))
new_app_name = args.app_name
# ARCHIVE PRE-CHECKS: Archive must contain only one app, no weird paths, ...
app_name = set()
app_conf = {}
files = 0
local_files = set()
a = extract_archive(args.tarball,
extract_filter=gaf_filter_name_like("app.conf"))
for gaf in sanity_checker(a):
gaf_app, gaf_relpath = gaf.path.split("/", 1)
files += 1
if gaf.path.endswith("app.conf") and gaf.payload:
conffile = StringIO(gaf.payload.decode(default_encoding))
conffile.name = os.path.join(args.tarball, gaf.path)
app_conf = parse_conf(conffile, profile=PARSECONF_LOOSE)
del conffile
elif gaf_relpath.startswith("local" + os.path.sep) or \
gaf_relpath.endswith("local.meta"):
local_files.add(gaf_relpath)
app_name.add(gaf.path.split("/", 1)[0])
del gaf_app, gaf_relpath
if len(app_name) > 1:
self.stderr.write(
"The 'unarchive' command only supports extracting a single splunk"
" app at a time.\nHowever the archive {} contains {} apps: {}\n"
"".format(args.tarball, len(app_name), ", ".join(app_name)))
return EXIT_CODE_FAILED_SAFETY_CHECK
else:
app_name = app_name.pop()
del a
if local_files:
self.stderr.write("Local {} files found in the archive. ".format(
len(local_files)))
if args.allow_local:
self.stderr.write(
"Keeping these due to the '--allow-local' flag\n")
else:
self.stderr.write("Excluding local files by default. "
"Use '--allow-local' to override.")
if not new_app_name and True: # if not --no-app-name-fixes
if app_name.endswith("-master"):
self.stdout.write(
"Automatically dropping '-master' from the app name. "
"This is often the result of a github export.\n")
# Trick, but it works...
new_app_name = app_name[:-7]
mo = re.search(r"(.*)-\d+\.[\d.-]+$", app_name)
if mo:
self.stdout.write(
"Automatically removing the version suffix from the app name. "
"'{}' will be extracted as '{}'\n".format(
app_name, mo.group(1)))
new_app_name = mo.group(1)
app_basename = new_app_name or app_name
dest_app = os.path.join(args.dest, app_basename)
self.stdout.write("Inspecting destination folder: {}\n".format(
os.path.abspath(dest_app)))
# FEEDBACK TO THE USER: UPGRADE VS INSTALL, GIT?, APP RENAME, ...
app_name_msg = app_name
git_ver = git_version()
if git_ver is None:
vc_msg = "without version control support (git not present)"
is_git = False
else:
vc_msg = "without version control support"
old_app_conf = {}
if os.path.isdir(dest_app):
mode = "upgrade"
if git_ver:
is_git = git_is_working_tree(dest_app)
try:
# Ignoring the 'local' entries since distributed apps shouldn't contain local
old_app_conf_file = os.path.join(dest_app, args.default_dir
or "default", "app.conf")
old_app_conf = parse_conf(old_app_conf_file,
profile=PARSECONF_LOOSE)
except ConfParserException:
self.stderr.write(
"Unable to read app.conf from existing install.\n")
# Assume upgrade form unknown version
else:
mode = "install"
if git_ver:
is_git = git_is_working_tree(args.dest)
if is_git:
vc_msg = "with git support"
if new_app_name and new_app_name != app_name:
app_name_msg = "{} (renamed from {})".format(
new_app_name, app_name)
def show_pkg_info(conf, label):
self.stdout.write(
"{} packaging info: '{}' by {} (version {})\n".format(
label,
conf.get("ui", {}).get("label", "Unknown"),
conf.get("launcher", {}).get("author", "Unknown"),
conf.get("launcher", {}).get("version", "Unknown")))
if old_app_conf:
show_pkg_info(old_app_conf, " Installed app")
if app_conf:
show_pkg_info(app_conf, " Tarball app")
self.stdout.write("About to {} the {} app {}.\n".format(
mode, app_name_msg, vc_msg))
existing_files = set()
if mode == "upgrade":
if is_git:
existing_files.update(git_ls_files(dest_app))
if not existing_files:
self.stderr.write(
"App is in a git repository but no files have been staged "
"or committed. Either commit or remove '{}' and try again."
"\n".format(dest_app))
return EXIT_CODE_FAILED_SAFETY_CHECK
if args.git_sanity_check == "off":
self.stdout.write(
"The 'git status' safety checks have been disabled via CLI"
"argument. Skipping.\n")
else:
d = {
# untracked, ignored
"changed": (False, False),
"untracked": (True, False),
"ignored": (True, True)
}
is_clean = git_is_clean(dest_app,
*d[args.git_sanity_check])
del d
if is_clean:
self.stdout.write(
"Git folder is clean. "
"Okay to proceed with the upgrade.\n")
else:
self.stderr.write(
"Unable to move forward without a clean working tree.\n"
"Clean up and try again. "
"Modifications are listed below.\n\n")
self.stderr.flush()
if args.git_sanity_check == "changed":
git_status_ui(dest_app, "--untracked-files=no")
elif args.git_sanity_check == "ignored":
git_status_ui(dest_app, "--ignored")
else:
git_status_ui(dest_app)
return EXIT_CODE_FAILED_SAFETY_CHECK
else:
for (root, dirs, filenames) in relwalk(dest_app):
for fn in filenames:
existing_files.add(os.path.join(root, fn))
self.stdout.write("Before upgrade. App has {} files\n".format(
len(existing_files)))
elif is_git:
self.stdout.write(
"Git clean check skipped. Not needed for a fresh app install.\n"
)
def fixup_pattern_bw(patterns, prefix=None):
modified = []
for pattern in patterns:
if pattern.startswith("./"):
if prefix:
pattern = "{0}/{1}".format(prefix, pattern[2:])
else:
pattern = pattern[2:]
modified.append(pattern)
# If a pattern like 'tags.conf' or '*.bak' is provided, use basename match (any dir)
elif "/" not in pattern:
modified.append("(^|.../)" + pattern)
else:
modified.append(pattern)
return modified
# PREP ARCHIVE EXTRACTION
installed_files = set()
excludes = list(args.exclude)
'''
for pattern in args.exclude:
# If a pattern like 'default.meta' or '*.bak' is provided, assume it's a basename match.
if "/" not in pattern:
excludes.append(".../" + pattern)
else:
excludes.append(pattern)
'''
if not args.allow_local:
for pattern in local_files:
excludes.append("./" + pattern)
excludes = fixup_pattern_bw(excludes, app_basename)
self.stderr.write(
"Extraction exclude patterns: {!r}\n".format(excludes))
path_rewrites = []
files_iter = extract_archive(args.tarball)
if True:
files_iter = sanity_checker(files_iter)
if args.default_dir:
rep = "/{}/".format(args.default_dir.strip("/"))
path_rewrites.append(("/default/", rep))
del rep
if new_app_name:
# We do have the "app_name" extracted from our first pass above, but
regex = re.compile(r'^([^/]+)(?=/)')
path_rewrites.append((regex, new_app_name))
if path_rewrites:
files_iter = gen_arch_file_remapper(files_iter, path_rewrites)
self.stdout.write("Extracting app now...\n")
for gaf in files_iter:
if match_bwlist(gaf.path, excludes, escape=False):
self.stdout.write("Skipping [blacklist] {}\n".format(gaf.path))
continue
if not is_git or args.git_mode in ("nochange", "stage"):
self.stdout.write("{0:60s} {2:o} {1:-6d}\n".format(
gaf.path, gaf.size, gaf.mode))
installed_files.add(gaf.path.split("/", 1)[1])
full_path = os.path.join(args.dest, gaf.path)
dir_exists(os.path.dirname(full_path))
with open(full_path, "wb") as fp:
fp.write(gaf.payload)
os.chmod(full_path, gaf.mode)
del fp, full_path
files_new, files_upd, files_del = _cmp_sets(installed_files,
existing_files)
if DEBUG:
print("New: \n\t{}".format("\n\t".join(sorted(files_new))))
print("Existing: \n\t{}".format("\n\t".join(sorted(files_upd))))
print("Removed: \n\t{}".format("\n\t".join(sorted(files_del))))
self.stdout.write(
"Extracted {} files: {} new, {} existing, and {} removed\n".
format(len(installed_files), len(files_new), len(files_upd),
len(files_del)))
# Filer out "removed" files; and let us keep some based on a keep-whitelist: This should
# include things like local, ".gitignore", ".gitattributes" and so on
keep_list = [".git*"]
keep_list.extend(args.keep)
if not args.allow_local:
keep_list += ["local/...", "local.meta"]
keep_list = fixup_pattern_bw(keep_list)
self.stderr.write("Keep file patterns: {!r}\n".format(keep_list))
files_to_delete = []
files_to_keep = []
for fn in files_del:
if match_bwlist(fn, keep_list, escape=False):
# How to handle a keep of "default.d/..." when we DO want to cleanup the default
# redirect folder of "default.d/10-upstream"?
# This may be an academic question since most apps will continue to send
# an ever increasing list of default files (to mask out old/unused ones)
self.stdout.write("Keeping {}\n".format(fn))
files_to_keep.append(fn)
else:
files_to_delete.append(fn)
if files_to_keep:
self.stdout.write(
"Keeping {} of {} files marked for deletion due to whitelist.\n"
.format(len(files_to_keep), len(files_del)))
git_rm_queue = []
if files_to_delete:
self.stdout.write(
"Removing files not present in the upgraded version of the app.\n"
)
for fn in files_to_delete:
path = os.path.join(dest_app, fn)
if is_git and args.git_mode in ("stage", "commit"):
self.stdout.write("git rm -f {}\n".format(path))
git_rm_queue.append(fn)
else:
self.stdout.write("rm -f {}\n".format(path))
os.unlink(path)
if git_rm_queue:
# Run 'git rm file1 file2 file3 ..." (using an xargs like mechanism)
git_cmd_iterable(["rm"], git_rm_queue, cwd=dest_app)
del git_rm_queue
if is_git:
if args.git_mode in ("stage", "commit"):
git_cmd(["add", os.path.basename(dest_app)],
cwd=os.path.dirname(dest_app))
# self.stdout.write("git add {}\n".format(os.path.basename(dest_app)))
'''
else:
self.stdout.write("git add {}\n".format(dest_app))
'''
# Is there anything to stage/commit?
if git_is_clean(os.path.dirname(dest_app), check_untracked=False):
self.stderr.write(
"No changes detected. Nothing to {}\n".format(
args.git_mode))
return
git_commit_app_name = app_conf.get("ui", {}).get(
"label", os.path.basename(dest_app))
git_commit_new_version = app_conf.get("launcher",
{}).get("version", None)
if mode == "install":
git_commit_message = "Install {}".format(git_commit_app_name)
if git_commit_new_version:
git_commit_message += " version {}".format(
git_commit_new_version)
else:
# Todo: Specify Upgrade/Downgrade/Refresh
git_commit_message = "Upgrade {}".format(git_commit_app_name)
git_commit_old_version = old_app_conf.get("launcher", {}).get(
"version", None)
if git_commit_old_version and git_commit_new_version:
git_commit_message += " version {} (was {})".format(
git_commit_new_version, git_commit_old_version)
elif git_commit_new_version:
git_commit_message += " to version {}".format(
git_commit_new_version)
# Could possibly include some CLI arg details, like what file patterns were excluded
git_commit_message += "\n\nSHA256 {} {}\n\nSplunk-App-managed-by: ksconf" \
.format(f_hash, os.path.basename(args.tarball))
git_commit_cmd = [
"commit",
os.path.basename(dest_app), "-m", git_commit_message
]
if not args.no_edit:
git_commit_cmd.append("--edit")
git_commit_cmd.extend(args.git_commit_args)
if args.git_mode == "commit":
capture_std = True if args.no_edit else False
proc = git_cmd(git_commit_cmd,
cwd=os.path.dirname(dest_app),
capture_std=capture_std)
if proc.returncode == 0:
self.stderr.write(
dedent("""\
Your changes have been committed. Please review before pushing. If you
find any issues, here are some possible solutions:
To fix issues in the last commit, edit and add the files to be fixed, then run:
git commit --amend
To roll back the last commit but KEEP the app upgrade, run:
git reset --soft HEAD^1
To roll back the last commit and REVERT the app upgrade, run:
git reset --hard HEAD^1
NOTE: Make sure you have *no* other uncommitted changes before running 'reset'.
"""))
else:
self.stderr.write(
"Git commit failed. Return code {}. Git args: git {}\n"
.format(proc.returncode, list2cmdline(git_commit_cmd)))
return EXIT_CODE_GIT_FAILURE
elif args.git_mode == "stage":
self.stdout.write("To commit later, use the following\n")
self.stdout.write("\tgit {}\n".format(
list2cmdline(git_commit_cmd).replace("\n", "\\n")))