def _compare_stanzas(a, b, stanza_name):
kv_a, kv_common, kv_b = _cmp_sets(list(a.keys()), list(b.keys()))
if not kv_common:
# No keys in common, just swap
yield DiffOp(DIFF_OP_REPLACE, DiffStanza("stanza", stanza_name), a, b)
return
# Level 2 - Key comparisons
for key in kv_a:
yield DiffOp(DIFF_OP_DELETE, DiffStzKey("key", stanza_name, key), a[key], None)
for key in kv_b:
yield DiffOp(DIFF_OP_INSERT, DiffStzKey("key", stanza_name, key), None, b[key])
for key in kv_common:
a_ = a[key]
b_ = b[key]
if a_ == b_:
yield DiffOp(DIFF_OP_EQUAL, DiffStzKey("key", stanza_name, key), a_, b_)
else:
yield DiffOp(DIFF_OP_REPLACE, DiffStzKey("key", stanza_name, key), a_, b_)
def compare_cfgs(a, b, allow_level0=True):
'''
Return list of 5-tuples describing how to turn a into b.
.. note:: The `Opcode` tags borrowed from :class:`SequenceMatcher` class in the :mod:`difflib`
standard Python module.
Each tuple takes the form:
(tag, location, a, b)
*tag:*
========= ============================================
Value Meaning
========= ============================================
'replace' same element in both, but different values.
'delete' remove value b
'insert' insert value a
'equal' same values in both
========= ============================================
*location* is a tuple that can take the following forms:
=================== ===============================================================
Tuple form Description
=================== ===============================================================
`(0)` Global file level context (e.g., both files are the same)
`(1, stanza)` Stanzas are the same, or completely different (no shared keys)
`(2, stanza, key)` Key level, indicating
=================== ===============================================================
Possible alternatives:
https://dictdiffer.readthedocs.io/en/latest/#dictdiffer.patch
'''
delta = []
# Level 0 - Compare entire file
if allow_level0:
stanza_a, stanza_common, stanza_b = _cmp_sets(list(a.keys()),
list(b.keys()))
if a == b:
return [DiffOp(DIFF_OP_EQUAL, DiffGlobal("global"), a, b)]
if not stanza_common:
# Q: Does this specific output make the consumer's job more difficult?
# Nothing in common between these two files
# Note: Stanza renames are not detected and are out of scope.
return [DiffOp(DIFF_OP_REPLACE, DiffGlobal("global"), a, b)]
# Level 1 - Compare stanzas
# Make sure GLOBAL stanza is output first
all_stanzas = set(a.keys()).union(list(b.keys()))
if GLOBAL_STANZA in all_stanzas:
all_stanzas.remove(GLOBAL_STANZA)
all_stanzas = [GLOBAL_STANZA] + list(all_stanzas)
else:
all_stanzas = list(all_stanzas)
all_stanzas = sorted(all_stanzas)
for stanza in all_stanzas:
if stanza in a and stanza in b:
a_ = a[stanza]
b_ = b[stanza]
# Note: make sure that '==' operator continues work with custom conf parsing classes.
if a_ == b_:
delta.append(
DiffOp(DIFF_OP_EQUAL, DiffStanza("stanza", stanza), a_,
b_))
continue
kv_a, kv_common, kv_b = _cmp_sets(list(a_.keys()), list(b_.keys()))
if not kv_common:
# No keys in common, just swap
delta.append(
DiffOp(DIFF_OP_REPLACE, DiffStanza("stanza", stanza), a_,
b_))
continue
# Level 2 - Key comparisons
for key in kv_a:
delta.append(
DiffOp(DIFF_OP_DELETE, DiffStzKey("key", stanza, key),
None, a_[key]))
for key in kv_b:
delta.append(
DiffOp(DIFF_OP_INSERT, DiffStzKey("key", stanza, key),
b_[key], None))
for key in kv_common:
a__ = a_[key]
b__ = b_[key]
if a__ == b__:
delta.append(
DiffOp(DIFF_OP_EQUAL, DiffStzKey("key", stanza, key),
a__, b__))
else:
delta.append(
DiffOp(DIFF_OP_REPLACE, DiffStzKey("key", stanza, key),
a__, b__))
elif stanza in a:
# A only
delta.append(
DiffOp(DIFF_OP_DELETE, DiffStanza("stanza", stanza), None,
a[stanza]))
else:
# B only
delta.append(
DiffOp(DIFF_OP_INSERT, DiffStanza("stanza", stanza), b[stanza],
None))
return delta
def run(self, args):
""" Install / upgrade a Splunk app from an archive file """
# Handle ignored files by preserving them as much as possible.
# Add --dry-run mode? j/k - that's what git is for!
if not os.path.isfile(args.tarball):
self.stderr.write("No such file or directory {}\n".format(args.tarball))
return EXIT_CODE_FAILED_SAFETY_CHECK
if not os.path.isdir(args.dest):
self.stderr.write("Destination directory does not exist: {}\n".format(args.dest))
return EXIT_CODE_FAILED_SAFETY_CHECK
f_hash = file_hash(args.tarball)
self.stdout.write("Inspecting archive: {}\n".format(args.tarball))
new_app_name = args.app_name
# ARCHIVE PRE-CHECKS: Archive must contain only one app, no weird paths, ...
app_name = set()
app_conf = {}
files = 0
local_files = set()
a = extract_archive(args.tarball, extract_filter=gaf_filter_name_like("app.conf"))
for gaf in sanity_checker(a):
gaf_app, gaf_relpath = gaf.path.split("/", 1)
files += 1
if gaf.path.endswith("app.conf") and gaf.payload:
conffile = StringIO(gaf.payload.decode(default_encoding))
conffile.name = os.path.join(args.tarball, gaf.path)
app_conf = parse_conf(conffile, profile=PARSECONF_LOOSE)
del conffile
elif gaf_relpath.startswith("local") or gaf_relpath.endswith("local.meta"):
local_files.add(gaf_relpath)
app_name.add(gaf.path.split("/", 1)[0])
del gaf_app, gaf_relpath
if len(app_name) > 1:
self.stderr.write("The 'unarchive' command only supports extracting a single splunk"
" app at a time.\nHowever the archive {} contains {} apps: {}\n"
"".format(args.tarball, len(app_name), ", ".join(app_name)))
return EXIT_CODE_FAILED_SAFETY_CHECK
else:
app_name = app_name.pop()
del a
if local_files:
self.stderr.write("Local {} files found in the archive. ".format(len(local_files)))
if args.allow_local:
self.stderr.write("Keeping these due to the '--allow-local' flag\n")
else:
self.stderr.write("Excluding local files by default. "
"Use '--allow-local' to override.")
if not new_app_name and True: # if not --no-app-name-fixes
if app_name.endswith("-master"):
self.stdout.write("Automatically dropping '-master' from the app name. "
"This is often the result of a github export.\n")
# Trick, but it works...
new_app_name = app_name[:-7]
mo = re.search(r"(.*)-\d+\.[\d.-]+$", app_name)
if mo:
self.stdout.write("Automatically removing the version suffix from the app name. "
"'{}' will be extracted as '{}'\n".format(app_name, mo.group(1)))
new_app_name = mo.group(1)
app_basename = new_app_name or app_name
dest_app = os.path.join(args.dest, app_basename)
self.stdout.write("Inspecting destination folder: {}\n".format(os.path.abspath(dest_app)))
# FEEDBACK TO THE USER: UPGRADE VS INSTALL, GIT?, APP RENAME, ...
app_name_msg = app_name
vc_msg = "without version control support"
old_app_conf = {}
if os.path.isdir(dest_app):
mode = "upgrade"
is_git = git_is_working_tree(dest_app)
try:
# Ignoring the 'local' entries since distributed apps shouldn't contain local
old_app_conf_file = os.path.join(dest_app, args.default_dir or "default", "app.conf")
old_app_conf = parse_conf(old_app_conf_file, profile=PARSECONF_LOOSE)
except ConfParserException:
self.stderr.write("Unable to read app.conf from existing install.\n")
else:
mode = "install"
is_git = git_is_working_tree(args.dest)
if is_git:
vc_msg = "with git support"
if new_app_name and new_app_name != app_name:
app_name_msg = "{} (renamed from {})".format(new_app_name, app_name)
def show_pkg_info(conf, label):
self.stdout.write("{} packaging info: '{}' by {} (version {})\n".format(
label,
conf.get("ui", {}).get("label", "Unknown"),
conf.get("launcher", {}).get("author", "Unknown"),
conf.get("launcher", {}).get("version", "Unknown")))
if old_app_conf:
show_pkg_info(old_app_conf, " Installed app")
if app_conf:
show_pkg_info(app_conf, " Tarball app")
self.stdout.write("About to {} the {} app {}.\n".format(mode, app_name_msg, vc_msg))
existing_files = set()
if mode == "upgrade":
if is_git:
existing_files.update(git_ls_files(dest_app))
if not existing_files:
self.stderr.write("App is in a git repository but no files have been staged "
"or committed. Either commit or remove '{}' and try again."
"\n".format(dest_app))
return EXIT_CODE_FAILED_SAFETY_CHECK
if args.git_sanity_check == "off":
self.stdout.write("The 'git status' safety checks have been disabled via CLI"
"argument. Skipping.\n")
else:
d = {
# untracked, ignored
"changed": (False, False),
"untracked": (True, False),
"ignored": (True, True)
}
is_clean = git_is_clean(dest_app, *d[args.git_sanity_check])
del d
if is_clean:
self.stdout.write("Git folder is clean. "
"Okay to proceed with the upgrade.\n")
else:
self.stderr.write("Unable to move forward without a clean working tree.\n"
"Clean up and try again. "
"Modifications are listed below.\n\n")
self.stderr.flush()
if args.git_sanity_check == "changed":
git_status_ui(dest_app, "--untracked-files=no")
elif args.git_sanity_check == "ignored":
git_status_ui(dest_app, "--ignored")
else:
git_status_ui(dest_app)
return EXIT_CODE_FAILED_SAFETY_CHECK
else:
for (root, dirs, filenames) in os.walk(dest_app):
for fn in filenames:
existing_files.add(os.path.join(root, fn))
self.stdout.write("Before upgrade. App has {} files\n".format(len(existing_files)))
elif is_git:
self.stdout.write("Git clean check skipped. Not needed for a fresh app install.\n")
def fixup_pattern_bw(patterns, prefix=None):
modified = []
for pattern in patterns:
if pattern.startswith("./"):
if prefix:
pattern = "{0}/{1}".format(prefix, pattern[2:])
else:
pattern = pattern[2:]
modified.append(pattern)
# If a pattern like 'tags.conf' or '*.bak' is provided, ues basename match (any dir)
elif "/" not in pattern:
modified.append("(^|.../)" + pattern)
else:
modified.append(pattern)
return modified
# PREP ARCHIVE EXTRACTION
installed_files = set()
excludes = list(args.exclude)
'''
for pattern in args.exclude:
# If a pattern like 'default.meta' or '*.bak' is provided, assume it's a basename match.
if "/" not in pattern:
excludes.append(".../" + pattern)
else:
excludes.append(pattern)
'''
if not args.allow_local:
for pattern in local_files:
excludes.append("./" + pattern)
excludes = fixup_pattern_bw(excludes, app_basename)
self.stderr.write("Extraction exclude patterns: {!r}\n".format(excludes))
path_rewrites = []
files_iter = extract_archive(args.tarball)
if True:
files_iter = sanity_checker(files_iter)
if args.default_dir:
rep = "/{}/".format(args.default_dir.strip("/"))
path_rewrites.append(("/default/", rep))
del rep
if new_app_name:
# We do have the "app_name" extracted from our first pass above, but
regex = re.compile(r'^([^/]+)(?=/)')
path_rewrites.append((regex, new_app_name))
if path_rewrites:
files_iter = gen_arch_file_remapper(files_iter, path_rewrites)
self.stdout.write("Extracting app now...\n")
for gaf in files_iter:
if match_bwlist(gaf.path, excludes, escape=False):
self.stdout.write("Skipping [blacklist] {}\n".format(gaf.path))
continue
if not is_git or args.git_mode in ("nochange", "stage"):
self.stdout.write("{0:60s} {2:o} {1:-6d}\n".format(gaf.path, gaf.size, gaf.mode))
installed_files.add(gaf.path.split("/", 1)[1])
full_path = os.path.join(args.dest, gaf.path)
dir_exists(os.path.dirname(full_path))
with open(full_path, "wb") as fp:
fp.write(gaf.payload)
os.chmod(full_path, gaf.mode)
del fp, full_path
files_new, files_upd, files_del = _cmp_sets(installed_files, existing_files)
'''
print "New: \n\t{}".format("\n\t".join(sorted(files_new)))
print "Existing: \n\t{}".format("\n\t".join(sorted(files_upd)))
print "Removed: \n\t{}".format("\n\t".join(sorted(files_del)))
'''
self.stdout.write("Extracted {} files: {} new, {} existing, and {} removed\n".format(
len(installed_files), len(files_new), len(files_upd), len(files_del)))
# Filer out "removed" files; and let us keep some based on a keep-whitelist: This should
# include things like local, ".gitignore", ".gitattributes" and so on
keep_list = [".git*"]
keep_list.extend(args.keep)
if not args.allow_local:
keep_list += ["local/...", "local.meta"]
keep_list = fixup_pattern_bw(keep_list)
self.stderr.write("Keep file patterns: {!r}\n".format(keep_list))
files_to_delete = []
files_to_keep = []
for fn in files_del:
if match_bwlist(fn, keep_list, escape=False):
# How to handle a keep of "default.d/..." when we DO want to cleanup the default
# redirect folder of "default.d/10-upstream"?
# This may be an academic question since most apps will continue to send
# an ever increasing list of default files (to mask out old/unused ones)
self.stdout.write("Keeping {}\n".format(fn))
files_to_keep.append(fn)
else:
files_to_delete.append(fn)
if files_to_keep:
self.stdout.write("Keeping {} of {} files marked for deletion due to whitelist.\n"
.format(len(files_to_keep), len(files_del)))
git_rm_queue = []
if files_to_delete:
self.stdout.write("Removing files not present in the upgraded version of the app.\n")
for fn in files_to_delete:
path = os.path.join(dest_app, fn)
if is_git and args.git_mode in ("stage", "commit"):
self.stdout.write("git rm -f {}\n".format(path))
git_rm_queue.append(fn)
else:
self.stdout.write("rm -f {}\n".format(path))
os.unlink(path)
if git_rm_queue:
# Run 'git rm file1 file2 file3 ..." (using an xargs like mechanism)
git_cmd_iterable(["rm"], git_rm_queue, cwd=dest_app)
del git_rm_queue
if is_git:
if args.git_mode in ("stage", "commit"):
git_cmd(["add", os.path.basename(dest_app)], cwd=os.path.dirname(dest_app))
# self.stdout.write("git add {}\n".format(os.path.basename(dest_app)))
'''
else:
self.stdout.write("git add {}\n".format(dest_app))
'''
# Is there anything to stage/commit?
if git_is_clean(os.path.dirname(dest_app), check_untracked=False):
self.stderr.write("No changes detected. Nothing to {}\n".format(args.git_mode))
return
git_commit_app_name = app_conf.get("ui", {}).get("label", os.path.basename(dest_app))
git_commit_new_version = app_conf.get("launcher", {}).get("version", None)
if mode == "install":
git_commit_message = "Install {}".format(git_commit_app_name)
if git_commit_new_version:
git_commit_message += " version {}".format(git_commit_new_version)
else:
# Todo: Specify Upgrade/Downgrade/Refresh
git_commit_message = "Upgrade {}".format(
git_commit_app_name)
git_commit_old_version = old_app_conf.get("launcher", {}).get("version", None)
if git_commit_old_version and git_commit_new_version:
git_commit_message += " version {} (was {})".format(git_commit_new_version,
git_commit_old_version)
elif git_commit_new_version:
git_commit_message += " to version {}".format(git_commit_new_version)
# Could possibly include some CLI arg details, like what file patterns were excluded
git_commit_message += "\n\nSHA256 {} {}\n\nSplunk-App-managed-by: ksconf" \
.format(f_hash, os.path.basename(args.tarball))
git_commit_cmd = ["commit", os.path.basename(dest_app), "-m", git_commit_message]
if not args.no_edit:
git_commit_cmd.append("--edit")
git_commit_cmd.extend(args.git_commit_args)
if args.git_mode == "commit":
capture_std = True if args.no_edit else False
proc = git_cmd(git_commit_cmd, cwd=os.path.dirname(dest_app),
capture_std=capture_std)
if proc.returncode == 0:
self.stderr.write(dedent("""\
Your changes have been committed. Please review before pushing. If you
find any issues, here are some possible solutions:
To fix issues in the last commit, edit and add the files to be fixed, then run:
git commit --amend
To roll back the last commit but KEEP the app upgrade, run:
git reset --soft HEAD^1
To roll back the last commit and REVERT the app upgrade, run:
git reset --hard HEAD^1
NOTE: Make sure you have *no* other uncommitted changes before running 'reset'.
"""))
else:
self.stderr.write("Git commit failed. Return code {}. Git args: git {}\n"
.format(proc.returncode, list2cmdline(git_commit_cmd)))
return EXIT_CODE_GIT_FAILURE
elif args.git_mode == "stage":
self.stdout.write("To commit later, use the following\n")
self.stdout.write(
"\tgit {}\n".format(list2cmdline(git_commit_cmd).replace("\n", "\\n")))
def compare_cfgs(a, b, allow_level0=True, preserve_empty=False):
'''
Return list of 5-tuples describing how to turn a into b.
.. note:: The `Opcode` tags borrowed from :class:`SequenceMatcher` class in the :mod:`difflib`
standard Python module.
Each tuple takes the form:
(tag, location, a, b)
*tag:*
========= ============================================
Value Meaning
========= ============================================
'replace' same element in both, but different values.
'delete' remove value b
'insert' insert value a
'equal' same values in both
========= ============================================
*location* is a tuple that can take the following forms:
=================== ===============================================================
Tuple form Description
=================== ===============================================================
`(0)` Global file level context (e.g., both files are the same)
`(1, stanza)` Stanzas are the same, or completely different (no shared keys)
`(2, stanza, key)` Key level, indicating
=================== ===============================================================
Possible alternatives:
https://dictdiffer.readthedocs.io/en/latest/#dictdiffer.patch
'''
delta = []
# Level 0 - Compare entire file
if allow_level0:
stanza_a, stanza_common, stanza_b = _cmp_sets(list(a.keys()), list(b.keys()))
if a == b:
return [DiffOp(DIFF_OP_EQUAL, DiffGlobal("global"), a, b)]
if not stanza_common:
# Q: Does this specific output make the consumer's job more difficult?
# Nothing in common between these two files
# Note: Stanza renames are not detected and are out of scope.
return [DiffOp(DIFF_OP_REPLACE, DiffGlobal("global"), a, b)]
# Level 1 - Compare stanzas
# Make sure GLOBAL stanza is output first
all_stanzas = set(a.keys()).union(list(b.keys()))
if GLOBAL_STANZA in all_stanzas:
all_stanzas.remove(GLOBAL_STANZA)
all_stanzas = [GLOBAL_STANZA] + list(all_stanzas)
else:
all_stanzas = list(all_stanzas)
all_stanzas = sorted(all_stanzas)
for stanza in all_stanzas:
delta.extend(compare_stanzas(a.get(stanza), b.get(stanza), stanza, preserve_empty))
return delta