def test_attribute_replace(self):
props1 = parse_string(r"""
[dhcp]
EVAL-lease_end = lease_start+lease_duration
EXTRACT-loglevel = ^\d+ (?<log_level>[^ ]+)
EXTRACT-report-ack = ACK (?<dest_ip>[0-9.]+)/\d+ for (?<lease_duration>\d+)
""")
props2 = parse_string(r"""
[dhcp]
EVAL-lease_end = lease_start + lease_duration
EXTRACT-loglevel = ^\d+ (?<log_level>[^ ]+)
# Support IPv6
EXTRACT-report-ack = ACK (?<dest_ip>[0-9A-Fa-f.:]+)/\d+ for (?<lease_duration>\d+)
""")
delta = compare_cfgs(props1, props2)
delta_search = partial(self.find_op_by_location, delta)
self.assertEqual(delta_search("key", stanza="dhcp", key="EXTRACT-loglevel").tag, DIFF_OP_EQUAL)
lease_end = delta_search("key", stanza="dhcp", key="EVAL-lease_end")
self.assertEqual(lease_end.tag, DIFF_OP_REPLACE)
self.assertEqual(lease_end.a, "lease_start+lease_duration")
self.assertEqual(lease_end.b, "lease_start + lease_duration")
reportack = delta_search("key", stanza="dhcp", key="EXTRACT-report-ack")
self.assertEqual(reportack.tag, DIFF_OP_REPLACE)
self.assertIn(r"(?<dest_ip>[0-9A-Fa-f.:]+)", reportack.b)
self.assertNotIn(r"(?<dest_ip>[0-9A-Fa-f.:]+)", reportack.a)
def test_empty_stanzas(self):
""" Comparison should detect the different between empty and missing stanzas."""
a = parse_string("""
[common]
a = 1
b = 2
[in_a1]
live_in = a
[in_a_only]
""")
b = parse_string("""
[common]
b = 2
a = 1
[in_b1]
live_in = b
[in_b_only]
""")
delta = compare_cfgs(a, b)
delta_search = partial(self.find_op_by_location, delta)
self.assertEqual(delta_search("stanza", stanza="common").tag, DIFF_OP_EQUAL)
self.assertEqual(delta_search("stanza", stanza="in_a_only").tag, DIFF_OP_DELETE)
self.assertEqual(delta_search("stanza", stanza="in_b_only").tag, DIFF_OP_INSERT)
op = delta_search("stanza", stanza="in_b1")
self.assertEqual(op.tag, DIFF_OP_INSERT)
self.assertEqual(op.b["live_in"], "b")
op = delta_search("stanza", stanza="in_a1")
self.assertEqual(op.tag, DIFF_OP_DELETE)
self.assertEqual(op.a["live_in"], "a")
def test_summarize_compare_results(self):
c1 = parse_string(self.cfg_props_imapsync_1)
c2 = parse_string(self.cfg_props_imapsync_2)
diffs = compare_cfgs(c1, c2)
output = StringIO()
summarize_cfg_diffs(diffs, output)
out = output.getvalue()
# Very basic check for now.
self.assertRegex(out, r"\[imapsync\]\s*3 keys")
self.assertRegex(out, r"\[other2\]")
def test_write_with_compare(self):
orig = self.sample01
new = self.twd.get_path("metadata/local.meta")
md = MetaData()
md.feed_file(orig)
with open(new, "w", encoding="utf-8") as stream:
md.write_stream(stream)
a = parse_conf(orig)
b = parse_conf(new)
diffs = compare_cfgs(a, b)
self.assertEqual(len(diffs), 1)
self.assertEqual(diffs[0].tag, DIFF_OP_EQUAL)
def run(self, args):
''' Compare two configuration files. '''
args.conf1.set_parser_option(keep_comments=args.comments)
args.conf2.set_parser_option(keep_comments=args.comments)
cfg1 = args.conf1.data
cfg2 = args.conf2.data
diffs = compare_cfgs(cfg1, cfg2)
rc = show_diff(args.output,
diffs,
headers=(args.conf1.name, args.conf2.name))
if rc == EXIT_CODE_DIFF_EQUAL:
self.stderr.write("Files are the same.\n")
elif rc == EXIT_CODE_DIFF_NO_COMMON:
self.stderr.write("No common stanzas between files.\n")
return rc
def test_imballanced_stanas(self):
""" Imbalanced stanzas """
a = parse_string("""
[s0]
_ = same
[s1]
a = 1
b = 2
[s3]
y = 75
[s4]
z = 99
""")
b = parse_string("""
[s0]
_ = same
[s1]
a = 1
c = 3
[s2]
x = 50
[s4]
zelda =
""")
delta = compare_cfgs(a, b)
delta_search = partial(self.find_op_by_location, delta)
op = delta_search("stanza", stanza="s0")
self.assertEqual(op.tag, DIFF_OP_EQUAL)
self.assertEqual(op.a, op.b)
self.assertEqual(
delta_search("key", stanza="s1", key="a").tag, DIFF_OP_EQUAL)
self.assertEqual(
delta_search("key", stanza="s1", key="b").tag, DIFF_OP_DELETE)
self.assertEqual(
delta_search("key", stanza="s1", key="c").tag, DIFF_OP_INSERT)
self.assertEqual(
delta_search("stanza", stanza="s2").tag, DIFF_OP_INSERT)
self.assertEqual(
delta_search("stanza", stanza="s3").tag, DIFF_OP_DELETE)
self.assertEqual(
delta_search("stanza", stanza="s4").tag, DIFF_OP_REPLACE)
def merge_conf_files(dest, configs, dry_run=False, banner_comment=None):
# Parse all config files
cfgs = [conf.data for conf in configs]
# Merge all config files:
merged_cfg = merge_conf_dicts(*cfgs)
if banner_comment:
if not banner_comment.startswith("#"):
banner_comment = "#" + banner_comment
inject_section_comments(merged_cfg.setdefault(GLOBAL_STANZA, {}), prepend=[banner_comment])
# Either show the diff (dry-run mode) or write to the destination file
if dry_run and dest.is_file():
if os.path.isfile(dest.name):
dest_cfg = dest.data
else:
dest_cfg = {}
show_diff(sys.stdout, compare_cfgs(merged_cfg, dest_cfg),
headers=(dest.name, dest.name + "-new"))
return SMART_UPDATE
return dest.dump(merged_cfg)
def _do_promote_list(self, cfg_src, cfg_tgt, args):
out_src = deepcopy(cfg_src)
out_cfg = deepcopy(cfg_tgt)
diff = [op for op in compare_cfgs(cfg_tgt, cfg_src, allow_level0=False)
if op.tag in (DIFF_OP_INSERT, DIFF_OP_REPLACE)]
for op in diff:
if self.stanza_filters.match(op.location.stanza) ^ args.invert_match:
if args.verbose:
show_diff(self.stdout, [op])
if isinstance(op.location, DiffStanza):
# Move entire stanza
out_cfg[op.location.stanza] = self.combine_stanza(op.a, op.b)
del out_src[op.location.stanza]
else:
# Move key
out_cfg[op.location.stanza][op.location.key] = op.b
del out_src[op.location.stanza][op.location.key]
# If last remaining key in the src stanza? Then delete the entire stanza
if not out_src[op.location.stanza]:
del out_src[op.location.stanza]
return (out_src, out_cfg)
def test_compare_keys_props(self):
c1 = parse_string(self.cfg_props_imapsync_1)
c2 = parse_string(self.cfg_props_imapsync_2)
diffs = compare_cfgs(c1, c2)
op = self.find_op_by_location(diffs,
"key",
stanza="imapsync",
key="NO_BINARY_CHECK")
self.assertEqual(op.tag, DIFF_OP_REPLACE)
self.assertEqual(op.a, "true")
self.assertEqual(op.b, "false")
op = self.find_op_by_location(diffs,
"key",
stanza="imapsync",
key="LINE_BREAKER")
self.assertEqual(op.tag, DIFF_OP_EQUAL)
self.assertEqual(op.a, op.b)
self.assertTrue(op.a.startswith(
r"([\r\n]+)")) # Don't bother to match the whole thing...
op = self.find_op_by_location(diffs,
"key",
stanza="imapsync",
key="DATETIME_CONFIG")
self.assertEqual(op.tag, DIFF_OP_DELETE)
self.assertIsNone(op.a)
self.assertIsNotNone(op.b)
op = self.find_op_by_location(diffs,
"key",
stanza="imapsync",
key="description")
self.assertEqual(op.tag, DIFF_OP_INSERT)
self.assertIsNotNone(op.a)
self.assertIsNone(op.b)
def run(self, args):
if args.explode_default:
# Is this the SAME as exploding the defaults AFTER the merge?;
# I think NOT. Needs testing
cfgs = [explode_default_stanza(conf.data) for conf in args.conf]
else:
cfgs = [conf.data for conf in args.conf]
# Merge all config files:
default_cfg = merge_conf_dicts(*cfgs)
del cfgs
local_cfg = args.target.data
orig_cfg = dict(args.target.data)
if args.explode_default:
# Make a skeleton default dict; at the highest level, that ensure that all default
default_stanza = default_cfg.get(GLOBAL_STANZA,
default_cfg.get("default"))
skeleton_default = dict([(k, {}) for k in args.target.data])
skeleton_default = explode_default_stanza(skeleton_default,
default_stanza)
default_cfg = merge_conf_dicts(skeleton_default, default_cfg)
local_cfg = explode_default_stanza(local_cfg)
local_cfg = explode_default_stanza(local_cfg, default_stanza)
minz_cfg = dict(local_cfg)
# This may be a bit too simplistic. Weird interplay may exit between if [default] stanza
# and ocal [Upstream] stanza line up, but [Upstream] in our default file does not.
# XXX: Add a unit test!
diffs = compare_cfgs(default_cfg, local_cfg, allow_level0=False)
for op in diffs:
if op.tag == DIFF_OP_DELETE:
# This is normal. Don't expect all default content to be mirrored into local
continue
elif op.tag == DIFF_OP_EQUAL:
if isinstance(op.location, DiffStanza):
del minz_cfg[op.location.stanza]
else:
# Todo: Only preserve keys for stanzas where at least 1 key has been modified
if match_bwlist(op.location.key, args.preserve_key):
'''
self.stderr.write("Skipping key [PRESERVED] [{0}] key={1} value={2!r}\n"
"".format(op.location.stanza, op.location.key, op.a))
'''
continue # pragma: no cover (peephole optimization)
del minz_cfg[op.location.stanza][op.location.key]
# If that was the last remaining key in the stanza, delete the entire stanza
if not _drop_stanza_comments(minz_cfg[op.location.stanza]):
del minz_cfg[op.location.stanza]
elif op.tag == DIFF_OP_INSERT:
'''
self.stderr.write("Keeping local change: <{0}> {1!r}\n-{2!r}\n+{3!r}\n\n\n".format(
op.tag, op.location, op.b, op.a))
'''
continue
elif op.tag == DIFF_OP_REPLACE:
'''
self.stderr.write("Keep change: <{0}> {1!r}\n-{2!r}\n+{3!r}\n\n\n".format(
op.tag, op.location, op.b, op.a))
'''
continue
if args.dry_run:
if args.explode_default:
rc = show_diff(self.stdout,
compare_cfgs(orig_cfg, minz_cfg),
headers=(args.target.name,
args.target.name + "-new"))
else:
rc = show_diff(self.stdout,
compare_cfgs(local_cfg, default_cfg),
headers=(args.target.name,
args.target.name + "-new"))
return rc
if args.output:
args.output.dump(minz_cfg)
else:
args.target.dump(minz_cfg)
'''
def test_summarize_compare_results(self):
c1 = parse_string(self.cfg_props_imapsync_1)
c2 = parse_string(self.cfg_props_imapsync_2)
diffs = compare_cfgs(c1, c2)
output = StringIO()
summarize_cfg_diffs(diffs, output)
def _do_promote_interactive(self, cfg_src, cfg_tgt, args):
''' Interactively "promote" settings from one configuration file into another
Model after git's "patch" mode, from git docs:
This lets you choose one path out of a status like selection. After choosing the path, it
presents the diff between the index and the working tree file and asks you if you want to
stage the change of each hunk. You can select one of the following options and type return:
y - stage this hunk
n - do not stage this hunk
q - quit; do not stage this hunk or any of the remaining ones
a - stage this hunk and all later hunks in the file
d - do not stage this hunk or any of the later hunks in the file
g - select a hunk to go to
/ - search for a hunk matching the given regex
j - leave this hunk undecided, see next undecided hunk
J - leave this hunk undecided, see next hunk
k - leave this hunk undecided, see previous undecided hunk
K - leave this hunk undecided, see previous hunk
s - split the current hunk into smaller hunks
e - manually edit the current hunk
? - print help
Note: In git's "edit" mode you are literally editing a patch file, so you can modify both
the working tree file as well as the file that's being staged. While this is nifty, as
git's own documentation points out (in other places), that "some changes may have confusing
results". Therefore, it probably makes sense to limit what the user can edit.
============================================================================================
Options we may be able to support:
Pri k Description
--- - -----------
[1] y - stage this section or key
[1] n - do not stage this section or key
[1] q - quit; do not stage this or any of the remaining sections or attributes
[2] a - stage this section or key and all later sections in the file
[2] d - do not stage this section or key or any of the later section or key in the file
[1] s - split the section into individual attributes
[3] e - edit the current section or key
[2] ? - print help
Q: Is it less confusing to the user to adopt the 'local' and 'default' paradigm here?
Even though we know that change promotions will not *always* be between default and local.
(We can and should assume some familiarity with Splunk conf terms, less so than familiarity
with git lingo.)
'''
def prompt_yes_no(prompt):
while True:
r = input(prompt + " (y/n)")
if r.lower().startswith("y"):
return True
elif r.lower().startswith("n"):
return False
out_src = deepcopy(cfg_src)
out_cfg = deepcopy(cfg_tgt)
### Todo: IMPLEMENT A MANUAL MERGE/DIFF HERE:
# What ever is migrated, move it OUT of cfg_src, and into cfg_tgt
diff = compare_cfgs(cfg_tgt, cfg_src, allow_level0=False)
for op in diff:
if op.tag == DIFF_OP_DELETE:
# This is normal. Not all default entries will be updated in local.
continue
elif op.tag == DIFF_OP_EQUAL:
# Q: Should we simply remove everything from the source file that already lines
# up with the target? (Probably?) For now just skip...
if prompt_yes_no("Remove matching entry {0} ".format(
op.location)):
if isinstance(op.location, DiffStanza):
del out_src[op.location.stanza]
else:
del out_src[op.location.stanza][op.location.key]
else:
'''
self.stderr.write("Found change: <{0}> {1!r}\n-{2!r}\n+{3!r}\n\n\n"
.format(op.tag, op.location, op.b, op.a))
'''
if isinstance(op.location, DiffStanza):
# Move entire stanza
show_diff(self.stdout, [op])
if prompt_yes_no("Apply [{0}]".format(
op.location.stanza)):
out_cfg[op.location.stanza] = op.a
del out_src[op.location.stanza]
else:
show_diff(self.stdout, [op])
if prompt_yes_no("Apply [{0}] {1}".format(
op.location.stanza, op.location.key)):
# Move key
out_cfg[op.location.stanza][op.location.key] = op.a
del out_src[op.location.stanza][op.location.key]
# If last remaining key in the src stanza? Then delete the entire stanza
if not out_src[op.location.stanza]:
del out_src[op.location.stanza]
return (out_src, out_cfg)
def run(self, args):
if isinstance(args.target, ConfDirProxy):
# If a directory is given instead of a target file, then assume the source filename
# and target filename are the same.
# Also handle local/default meta: e.g.: ksconf promote local.meta .
source_basename = os.path.basename(args.source.name)
if source_basename == "local.meta":
args.target = args.target.get_file("default.meta")
else:
args.target = args.target.get_file(source_basename)
del source_basename
if not os.path.isfile(args.target.name):
self.stdout.write(
"Target file {} does not exist. Moving source file {} to the target."
.format(args.target.name, args.source.name))
# For windows: Close out any open file descriptors first
args.target.close()
args.source.close()
if args.keep:
shutil.copy2(args.source.name, args.target.name)
else:
shutil.move(args.source.name, args.target.name)
return
# If src/dest are the same, then the file ends up being deleted. Whoops!
if _samefile(args.source.name, args.target.name):
self.stderr.write(
"Aborting. SOURCE and TARGET are the same file!\n")
return EXIT_CODE_FAILED_SAFETY_CHECK
fp_source = file_fingerprint(args.source.name)
fp_target = file_fingerprint(args.target.name)
# Todo: Add a safety check prevent accidental merge of unrelated files.
# Scenario: promote local/props.conf into default/transforms.conf
# Possible check (1) Are basenames are different? (props.conf vs transforms.conf)
# Possible check (2) Are there key's in common? (DEST_KEY vs REPORT)
# Using #1 for now, consider if there's value in #2
bn_source = os.path.basename(args.source.name)
bn_target = os.path.basename(args.target.name)
if bn_source.endswith(".meta") and bn_target.endswith(".meta"):
# Allow local.meta -> default.meta without --force or a warning message
pass
elif bn_source != bn_target:
# Todo: Allow for interactive prompting when in interactive but not force mode.
if args.force:
self.stderr.write(
"Promoting content across conf file types ({0} --> {1}) because the "
"'--force' CLI option was set.\n".format(
bn_source, bn_target))
else:
self.stderr.write(
"Refusing to promote content between different types of configuration "
"files. {0} --> {1} If this is intentional, override this safety"
"check with '--force'\n".format(bn_source, bn_target))
return EXIT_CODE_FAILED_SAFETY_CHECK
# Todo: Preserve comments in the TARGET file. Worry with promoting of comments later...
# Parse all config files
cfg_src = args.source.data
cfg_tgt = args.target.data
if not cfg_src:
self.stderr.write(
"No settings in {}. Nothing to promote.\n".format(
args.source.name))
return EXIT_CODE_NOTHING_TO_DO
if args.mode == "ask":
# Show a summary of how many new stanzas would be copied across; how many key changes.
# And either accept all (batch) or pick selectively (batch)
delta = compare_cfgs(cfg_tgt, cfg_src, allow_level0=False)
delta = [op for op in delta if op.tag != DIFF_OP_DELETE]
summarize_cfg_diffs(delta, self.stderr)
while True:
resp = input("Would you like to apply ALL changes? (y/n/d/q)")
resp = resp[:1].lower()
if resp == 'q':
return EXIT_CODE_USER_QUIT
elif resp == 'd':
show_diff(self.stdout,
delta,
headers=(args.source.name, args.target.name))
elif resp == 'y':
args.mode = "batch"
break
elif resp == 'n':
args.mode = "interactive"
break
if args.mode == "interactive":
(cfg_final_src, cfg_final_tgt) = self._do_promote_interactive(
cfg_src, cfg_tgt, args)
else:
(cfg_final_src, cfg_final_tgt) = self._do_promote_automatic(
cfg_src, cfg_tgt, args)
# Minimize race condition: Do file mtime/hash check here. Abort on external change.
# Todo: Eventually use temporary files and atomic renames to further minimize the risk
# Todo: Make backup '.bak' files (user configurable)
# Todo: Avoid rewriting files if NO changes were made. (preserve prior backups)
# Todo: Restore file modes and such
if file_fingerprint(args.source.name, fp_source):
self.stderr.write(
"Aborting! External source file changed: {0}\n".format(
args.source.name))
return EXIT_CODE_EXTERNAL_FILE_EDIT
if file_fingerprint(args.target.name, fp_target):
self.stderr.write(
"Aborting! External target file changed: {0}\n".format(
args.target.name))
return EXIT_CODE_EXTERNAL_FILE_EDIT
# Reminder: conf entries are being removed from source and promoted into target
args.target.dump(cfg_final_tgt)
if not args.keep:
# If --keep is set, we never touch the source file.
if cfg_final_src:
args.source.dump(cfg_final_src)
else:
# Config file is empty. Should we write an empty file, or remove it?
if args.keep_empty:
args.source.dump(cfg_final_src)
else:
args.source.unlink()
def test_compare_no_common(self):
c1 = parse_string(self.cfg_macros_1)
c2 = parse_string(self.cfg_props_imapsync_1)
diffs = compare_cfgs(c1, c2)
self.assertEqual(len(diffs), 1)
self.assertEqual(diffs[0].location.type, "global")
