def test_basic(self):
"""Check get/parse cycle"""
dup_time.setprevtime(10)
dup_time.setcurtime(20)
file_naming.prepare_regex(force = True)
filename = file_naming.get("inc", volume_number = 23)
log.Info(u"Inc filename: " + filename)
pr = file_naming.parse(filename)
assert pr and pr.type == "inc", pr
assert pr.start_time == 10
assert pr.end_time == 20
assert pr.volume_number == 23
assert not pr.partial
filename = file_naming.get("full-sig")
log.Info(u"Full sig filename: " + filename)
pr = file_naming.parse(filename)
assert pr.type == "full-sig"
assert pr.time == 20
assert not pr.partial
filename = file_naming.get("new-sig")
pr = file_naming.parse(filename)
assert pr.type == "new-sig"
assert pr.start_time == 10
assert pr.end_time == 20
assert not pr.partial
def test_basic(self):
"""Check get/parse cycle"""
dup_time.setprevtime(10)
dup_time.setcurtime(20)
filename = file_naming.get("inc", volume_number=23)
log.Info("Inc filename: " + filename)
pr = file_naming.parse(filename)
assert pr and pr.type == "inc", pr
assert pr.start_time == 10
assert pr.end_time == 20
assert pr.volume_number == 23
assert not pr.partial
filename = file_naming.get("full-sig")
log.Info("Full sig filename: " + filename)
pr = file_naming.parse(filename)
assert pr.type == "full-sig"
assert pr.time == 20
assert not pr.partial
filename = file_naming.get("new-sig")
pr = file_naming.parse(filename)
assert pr.type == "new-sig"
assert pr.start_time == 10
assert pr.end_time == 20
assert not pr.partial
def test_basic(self):
u"""Check get/parse cycle"""
dup_time.setprevtime(10)
dup_time.setcurtime(20)
file_naming.prepare_regex(force=True)
filename = file_naming.get(u"inc", volume_number=23)
log.Info(u"Inc filename: " + util.fsdecode(filename))
pr = file_naming.parse(filename)
assert pr and pr.type == u"inc", pr
assert pr.start_time == 10
assert pr.end_time == 20
assert pr.volume_number == 23
assert not pr.partial
filename = file_naming.get(u"full-sig")
log.Info(u"Full sig filename: " + util.fsdecode(filename))
pr = file_naming.parse(filename)
assert pr.type == u"full-sig"
assert pr.time == 20
assert not pr.partial
filename = file_naming.get(u"new-sig")
pr = file_naming.parse(filename)
assert pr.type == u"new-sig"
assert pr.start_time == 10
assert pr.end_time == 20
assert not pr.partial
def testConversion(self):
u"""test timetostring and stringtotime"""
dup_time.setcurtime()
assert type(dup_time.curtime) in integer_types
assert isinstance(dup_time.curtimestr, (str, u"".__class__))
assert (dup_time.cmp(int(dup_time.curtime), dup_time.curtimestr) == 0 or
dup_time.cmp(int(dup_time.curtime) + 1, dup_time.curtimestr) == 0)
time.sleep(1.05)
assert dup_time.cmp(time.time(), dup_time.curtime) == 1
assert dup_time.cmp(dup_time.timetostring(time.time()), dup_time.curtimestr) == 1
def testConversion(self):
"""test timetostring and stringtotime"""
dup_time.setcurtime()
assert type(dup_time.curtime) in (types.IntType, types.LongType)
assert type(dup_time.curtimestr) is types.StringType
assert (dup_time.cmp(int(dup_time.curtime), dup_time.curtimestr) == 0 or
dup_time.cmp(int(dup_time.curtime) + 1, dup_time.curtimestr) == 0)
time.sleep(1.05)
assert dup_time.cmp(time.time(), dup_time.curtime) == 1
assert dup_time.cmp(dup_time.timetostring(time.time()), dup_time.curtimestr) == 1
def testConversion(self):
"""test timetostring and stringtotime"""
dup_time.setcurtime()
assert type(dup_time.curtime) in integer_types
assert isinstance(dup_time.curtimestr, types.StringType)
assert (dup_time.cmp(int(dup_time.curtime), dup_time.curtimestr) == 0 or
dup_time.cmp(int(dup_time.curtime) + 1, dup_time.curtimestr) == 0)
time.sleep(1.05)
assert dup_time.cmp(time.time(), dup_time.curtime) == 1
assert dup_time.cmp(dup_time.timetostring(time.time()), dup_time.curtimestr) == 1
def testConversion(self):
"""test timetostring and stringtotime"""
dup_time.setcurtime()
assert type(dup_time.curtime) in (types.IntType, types.LongType)
assert type(dup_time.curtimestr) is types.StringType
assert (dup_time.cmp(int(dup_time.curtime), dup_time.curtimestr) == 0
or dup_time.cmp(
int(dup_time.curtime) + 1, dup_time.curtimestr) == 0)
time.sleep(1.05)
assert dup_time.cmp(time.time(), dup_time.curtime) == 1
assert dup_time.cmp(dup_time.timetostring(time.time()),
dup_time.curtimestr) == 1
def try_fileobj_ops(self, backend):
"""Test above try_fileobj_filename with a few filenames"""
# Must set dup_time strings because they are used by file_naming
dup_time.setcurtime(2000)
dup_time.setprevtime(1000)
# Also set profile for encryption
globals.gpg_profile = gpg.GPGProfile(passphrase = "foobar")
filename1 = file_naming.get('full', manifest = 1, gzipped = 1)
self.try_fileobj_filename(backend, filename1)
filename2 = file_naming.get('new-sig', encrypted = 1)
self.try_fileobj_filename(backend, filename2)
def try_fileobj_ops(self, backend):
"""Test above try_fileobj_filename with a few filenames"""
# Must set dup_time strings because they are used by file_naming
dup_time.setcurtime(2000)
dup_time.setprevtime(1000)
# Also set profile for encryption
globals.gpg_profile = gpg.GPGProfile(passphrase = "foobar")
filename1 = file_naming.get('full', manifest = 1, gzipped = 1)
self.try_fileobj_filename(backend, filename1)
filename2 = file_naming.get('new-sig', encrypted = 1)
self.try_fileobj_filename(backend, filename2)
def main():
output = []
def Log(s, verb_level, code=1, extra=None, force_print=False):
if verb_level <= log.getverbosity():
output.extend(s.split("\n"))
# def PrintCollectionStatus(col_stats, force_print=False):
# # raise ValueError(type(col_stats.matched_chain_pair[1]))
# output.append({
# "num_backup_sets":
# })
# log.PrintCollectionStatus = PrintCollectionStatus
results = None
try:
settings = dict()
Intersplunk.readResults(None, settings, True)
dup_time.setcurtime()
archive_dir = os.path.join(app_dir, "local", "data", "archive")
try:
os.makedirs(archive_dir)
except:
pass
if sys.argv[1] == "splunk-last-backups":
ap = argparse.ArgumentParser()
ap.add_argument("--time", type=int)
ap.add_argument("backend")
args = ap.parse_args(sys.argv[2:])
dup_globals.gpg_profile = gpg.GPGProfile()
dup_globals.gpg_profile.passphrase = os.environ["PASSPHRASE"]
backend.import_backends()
dup_globals.backend = backend.get_backend(args.backend)
if dup_globals.backup_name is None:
dup_globals.backup_name = commandline.generate_default_backup_name(
args.backend)
commandline.set_archive_dir(archive_dir)
results = []
time = args.time
col_stats = dup_collections.CollectionsStatus(
dup_globals.backend, dup_globals.archive_dir_path,
"list-current").set_values()
try:
sig_chain = col_stats.get_backup_chain_at_time(time)
except dup_collections.CollectionsError:
results.append({
"last_full_backup_time": 0,
"last_incr_backup_time": 0,
})
else:
if sig_chain.incset_list:
last_incr_backup_time = max(
[incset.end_time for incset in sig_chain.incset_list])
else:
last_incr_backup_time = 0
results.append({
"last_full_backup_time":
col_stats.get_last_full_backup_time(),
"last_incr_backup_time":
last_incr_backup_time
})
elif sys.argv[1] == "splunk-file-list":
ap = argparse.ArgumentParser()
ap.add_argument("--time")
ap.add_argument("backend")
args = ap.parse_args(sys.argv[2:])
args.time = int(args.time.split(".")[0])
dup_time.setcurtime(args.time)
dup_globals.restore_time = args.time
dup_globals.gpg_profile = gpg.GPGProfile()
dup_globals.gpg_profile.passphrase = os.environ["PASSPHRASE"]
backend.import_backends()
dup_globals.backend = backend.get_backend(args.backend)
if dup_globals.backup_name is None:
dup_globals.backup_name = commandline.generate_default_backup_name(
args.backend)
commandline.set_archive_dir(archive_dir)
results = []
col_stats = dup_collections.CollectionsStatus(
dup_globals.backend, dup_globals.archive_dir_path,
"list-current").set_values()
time = args.time
sig_chain = col_stats.get_signature_chain_at_time(time)
path_iter = diffdir.get_combined_path_iter(
sig_chain.get_fileobjs(time))
for path in path_iter:
if path.difftype != u"deleted" and path.index:
mode = bin(path.mode)[2:]
perms = ""
for p, val in enumerate(mode):
if p in (0, 3, 6):
c = "r"
elif p in (1, 4, 7):
c = "w"
elif p in (2, 5, 8):
c = "x"
perms += c if int(val) else "-"
if path.type == "dir":
perms = "d" + perms
elif path.type == "sym":
perms = "l" + perms
else:
perms = "-" + perms
results.append({
"perms": perms,
"owner": path.stat.st_uid,
"group": path.stat.st_gid,
"size": path.stat.st_size,
"modtime": path.stat.st_mtime,
"filename": os.path.join(*path.index),
})
else:
args = ["--archive-dir", archive_dir] + sys.argv[1:]
action = commandline.ProcessCommandLine(args)
log.Log = Log
try:
dup_main.do_backup(action)
except dup_collections.CollectionsError:
results = []
except SystemExit:
pass
except Exception as e:
import traceback
# sys.stderr.write(traceback.format_exc())
Intersplunk.generateErrorResults("Traceback: %s" %
traceback.format_exc())
return
if output and not results:
import time
results = [{"_raw": "\n".join(output), "_time": time.time()}]
if results:
try:
Intersplunk.outputResults(results)
except Exception:
import traceback
sys.stderr.write(traceback.format_exc())
results = Intersplunk.generateErrorResults("Traceback: %s" %
traceback.format_exc())
Intersplunk.outputResults(results)
def stream_events(self, inputs, ew):
# Splunk Enterprise calls the modular input,
# streams XML describing the inputs to stdin,
# and waits for XML on stdout describing events.
try:
output = []
def Log(s, verb_level, code=1, extra=None, force_print=False):
if verb_level <= 5:
for line in s.split("\n"):
output.append(line)
(backup_name, config) = inputs.inputs.popitem()
config_to_arg = {
"full_if_older_than": ("--full-if-older-than", "30D")
}
params = {}
for conf_opt in config_to_arg:
(_, default) = config_to_arg[conf_opt]
if default is not None:
params[conf_opt] = default
for conf_opt in config:
if conf_opt not in config_to_arg:
continue
params[conf_opt] = config[conf_opt]
args = []
for (param, value) in params.items():
(arg, _) = config_to_arg[param]
if value is True:
args.append(arg)
elif value:
args.extend((arg, value))
if "extra_duplicity_args" in config:
args.extend(shlex.split(config["extra_duplicity_args"]))
archive_dir = os.path.join(app_dir, "local", "data", "archive")
backup_base_dir = make_splunkhome_path(["etc"])
# archive_dir_relative = archive_dir[len(backup_base_dir) + 1:]
try:
os.makedirs(archive_dir)
except:
pass
if config.get("whitelist"):
whitelist_file = tempfile.NamedTemporaryFile()
with open(whitelist_file.name, "w") as f:
for entry in config["whitelist"].split(";"):
entry = os.path.expandvars(entry)
f.write("%s\n" % entry)
args.extend(("--include-filelist", whitelist_file.name))
if config.get("blacklist"):
blacklist_file = tempfile.NamedTemporaryFile()
with open(blacklist_file.name, "w") as f:
for entry in config["blacklist"].split(";"):
entry = os.path.expandvars(entry)
f.write("%s\n" % entry)
args.extend(("--exclude-filelist", blacklist_file.name))
args.extend(["--exclude", archive_dir])
args.extend(["--archive-dir", archive_dir])
args.append(backup_base_dir)
args.append(config["target_url"])
dup_time.setcurtime()
action = commandline.ProcessCommandLine(args)
log.Log = Log
dup_main.do_backup(action)
event = {}
in_stats = False
for line in output:
line = line.strip()
if line.startswith("---"):
if in_stats:
break
elif "Backup Statistics" in line:
in_stats = True
continue
elif not in_stats:
continue
key, value = line.split(" ")[:2]
event[key] = value
ew.write_event(
Event(data=json.dumps(event), sourcetype="duplicity"))
except:
import traceback
sys.stderr.write("%s\n" % traceback.print_exc())