def test_basic(self):
"""Check get/parse cycle"""
dup_time.setprevtime(10)
dup_time.setcurtime(20)
file_naming.prepare_regex(force = True)
filename = file_naming.get("inc", volume_number = 23)
log.Info(u"Inc filename: " + filename)
pr = file_naming.parse(filename)
assert pr and pr.type == "inc", pr
assert pr.start_time == 10
assert pr.end_time == 20
assert pr.volume_number == 23
assert not pr.partial
filename = file_naming.get("full-sig")
log.Info(u"Full sig filename: " + filename)
pr = file_naming.parse(filename)
assert pr.type == "full-sig"
assert pr.time == 20
assert not pr.partial
filename = file_naming.get("new-sig")
pr = file_naming.parse(filename)
assert pr.type == "new-sig"
assert pr.start_time == 10
assert pr.end_time == 20
assert not pr.partial
def test_more(self):
"""More file_parsing tests"""
file_naming.prepare_regex(force=True)
pr = file_naming.parse(globals.file_prefix +
globals.file_prefix_signature +
"dns.h112bi.h14rg0.st.g")
assert pr, pr
assert pr.type == "new-sig"
assert pr.end_time == 1029826800
if not globals.short_filenames:
pr = file_naming.parse(
globals.file_prefix + globals.file_prefix_signature +
"duplicity-new-signatures.2002-08-18T00:04:30-07:00.to.2002-08-20T00:00:00-07:00.sigtar.gpg"
)
assert pr, pr
assert pr.type == "new-sig"
assert pr.end_time == 1029826800
pr = file_naming.parse(globals.file_prefix +
globals.file_prefix_signature +
"dfs.h5dixs.st.g")
assert pr, pr
assert pr.type == "full-sig"
assert pr.time == 1036954144, repr(pr.time)
def test_partial(self):
u"""Test addition of partial flag"""
file_naming.prepare_regex(force=True)
pr = file_naming.parse(config.file_prefix +
config.file_prefix_signature +
b"dns.h112bi.h14rg0.st.p.g")
assert pr, pr
assert pr.partial
assert pr.type == u"new-sig"
assert pr.end_time == 1029826800
if not config.short_filenames:
pr = file_naming.parse(
config.file_prefix + config.file_prefix_signature +
b"duplicity-new-signatures.2002-08-18T00:04:30-07:00.to.2002-08-20T00:00:00-07:00.sigtar.part.gpg"
)
assert pr, pr
assert pr.partial
assert pr.type == u"new-sig"
assert pr.end_time == 1029826800
pr = file_naming.parse(config.file_prefix +
config.file_prefix_signature +
b"dfs.h5dixs.st.p.g")
assert pr, pr
assert pr.partial
assert pr.type == u"full-sig"
assert pr.time == 1036954144, repr(pr.time)
def test_basic(self):
u"""Check get/parse cycle"""
dup_time.setprevtime(10)
dup_time.setcurtime(20)
file_naming.prepare_regex(force=True)
filename = file_naming.get(u"inc", volume_number=23)
log.Info(u"Inc filename: " + util.fsdecode(filename))
pr = file_naming.parse(filename)
assert pr and pr.type == u"inc", pr
assert pr.start_time == 10
assert pr.end_time == 20
assert pr.volume_number == 23
assert not pr.partial
filename = file_naming.get(u"full-sig")
log.Info(u"Full sig filename: " + util.fsdecode(filename))
pr = file_naming.parse(filename)
assert pr.type == u"full-sig"
assert pr.time == 20
assert not pr.partial
filename = file_naming.get(u"new-sig")
pr = file_naming.parse(filename)
assert pr.type == u"new-sig"
assert pr.start_time == 10
assert pr.end_time == 20
assert not pr.partial
def test_basic(self):
"""Check get/parse cycle"""
dup_time.setprevtime(10)
dup_time.setcurtime(20)
filename = file_naming.get("inc", volume_number=23)
log.Info("Inc filename: " + filename)
pr = file_naming.parse(filename)
assert pr and pr.type == "inc", pr
assert pr.start_time == 10
assert pr.end_time == 20
assert pr.volume_number == 23
assert not pr.partial
filename = file_naming.get("full-sig")
log.Info("Full sig filename: " + filename)
pr = file_naming.parse(filename)
assert pr.type == "full-sig"
assert pr.time == 20
assert not pr.partial
filename = file_naming.get("new-sig")
pr = file_naming.parse(filename)
assert pr.type == "new-sig"
assert pr.start_time == 10
assert pr.end_time == 20
assert not pr.partial
def test_suffix(self):
"""Test suffix (encrypt/compressed) encoding and generation"""
filename = file_naming.get("inc", manifest = 1, gzipped = 1)
pr = file_naming.parse(filename)
assert pr and pr.compressed == 1
assert pr.manifest
filename2 = file_naming.get("full", volume_number = 23, encrypted = 1)
pr = file_naming.parse(filename2)
assert pr and pr.encrypted == 1
assert pr.volume_number == 23
def test_suffix(self):
"""Test suffix (encrypt/compressed) encoding and generation"""
filename = file_naming.get("inc", manifest=1, gzipped=1)
pr = file_naming.parse(filename)
assert pr and pr.compressed == 1
assert pr.manifest
filename2 = file_naming.get("full", volume_number=23, encrypted=1)
pr = file_naming.parse(filename2)
assert pr and pr.encrypted == 1
assert pr.volume_number == 23
def get_signature_chains(self, local, filelist=None):
u"""
Find chains in archive_dir_path (if local is true) or backend
Use filelist if given, otherwise regenerate. Return value is
pair (list of chains, list of signature paths not in any
chains).
"""
def get_filelist():
if filelist is not None:
return filelist
elif local:
if self.action != u"replicate":
return self.archive_dir_path.listdir()
else:
return []
else:
return self.backend.list()
def get_new_sigchain():
u"""
Return new empty signature chain
"""
if local:
return SignatureChain(True, self.archive_dir_path)
else:
return SignatureChain(False, self.backend)
# Build initial chains from full sig filenames
chains, new_sig_filenames = [], []
for filename in get_filelist():
pr = file_naming.parse(filename)
if pr:
if pr.type == u"full-sig":
new_chain = get_new_sigchain()
assert new_chain.add_filename(filename, pr)
chains.append(new_chain)
elif pr.type == u"new-sig":
new_sig_filenames.append(filename)
# Try adding new signatures to existing chains
orphaned_filenames = []
new_sig_filenames.sort(
key=lambda x: int(file_naming.parse(x).start_time))
for sig_filename in new_sig_filenames:
for chain in chains:
if chain.add_filename(sig_filename):
break
else:
orphaned_filenames.append(sig_filename)
return (chains, orphaned_filenames)
def delete(self):
"""
Remove all files in set, both local and remote
"""
rfn = self.get_filenames()
rfn.reverse()
try:
self.backend.delete(rfn)
except Exception:
log.Debug(_("BackupSet.delete: missing %s") % [util.ufn(f) for f in rfn])
pass
if self.action not in ["collection-status", "replicate"]:
local_filename_list = globals.archive_dir_path.listdir()
else:
local_filename_list = []
for lfn in local_filename_list:
pr = file_naming.parse(lfn)
if (pr and pr.time == self.time and
pr.start_time == self.start_time and
pr.end_time == self.end_time):
try:
globals.archive_dir_path.append(lfn).delete()
except Exception:
log.Debug(_("BackupSet.delete: missing %s") % [util.ufn(f) for f in lfn])
pass
util.release_lockfile()
def get_fileobj_write(self, filename,
parseresults = None,
sizelist = None):
"""
Return fileobj opened for writing, which will cause the file
to be written to the backend on close().
The file will be encoded as specified in parseresults (or as
read from the filename), and stored in a temp file until it
can be copied over and deleted.
If sizelist is not None, it should be set to an empty list.
The number of bytes will be inserted into the list.
"""
if not parseresults:
parseresults = file_naming.parse(filename)
assert parseresults, "Filename %s not correctly parsed" % filename
tdp = dup_temp.new_tempduppath(parseresults)
def close_file_hook():
"""This is called when returned fileobj is closed"""
self.put(tdp, filename)
if sizelist is not None:
tdp.setdata()
sizelist.append(tdp.getsize())
tdp.delete()
fh = dup_temp.FileobjHooked(tdp.filtered_open("wb"))
fh.addhook(close_file_hook)
return fh
def delete(self):
"""
Remove all files in set, both local and remote
"""
rfn = self.get_filenames()
rfn.reverse()
try:
self.backend.delete(rfn)
except Exception:
log.Debug(
_("BackupSet.delete: missing %s") % [util.ufn(f) for f in rfn])
pass
for lfn in globals.archive_dir.listdir():
pr = file_naming.parse(lfn)
if (pr and pr.time == self.time
and pr.start_time == self.start_time
and pr.end_time == self.end_time):
try:
globals.archive_dir.append(lfn).delete()
except Exception:
log.Debug(
_("BackupSet.delete: missing %s") %
[util.ufn(f) for f in lfn])
pass
util.release_lockfile()
def get_fileobj_duppath(dirpath, partname, permname, remname):
"""
Return a file object open for writing, will write to filename
Data will be processed and written to a temporary file. When the
return fileobject is closed, rename to final position. filename
must be a recognizable duplicity data file.
"""
if not globals.restart:
td = tempdir.TemporaryDirectory(dirpath.name)
tdpname = td.mktemp()
tdp = TempDupPath(tdpname, parseresults = file_naming.parse(partname))
fh = FileobjHooked(tdp.filtered_open("wb"), tdp = tdp, dirpath = dirpath,
partname = partname, permname = permname, remname = remname)
else:
dp = path.DupPath(dirpath.name, index = (partname,))
fh = FileobjHooked(dp.filtered_open("ab"), tdp = None, dirpath = dirpath,
partname = partname, permname = permname, remname = remname)
def rename_and_forget():
tdp.rename(dirpath.append(partname))
td.forget(tdpname)
if not globals.restart:
fh.addhook(rename_and_forget)
return fh
def delete(self):
"""
Remove all files in set, both local and remote
"""
rfn = self.get_filenames()
rfn.reverse()
try:
self.backend.delete(rfn)
except Exception:
log.Debug(_("BackupSet.delete: missing %s") % [util.ufn(f) for f in rfn])
pass
if self.action not in ["collection-status"]:
local_filename_list = globals.archive_dir.listdir()
else:
local_filename_list = []
for lfn in local_filename_list:
pr = file_naming.parse(lfn)
if (pr and pr.time == self.time and
pr.start_time == self.start_time and
pr.end_time == self.end_time):
try:
globals.archive_dir.append(lfn).delete()
except Exception:
log.Debug(_("BackupSet.delete: missing %s") % [util.ufn(f) for f in lfn])
pass
util.release_lockfile()
def add_filename(self, filename, pr = None):
"""
Add new sig filename to current chain. Return true if fits
"""
if not pr:
pr = file_naming.parse(filename)
if not pr:
return None
if self.fullsig:
if pr.type != "new-sig":
return None
if pr.start_time != self.end_time:
return None
self.inclist.append(filename)
self.check_times([pr.end_time])
self.end_time = pr.end_time
return 1
else:
if pr.type != "full-sig":
return None
self.fullsig = filename
self.check_times([pr.time, pr.time])
self.start_time, self.end_time = pr.time, pr.time
return 1
def add_filename(self, filename):
"""
Add a filename to given set. Return true if it fits.
The filename will match the given set if it has the right
times and is of the right type. The information will be set
from the first filename given.
@param filename: name of file to add
@type filename: string
"""
pr = file_naming.parse(filename)
if not pr or not (pr.type == "full" or pr.type == "inc"):
return False
if not self.info_set:
self.set_info(pr)
else:
if pr.type != self.type:
return False
if pr.time != self.time:
return False
if (pr.start_time != self.start_time
or pr.end_time != self.end_time):
return False
if pr.manifest:
self.set_manifest(filename)
else:
assert pr.volume_number is not None
assert not self.volume_name_dict.has_key(pr.volume_number), \
(self.volume_name_dict, filename)
self.volume_name_dict[pr.volume_number] = filename
return True
def set_manifest(self, remote_filename):
u"""
Add local and remote manifest filenames to backup set
"""
assert not self.remote_manifest_name, \
u"Cannot set filename of remote manifest to %s; already set to %s." % (
remote_filename,
self.remote_manifest_name,
)
self.remote_manifest_name = remote_filename
if self.action != u"replicate":
local_filename_list = config.archive_dir_path.listdir()
else:
local_filename_list = []
for local_filename in local_filename_list:
pr = file_naming.parse(local_filename)
if (pr and pr.manifest and pr.type == self.type
and pr.time == self.time
and pr.start_time == self.start_time
and pr.end_time == self.end_time):
self.local_manifest_path = \
config.archive_dir_path.append(local_filename)
self.set_files_changed()
break
def delete(self):
"""
Remove all files in set, both local and remote
"""
rfn = self.get_filenames()
rfn.reverse()
try:
self.backend.delete(rfn)
except Exception:
log.Debug("BackupSet.delete: missing %s" % rfn)
pass
for lfn in globals.archive_dir.listdir():
pr = file_naming.parse(lfn)
if (pr and pr.time == self.time
and pr.start_time == self.start_time
and pr.end_time == self.end_time and pr.type != "new-sig"):
# do not remove new sigs from the cache:
# they aren't removed from the remote archive,
# and subsequent backups will have to resync
# which is bad if running non-interactive with encrypt-key
try:
globals.archive_dir.append(lfn).delete()
except Exception:
log.Debug("BackupSet.delete: missing %s" % lfn)
pass
def add_filename(self, filename, pr=None):
"""
Add new sig filename to current chain. Return true if fits
"""
if not pr:
pr = file_naming.parse(filename)
if not pr:
return None
if self.fullsig:
if pr.type != "new-sig":
return None
if pr.start_time != self.end_time:
return None
self.inclist.append(filename)
self.check_times([pr.end_time])
self.end_time = pr.end_time
return 1
else:
if pr.type != "full-sig":
return None
self.fullsig = filename
self.check_times([pr.time, pr.time])
self.start_time, self.end_time = pr.time, pr.time
return 1
def get_fileobj_duppath(dirpath, partname, permname, remname, overwrite=False):
"""
Return a file object open for writing, will write to filename
Data will be processed and written to a temporary file. When the
return fileobject is closed, rename to final position. filename
must be a recognizable duplicity data file.
"""
if not globals.restart:
td = tempdir.TemporaryDirectory(dirpath.name)
tdpname = td.mktemp()
tdp = TempDupPath(tdpname, parseresults=file_naming.parse(partname))
fh = FileobjHooked(tdp.filtered_open("wb"), tdp=tdp, dirpath=dirpath,
partname=partname, permname=permname, remname=remname)
else:
dp = path.DupPath(dirpath.name, index=(partname,))
mode = "ab"
if overwrite:
mode = "wb"
fh = FileobjHooked(dp.filtered_open(mode), tdp=None, dirpath=dirpath,
partname=partname, permname=permname, remname=remname)
def rename_and_forget():
tdp.rename(dirpath.append(partname))
td.forget(tdpname)
if not globals.restart:
fh.addhook(rename_and_forget)
return fh
def delete(self):
u"""
Remove all files in set, both local and remote
"""
rfn = self.get_filenames()
rfn.reverse()
try:
self.backend.delete(rfn)
except Exception:
log.Debug(
_(u"BackupSet.delete: missing %s") %
[util.fsdecode(f) for f in rfn])
pass
if self.action != u"replicate":
local_filename_list = config.archive_dir_path.listdir()
else:
local_filename_list = []
for lfn in local_filename_list:
pr = file_naming.parse(lfn)
if (pr and pr.time == self.time
and pr.start_time == self.start_time
and pr.end_time == self.end_time):
try:
config.archive_dir_path.append(lfn).delete()
except Exception:
log.Debug(
_(u"BackupSet.delete: missing %s") %
[util.fsdecode(f) for f in lfn])
pass
util.release_lockfile()
def get_fileobj_write(self, filename, parseresults=None, sizelist=None):
"""
Return fileobj opened for writing, which will cause the file
to be written to the backend on close().
The file will be encoded as specified in parseresults (or as
read from the filename), and stored in a temp file until it
can be copied over and deleted.
If sizelist is not None, it should be set to an empty list.
The number of bytes will be inserted into the list.
"""
if not parseresults:
parseresults = file_naming.parse(filename)
assert parseresults, "Filename %s not correctly parsed" % filename
tdp = dup_temp.new_tempduppath(parseresults)
def close_file_hook():
"""This is called when returned fileobj is closed"""
self.put(tdp, filename)
if sizelist is not None:
tdp.setdata()
sizelist.append(tdp.getsize())
tdp.delete()
fh = dup_temp.FileobjHooked(tdp.filtered_open("wb"))
fh.addhook(close_file_hook)
return fh
def add_filename(self, filename):
"""
Add a filename to given set. Return true if it fits.
The filename will match the given set if it has the right
times and is of the right type. The information will be set
from the first filename given.
@param filename: name of file to add
@type filename: string
"""
pr = file_naming.parse(filename)
if not pr or not (pr.type == "full" or pr.type == "inc"):
return False
if not self.info_set:
self.set_info(pr)
else:
if pr.type != self.type:
return False
if pr.time != self.time:
return False
if (pr.start_time != self.start_time or
pr.end_time != self.end_time):
return False
if pr.manifest:
self.set_manifest(filename)
else:
assert pr.volume_number is not None
assert not self.volume_name_dict.has_key(pr.volume_number), \
(self.volume_name_dict, filename)
self.volume_name_dict[pr.volume_number] = filename
return True
def delete(self):
"""
Remove all files in set, both local and remote
"""
rfn = self.get_filenames()
rfn.reverse()
try:
self.backend.delete(rfn)
except Exception:
log.Debug("BackupSet.delete: missing %s" % rfn)
pass
for lfn in globals.archive_dir.listdir():
pr = file_naming.parse(lfn)
if (pr
and pr.time == self.time
and pr.start_time == self.start_time
and pr.end_time == self.end_time
and pr.type != "new-sig" ):
# do not remove new sigs from the cache:
# they aren't removed from the remote archive,
# and subsequent backups will have to resync
# which is bad if running non-interactive with encrypt-key
try:
globals.archive_dir.append(lfn).delete()
except Exception:
log.Debug("BackupSet.delete: missing %s" % lfn)
pass
def test_more(self):
"""More file_parsing tests"""
pr = file_naming.parse("dns.h112bi.h14rg0.st.g")
assert pr, pr
assert pr.type == "new-sig"
assert pr.end_time == 1029826800L
if not globals.short_filenames:
pr = file_naming.parse("duplicity-new-signatures.2002-08-18T00:04:30-07:00.to.2002-08-20T00:00:00-07:00.sigtar.gpg")
assert pr, pr
assert pr.type == "new-sig"
assert pr.end_time == 1029826800L
pr = file_naming.parse("dfs.h5dixs.st.g")
assert pr, pr
assert pr.type == "full-sig"
assert pr.time == 1036954144, repr(pr.time)
def get_signature_chains(self, local, filelist=None):
"""
Find chains in archive_dir (if local is true) or backend
Use filelist if given, otherwise regenerate. Return value is
pair (list of chains, list of signature paths not in any
chains).
"""
def get_filelist():
if filelist is not None:
return filelist
elif local:
if self.action not in ["collection-status"]:
return self.archive_dir.listdir()
else:
return []
else:
return self.backend.list()
def get_new_sigchain():
"""
Return new empty signature chain
"""
if local:
return SignatureChain(True, self.archive_dir)
else:
return SignatureChain(False, self.backend)
# Build initial chains from full sig filenames
chains, new_sig_filenames = [], []
for filename in get_filelist():
pr = file_naming.parse(filename)
if pr:
if pr.type == "full-sig":
new_chain = get_new_sigchain()
assert new_chain.add_filename(filename, pr)
chains.append(new_chain)
elif pr.type == "new-sig":
new_sig_filenames.append(filename)
# compare by file time
def by_start_time(a, b):
return int(file_naming.parse(a).start_time) - int(file_naming.parse(b).start_time)
# Try adding new signatures to existing chains
orphaned_filenames = []
new_sig_filenames.sort(by_start_time)
for sig_filename in new_sig_filenames:
for chain in chains:
if chain.add_filename(sig_filename):
break
else:
orphaned_filenames.append(sig_filename)
return (chains, orphaned_filenames)
def test_partial(self):
"""Test addition of partial flag"""
file_naming.prepare_regex(force = True)
pr = file_naming.parse(globals.file_prefix + globals.file_prefix_signature + "dns.h112bi.h14rg0.st.p.g")
assert pr, pr
assert pr.partial
assert pr.type == "new-sig"
assert pr.end_time == 1029826800L
if not globals.short_filenames:
pr = file_naming.parse(globals.file_prefix + globals.file_prefix_signature + "duplicity-new-signatures.2002-08-18T00:04:30-07:00.to.2002-08-20T00:00:00-07:00.sigtar.part.gpg")
assert pr, pr
assert pr.partial
assert pr.type == "new-sig"
assert pr.end_time == 1029826800L
pr = file_naming.parse(globals.file_prefix + globals.file_prefix_signature + "dfs.h5dixs.st.p.g")
assert pr, pr
assert pr.partial
assert pr.type == "full-sig"
assert pr.time == 1036954144, repr(pr.time)
def set_values(self, sig_chain_warning=1):
"""
Set values from archive_dir and backend.
Returns self for convenience. If sig_chain_warning is set to None,
do not warn about unnecessary sig chains. This is because there may
naturally be some unecessary ones after a full backup.
"""
self.values_set = 1
# get remote filename list
backend_filename_list = self.backend.list()
log.Debug(ngettext("%d file exists on backend",
"%d files exist on backend",
len(backend_filename_list)) %
len(backend_filename_list))
# get local filename list
if self.action not in ["collection-status"]:
local_filename_list = self.archive_dir.listdir()
else:
local_filename_list = []
log.Debug(ngettext("%d file exists in cache",
"%d files exist in cache",
len(local_filename_list)) %
len(local_filename_list))
# check for partial backups
partials = []
for local_filename in local_filename_list:
pr = file_naming.parse(local_filename)
if pr and pr.partial:
partials.append(local_filename)
# get various backup sets and chains
(backup_chains, self.orphaned_backup_sets,
self.incomplete_backup_sets) = \
self.get_backup_chains(partials + backend_filename_list)
backup_chains = self.get_sorted_chains(backup_chains)
self.all_backup_chains = backup_chains
assert len(backup_chains) == len(self.all_backup_chains), \
"get_sorted_chains() did something more than re-ordering"
local_sig_chains, self.local_orphaned_sig_names = \
self.get_signature_chains(True)
remote_sig_chains, self.remote_orphaned_sig_names = \
self.get_signature_chains(False, filelist=backend_filename_list)
self.set_matched_chain_pair(local_sig_chains + remote_sig_chains,
backup_chains)
self.warn(sig_chain_warning)
return self
def set_values(self, sig_chain_warning=1):
"""
Set values from archive_dir_path and backend.
Returns self for convenience. If sig_chain_warning is set to None,
do not warn about unnecessary sig chains. This is because there may
naturally be some unecessary ones after a full backup.
"""
self.values_set = 1
# get remote filename list
backend_filename_list = self.backend.list()
log.Debug(ngettext("%d file exists on backend",
"%d files exist on backend",
len(backend_filename_list)) %
len(backend_filename_list))
# get local filename list
if self.action not in ["collection-status", "replicate"]:
local_filename_list = self.archive_dir_path.listdir()
else:
local_filename_list = []
log.Debug(ngettext("%d file exists in cache",
"%d files exist in cache",
len(local_filename_list)) %
len(local_filename_list))
# check for partial backups
partials = []
for local_filename in local_filename_list:
pr = file_naming.parse(local_filename)
if pr and pr.partial:
partials.append(local_filename)
# get various backup sets and chains
(backup_chains, self.orphaned_backup_sets,
self.incomplete_backup_sets) = \
self.get_backup_chains(partials + backend_filename_list)
backup_chains = self.get_sorted_chains(backup_chains)
self.all_backup_chains = backup_chains
assert len(backup_chains) == len(self.all_backup_chains), \
"get_sorted_chains() did something more than re-ordering"
local_sig_chains, self.local_orphaned_sig_names = \
self.get_signature_chains(True)
remote_sig_chains, self.remote_orphaned_sig_names = \
self.get_signature_chains(False, filelist=backend_filename_list)
self.set_matched_chain_pair(local_sig_chains + remote_sig_chains,
backup_chains)
self.warn(sig_chain_warning)
return self
def test_partial(self):
"""Test addition of partial flag"""
pr = file_naming.parse("dns.h112bi.h14rg0.st.p.g")
assert pr, pr
assert pr.partial
assert pr.type == "new-sig"
assert pr.end_time == 1029826800L
if not globals.short_filenames:
pr = file_naming.parse(
"duplicity-new-signatures.2002-08-18T00:04:30-07:00.to.2002-08-20T00:00:00-07:00.sigtar.part.gpg"
)
assert pr, pr
assert pr.partial
assert pr.type == "new-sig"
assert pr.end_time == 1029826800L
pr = file_naming.parse("dfs.h5dixs.st.p.g")
assert pr, pr
assert pr.partial
assert pr.type == "full-sig"
assert pr.time == 1036954144, repr(pr.time)
def to_final(self):
"""
We are finished, rename to final, gzip if needed.
"""
src = self.dirpath.append(self.partname)
tgt = self.dirpath.append(self.permname)
src_iter = SrcIter(src)
pr = file_naming.parse(self.permname)
if pr.compressed:
gpg.GzipWriteFile(src_iter, tgt.name, size=sys.maxint)
os.unlink(src.name)
else:
os.rename(src.name, tgt.name)
def to_final(self):
"""
We are finished, rename to final, gzip if needed.
"""
src = self.dirpath.append(self.partname)
tgt = self.dirpath.append(self.permname)
src_iter = SrcIter(src)
pr = file_naming.parse(self.permname)
if pr.compressed:
gpg.GzipWriteFile(src_iter, tgt.name, size = sys.maxint)
os.unlink(src.name)
else:
os.rename(src.name, tgt.name)
def get_fileobj_read(self, filename, parseresults=None):
"""
Return fileobject opened for reading of filename on backend
The file will be downloaded first into a temp file. When the
returned fileobj is closed, the temp file will be deleted.
"""
if not parseresults:
parseresults = file_naming.parse(filename)
assert parseresults, "Filename not correctly parsed"
tdp = dup_temp.new_tempduppath(parseresults)
self.get(filename, tdp)
tdp.setdata()
return tdp.filtered_open_with_delete("rb")
def get_filenames(self, time=None):
u"""
Return ordered list of filenames in set, up to a provided time
"""
if self.fullsig:
l = [self.fullsig]
else:
l = []
inclist = self.inclist
if time:
inclist = [n for n in inclist if file_naming.parse(n).end_time <= time]
l.extend(inclist)
return l
def get_filenames(self, time=None):
"""
Return ordered list of filenames in set, up to a provided time
"""
if self.fullsig:
l = [self.fullsig]
else:
l = []
inclist = self.inclist
if time:
inclist = filter(lambda n: file_naming.parse(n).end_time <= time, inclist)
l.extend(inclist)
return l
def add_filename(self, filename, pr=None):
u"""
Add a filename to given set. Return true if it fits.
The filename will match the given set if it has the right
times and is of the right type. The information will be set
from the first filename given.
@param filename: name of file to add
@type filename: string
@param pr: pre-computed result of file_naming.parse(filename)
@type pr: Optional[ParseResults]
"""
if not pr:
pr = file_naming.parse(filename)
if not pr or not (pr.type == u"full" or pr.type == u"inc"):
return False
if not self.info_set:
self.set_info(pr)
else:
if pr.type != self.type:
return False
if pr.time != self.time:
return False
if (pr.start_time != self.start_time
or pr.end_time != self.end_time):
return False
if bool(pr.encrypted) != bool(self.encrypted):
if self.partial and pr.encrypted:
self.encrypted = pr.encrypted
if pr.manifest:
self.set_manifest(filename)
else:
assert pr.volume_number is not None
assert pr.volume_number not in self.volume_name_dict, \
u"""Volume %d is already in the volume list as "%s".
"%s" has the same volume number.
Please check your command line and retry.""" % (
pr.volume_number,
util.fsdecode(self.volume_name_dict[pr.volume_number]),
util.fsdecode(filename)
)
self.volume_name_dict[pr.volume_number] = filename
return True
def __init__(self, base, index=(), parseresults=None):
"""
DupPath initializer
The actual filename (no directory) must be the single element
of the index, unless parseresults is given.
"""
if parseresults:
self.pr = parseresults
else:
assert len(index) == 1
self.pr = file_naming.parse(index[0])
assert self.pr, "must be a recognizable duplicity file"
Path.__init__(self, base, index)
def to_remote(self):
"""
We have written the last checkpoint, now encrypt or compress
and send a copy of it to the remote for final storage.
"""
pr = file_naming.parse(self.remname)
src = self.dirpath.append(self.partname)
tgt = self.dirpath.append(self.remname)
src_iter = SrcIter(src)
if pr.compressed:
gpg.GzipWriteFile(src_iter, tgt.name, size = sys.maxint)
elif pr.encrypted:
gpg.GPGWriteFile(src_iter, tgt.name, globals.gpg_profile, size = sys.maxint)
else:
os.system("cp -p \"%s\" \"%s\"" % (src.name, tgt.name))
globals.backend.move(tgt) #@UndefinedVariable
def restore_get_enc_fileobj(backend, filename, volume_info):
"""Return plaintext fileobj from encrypted filename on backend """
parseresults = file_naming.parse(filename)
if filename in filename_tdp:
tdp = filename_tdp[filename]
else:
tdp = dup_temp.new_tempduppath(parseresults)
filename_tdp[filename] = tdp
backend.get(filename, tdp)
if not restore_check_hash(volume_info, tdp):
return None
fileobj = tdp.filtered_open_with_delete("rb")
if parseresults.encrypted and globals.gpg_profile.sign_key:
restore_add_sig_check(fileobj)
return fileobj
def to_remote(self):
"""
We have written the last checkpoint, now encrypt or compress
and send a copy of it to the remote for final storage.
"""
pr = file_naming.parse(self.remname)
src = self.dirpath.append(self.partname)
tgt = self.dirpath.append(self.remname)
src_iter = SrcIter(src)
if pr.compressed:
gpg.GzipWriteFile(src_iter, tgt.name, size=sys.maxsize)
elif pr.encrypted:
gpg.GPGWriteFile(src_iter, tgt.name, globals.gpg_profile, size=sys.maxsize)
else:
shutil.copyfile(src.name, tgt.name)
globals.backend.move(tgt) # @UndefinedVariable
def add_to_sets(filename):
u"""
Try adding filename to existing sets, or make new one
"""
pr = file_naming.parse(filename)
for set in sets: # pylint: disable=redefined-builtin
if set.add_filename(filename, pr):
log.Debug(_(u"File %s is part of known set") % (util.fsdecode(filename),))
break
else:
log.Debug(_(u"File %s is not part of a known set; creating new set") % (util.fsdecode(filename),))
new_set = BackupSet(self.backend, self.action)
if new_set.add_filename(filename, pr):
sets.append(new_set)
else:
log.Debug(_(u"Ignoring file (rejected by backup set) '%s'") % util.fsdecode(filename))
def __init__(self, base, index = (), parseresults = None):
"""
DupPath initializer
The actual filename (no directory) must be the single element
of the index, unless parseresults is given.
"""
if parseresults:
self.pr = parseresults
else:
assert len(index) == 1
self.pr = file_naming.parse(index[0])
assert self.pr, "must be a recognizable duplicity file"
Path.__init__(self, base, index)
def get_filenames(self):
"""
Return sorted list of (remote) filenames of files in set
"""
assert self.info_set
volume_num_list = self.volume_name_dict.keys()
volume_num_list.sort()
volume_filenames = [self.volume_name_dict[x] for x in volume_num_list]
if self.remote_manifest_name:
# For convenience of implementation for restart support, we treat
# local partial manifests as this set's remote manifest. But
# when specifically asked for a list of remote filenames, we
# should not include it.
pr = file_naming.parse(self.remote_manifest_name)
if pr and not pr.partial:
volume_filenames.append(self.remote_manifest_name)
return volume_filenames
def set_manifest(self, remote_filename):
"""
Add local and remote manifest filenames to backup set
"""
assert not self.remote_manifest_name, (self.remote_manifest_name,
remote_filename)
self.remote_manifest_name = remote_filename
for local_filename in globals.archive_dir.listdir():
pr = file_naming.parse(local_filename)
if (pr and pr.manifest and pr.type == self.type
and pr.time == self.time
and pr.start_time == self.start_time
and pr.end_time == self.end_time):
self.local_manifest_path = \
globals.archive_dir.append(local_filename)
break
def get_filenames(self):
"""
Return sorted list of (remote) filenames of files in set
"""
assert self.info_set
volume_num_list = self.volume_name_dict.keys()
volume_num_list.sort()
volume_filenames = [self.volume_name_dict[x] for x in volume_num_list]
if self.remote_manifest_name:
# For convenience of implementation for restart support, we treat
# local partial manifests as this set's remote manifest. But
# when specifically asked for a list of remote filenames, we
# should not include it.
pr = file_naming.parse(self.remote_manifest_name)
if not pr or not pr.partial:
volume_filenames.append(self.remote_manifest_name)
return volume_filenames
def set_manifest(self, remote_filename):
"""
Add local and remote manifest filenames to backup set
"""
assert not self.remote_manifest_name, (self.remote_manifest_name,
remote_filename)
self.remote_manifest_name = remote_filename
for local_filename in globals.archive_dir.listdir():
pr = file_naming.parse(local_filename)
if (pr and pr.manifest
and pr.type == self.type
and pr.time == self.time
and pr.start_time == self.start_time
and pr.end_time == self.end_time):
self.local_manifest_path = \
globals.archive_dir.append(local_filename)
break
def delete(self):
"""
Remove all files in set, both local and remote
"""
rfn = self.get_filenames()
rfn.reverse()
try:
self.backend.delete(rfn)
except Exception:
log.Debug("BackupSet.delete: missing %s" % rfn)
pass
for lfn in globals.archive_dir.listdir():
pr = file_naming.parse(lfn)
if pr and pr.time == self.time and pr.start_time == self.start_time and pr.end_time == self.end_time:
try:
globals.archive_dir.append(lfn).delete()
except Exception:
log.Debug("BackupSet.delete: missing %s" % lfn)
pass
def to_remote(self):
"""
We have written the last checkpoint, now encrypt or compress
and send a copy of it to the remote for final storage.
"""
pr = file_naming.parse(self.remname)
src = self.dirpath.append(self.partname)
tgt = self.dirpath.append(self.remname)
src_iter = SrcIter(src)
if pr.compressed:
gpg.GzipWriteFile(src_iter, tgt.name, size = sys.maxint)
elif pr.encrypted:
gpg.GPGWriteFile(src_iter, tgt.name, globals.gpg_profile, size = sys.maxint)
else:
os.system("cp -p \"%s\" \"%s\"" % (src.name, tgt.name))
globals.backend.put(tgt) #@UndefinedVariable
try:
util.ignore_missing(os.unlink, tgt.name)
except Exception, e:
log.Warn(_("Unable to delete %s: %s" % (tgt.name, str(e))))
def to_remote(self):
u"""
We have written the last checkpoint, now encrypt or compress
and send a copy of it to the remote for final storage.
"""
log.Debug(u"TO_REMOTE")
pr = file_naming.parse(self.remname)
src = self.dirpath.append(self.partname)
tgt = self.dirpath.append(self.remname)
src_iter = SrcIter(src)
if pr.compressed:
gpg.GzipWriteFile(src_iter, tgt.name, size=sys.maxsize)
elif pr.encrypted:
gpg.GPGWriteFile(src_iter,
tgt.name,
config.gpg_profile,
size=sys.maxsize)
else:
shutil.copyfile(src.name, tgt.name)
config.backend.move(tgt)
def set_manifest(self, remote_filename):
"""
Add local and remote manifest filenames to backup set
"""
assert not self.remote_manifest_name, (self.remote_manifest_name,
remote_filename)
self.remote_manifest_name = remote_filename
if self.action not in ["collection-status"]:
local_filename_list = globals.archive_dir.listdir()
else:
local_filename_list = []
for local_filename in local_filename_list:
pr = file_naming.parse(local_filename)
if (pr and pr.manifest and pr.type == self.type and
pr.time == self.time and
pr.start_time == self.start_time and
pr.end_time == self.end_time):
self.local_manifest_path = \
globals.archive_dir.append(local_filename)
break
def set_manifest(self, remote_filename):
"""
Add local and remote manifest filenames to backup set
"""
assert not self.remote_manifest_name, (self.remote_manifest_name,
remote_filename)
self.remote_manifest_name = remote_filename
if self.action not in ["collection-status"]:
local_filename_list = globals.archive_dir.listdir()
else:
local_filename_list = []
for local_filename in local_filename_list:
pr = file_naming.parse(local_filename)
if (pr and pr.manifest and pr.type == self.type
and pr.time == self.time
and pr.start_time == self.start_time
and pr.end_time == self.end_time):
self.local_manifest_path = \
globals.archive_dir.append(local_filename)
break
def to_remote(self):
"""
We have written the last checkpoint, now encrypt or compress
and send a copy of it to the remote for final storage.
"""
pr = file_naming.parse(self.remname)
src = self.dirpath.append(self.partname)
tgt = self.dirpath.append(self.remname)
src_iter = SrcIter(src)
if pr.compressed:
gpg.GzipWriteFile(src_iter, tgt.name, size=sys.maxint)
elif pr.encrypted:
gpg.GPGWriteFile(src_iter,
tgt.name,
globals.gpg_profile,
size=sys.maxint)
else:
os.system("cp -p \"%s\" \"%s\"" % (src.name, tgt.name))
globals.backend.put(tgt) #@UndefinedVariable
try:
util.ignore_missing(os.unlink, tgt.name)
except Exception, e:
log.Warn(_("Unable to delete %s: %s" % (tgt.name, str(e))))
def by_start_time(a, b):
return int(file_naming.parse(a).start_time) - int(file_naming.parse(b).start_time)
