def patch_diff_tarfile(base_path, diff_tarfile, restrict_index=()):
"""Patch given Path object using delta tarfile (as in tarfile.TarFile)
If restrict_index is set, ignore any deltas in diff_tarfile that
don't start with restrict_index.
"""
if base_path.exists():
path_iter = selection.Select(base_path).set_iter()
else:
path_iter = empty_iter() # probably untarring full backup
diff_path_iter = difftar2path_iter(diff_tarfile)
if restrict_index:
diff_path_iter = filter_path_iter(diff_path_iter, restrict_index)
collated = diffdir.collate2iters(path_iter, diff_path_iter)
ITR = IterTreeReducer(PathPatcher, [base_path])
for basis_path, diff_ropath in collated:
if basis_path:
log.Info(
_("Patching %s") % (util.ufn(basis_path.get_relative_path())),
log.InfoCode.patch_file_patching,
util.escape(basis_path.get_relative_path()))
ITR(basis_path.index, basis_path, diff_ropath)
else:
log.Info(
_("Patching %s") % (util.ufn(diff_ropath.get_relative_path())),
log.InfoCode.patch_file_patching,
util.escape(diff_ropath.get_relative_path()))
ITR(diff_ropath.index, basis_path, diff_ropath)
ITR.Finish()
base_path.setdata()
def delete(self):
"""
Remove all files in set, both local and remote
"""
rfn = self.get_filenames()
rfn.reverse()
try:
self.backend.delete(rfn)
except Exception:
log.Debug(
_("BackupSet.delete: missing %s") % [util.ufn(f) for f in rfn])
pass
for lfn in globals.archive_dir.listdir():
pr = file_naming.parse(lfn)
if (pr and pr.time == self.time
and pr.start_time == self.start_time
and pr.end_time == self.end_time):
try:
globals.archive_dir.append(lfn).delete()
except Exception:
log.Debug(
_("BackupSet.delete: missing %s") %
[util.ufn(f) for f in lfn])
pass
util.release_lockfile()
def process_local_dir(action, local_pathname):
"""Check local directory, set globals.local_path"""
local_path = path.Path(path.Path(local_pathname).get_canonical())
if action == "restore":
if (local_path.exists() and not local_path.isemptydir()) and not globals.force:
log.FatalError(
_("Restore destination directory %s already " "exists.\nWill not overwrite.")
% (util.ufn(local_path.name),),
log.ErrorCode.restore_dir_exists,
)
elif action == "verify":
if not local_path.exists():
log.FatalError(
_("Verify directory %s does not exist") % (util.ufn(local_path.name),),
log.ErrorCode.verify_dir_doesnt_exist,
)
else:
assert action == "full" or action == "inc"
if not local_path.exists():
log.FatalError(
_("Backup source directory %s does not exist.") % (util.ufn(local_path.name),),
log.ErrorCode.backup_dir_doesnt_exist,
)
globals.local_path = local_path
def set_from_stat(self):
"""Set the value of self.type, self.mode from self.stat"""
if not self.stat:
self.type = None
st_mode = self.stat.st_mode
if stat.S_ISREG(st_mode):
self.type = "reg"
elif stat.S_ISDIR(st_mode):
self.type = "dir"
elif stat.S_ISLNK(st_mode):
self.type = "sym"
elif stat.S_ISFIFO(st_mode):
self.type = "fifo"
elif stat.S_ISSOCK(st_mode):
raise PathException(util.ufn(self.get_relative_path()) +
u"is a socket, unsupported by tar")
self.type = "sock"
elif stat.S_ISCHR(st_mode):
self.type = "chr"
elif stat.S_ISBLK(st_mode):
self.type = "blk"
else:
raise PathException("Unknown type")
self.mode = stat.S_IMODE(st_mode)
if self.type in ("chr", "blk"):
try:
self.devnums = (os.major(self.stat.st_rdev),
os.minor(self.stat.st_rdev))
except:
log.Warn(_("Warning: %s invalid devnums (0x%X), treating as (0, 0).")
% (util.ufn(self.get_relative_path()), self.stat.st_rdev))
self.devnums = (0, 0)
def set_from_stat(self):
"""Set the value of self.type, self.mode from self.stat"""
if not self.stat:
self.type = None
st_mode = self.stat.st_mode
if stat.S_ISREG(st_mode):
self.type = "reg"
elif stat.S_ISDIR(st_mode):
self.type = "dir"
elif stat.S_ISLNK(st_mode):
self.type = "sym"
elif stat.S_ISFIFO(st_mode):
self.type = "fifo"
elif stat.S_ISSOCK(st_mode):
raise PathException(util.ufn(self.get_relative_path()) +
u"is a socket, unsupported by tar")
self.type = "sock"
elif stat.S_ISCHR(st_mode):
self.type = "chr"
elif stat.S_ISBLK(st_mode):
self.type = "blk"
else:
raise PathException("Unknown type")
self.mode = stat.S_IMODE(st_mode)
if self.type in ("chr", "blk"):
try:
self.devnums = (os.major(self.stat.st_rdev),
os.minor(self.stat.st_rdev))
except:
log.Warn(_("Warning: %s invalid devnums (0x%X), treating as (0, 0).")
% (util.ufn(self.get_relative_path()), self.stat.st_rdev))
self.devnums = (0, 0)
def delete(self):
"""
Remove all files in set, both local and remote
"""
rfn = self.get_filenames()
rfn.reverse()
try:
self.backend.delete(rfn)
except Exception:
log.Debug(_("BackupSet.delete: missing %s") % [util.ufn(f) for f in rfn])
pass
if self.action not in ["collection-status", "replicate"]:
local_filename_list = globals.archive_dir_path.listdir()
else:
local_filename_list = []
for lfn in local_filename_list:
pr = file_naming.parse(lfn)
if (pr and pr.time == self.time and
pr.start_time == self.start_time and
pr.end_time == self.end_time):
try:
globals.archive_dir_path.append(lfn).delete()
except Exception:
log.Debug(_("BackupSet.delete: missing %s") % [util.ufn(f) for f in lfn])
pass
util.release_lockfile()
def cleanup(self):
"""
Cleanup any files created in the temporary directory (that
have not been forgotten), and clean up the temporary directory
itself.
On failure they are logged, but this method will not raise an
exception.
"""
self.__lock.acquire()
try:
if self.__dir is not None:
for file in self.__pending.keys():
try:
log.Debug(_("Removing still remembered temporary file %s") % util.ufn(file))
util.ignore_missing(os.unlink, file)
except Exception:
log.Info(_("Cleanup of temporary file %s failed") % util.ufn(file))
pass
try:
os.rmdir(self.__dir)
except Exception:
log.Warn(_("Cleanup of temporary directory %s failed - this is probably a bug.") % util.ufn(self.__dir))
pass
self.__pending = None
self.__dir = None
finally:
self.__lock.release()
def patch_diff_tarfile(base_path, diff_tarfile, restrict_index=()):
"""Patch given Path object using delta tarfile (as in tarfile.TarFile)
If restrict_index is set, ignore any deltas in diff_tarfile that
don't start with restrict_index.
"""
if base_path.exists():
path_iter = selection.Select(base_path).set_iter()
else:
path_iter = empty_iter() # probably untarring full backup
diff_path_iter = difftar2path_iter(diff_tarfile)
if restrict_index:
diff_path_iter = filter_path_iter(diff_path_iter, restrict_index)
collated = diffdir.collate2iters(path_iter, diff_path_iter)
ITR = IterTreeReducer(PathPatcher, [base_path])
for basis_path, diff_ropath in collated:
if basis_path:
log.Info(_("Patching %s") % (util.ufn(basis_path.get_relative_path())),
log.InfoCode.patch_file_patching,
util.escape(basis_path.get_relative_path()))
ITR(basis_path.index, basis_path, diff_ropath)
else:
log.Info(_("Patching %s") % (util.ufn(diff_ropath.get_relative_path())),
log.InfoCode.patch_file_patching,
util.escape(diff_ropath.get_relative_path()))
ITR(diff_ropath.index, basis_path, diff_ropath)
ITR.Finish()
base_path.setdata()
def process_local_dir(action, local_pathname):
"""Check local directory, set globals.local_path"""
local_path = path.Path(path.Path(local_pathname).get_canonical())
if action == "restore":
if (local_path.exists()
and not local_path.isemptydir()) and not globals.force:
log.FatalError(
_("Restore destination directory %s already "
"exists.\nWill not overwrite.") %
(util.ufn(local_path.name), ),
log.ErrorCode.restore_dir_exists)
elif action == "verify":
if not local_path.exists():
log.FatalError(
_("Verify directory %s does not exist") %
(util.ufn(local_path.name), ),
log.ErrorCode.verify_dir_doesnt_exist)
else:
assert action == "full" or action == "inc"
if not local_path.exists():
log.FatalError(
_("Backup source directory %s does not exist.") %
(util.ufn(local_path.name), ),
log.ErrorCode.backup_dir_doesnt_exist)
globals.local_path = local_path
def delete(self):
"""
Remove all files in set, both local and remote
"""
rfn = self.get_filenames()
rfn.reverse()
try:
self.backend.delete(rfn)
except Exception:
log.Debug(_("BackupSet.delete: missing %s") % [util.ufn(f) for f in rfn])
pass
if self.action not in ["collection-status"]:
local_filename_list = globals.archive_dir.listdir()
else:
local_filename_list = []
for lfn in local_filename_list:
pr = file_naming.parse(lfn)
if (pr and pr.time == self.time and
pr.start_time == self.start_time and
pr.end_time == self.end_time):
try:
globals.archive_dir.append(lfn).delete()
except Exception:
log.Debug(_("BackupSet.delete: missing %s") % [util.ufn(f) for f in lfn])
pass
util.release_lockfile()
def mkdir(self):
"""Make directory(s) at specified path"""
log.Info(_("Making directory %s") % util.ufn(self.name))
try:
os.makedirs(self.name)
except OSError:
if (not globals.force):
raise PathException("Error creating directory %s" % util.ufn(self.name), 7)
self.setdata()
def mkdir(self):
"""Make directory(s) at specified path"""
log.Info(_("Making directory %s") % util.ufn(self.name))
try:
os.makedirs(self.name)
except OSError:
if (not globals.force):
raise PathException("Error creating directory %s" % util.ufn(self.name), 7)
self.setdata()
def get_remote_manifest(self):
"""
Return manifest by reading remote manifest on backend
"""
assert self.remote_manifest_name
try:
manifest_buffer = self.backend.get_data(self.remote_manifest_name)
except GPGError as message:
log.Error(_("Error processing remote manifest (%s): %s") %
(util.ufn(self.remote_manifest_name), util.uexc(message)))
return None
log.Info(_("Processing remote manifest %s (%s)") % (
util.ufn(self.remote_manifest_name), len(manifest_buffer)))
return manifest.Manifest().from_string(manifest_buffer)
def get_remote_manifest(self):
"""
Return manifest by reading remote manifest on backend
"""
assert self.remote_manifest_name
try:
manifest_buffer = self.backend.get_data(self.remote_manifest_name)
except GPGError as message:
log.Error(_("Error processing remote manifest (%s): %s") %
(util.ufn(self.remote_manifest_name), util.uexc(message)))
return None
log.Info(_("Processing remote manifest %s (%s)") % (
util.ufn(self.remote_manifest_name), len(manifest_buffer)))
return manifest.Manifest().from_string(manifest_buffer)
def add_to_sets(filename):
"""
Try adding filename to existing sets, or make new one
"""
for set in sets:
if set.add_filename(filename):
log.Debug(_("File %s is part of known set") % (util.ufn(filename),))
break
else:
log.Debug(_("File %s is not part of a known set; creating new set") % (util.ufn(filename),))
new_set = BackupSet(self.backend, self.action)
if new_set.add_filename(filename):
sets.append(new_set)
else:
log.Debug(_("Ignoring file (rejected by backup set) '%s'") % util.ufn(filename))
def add_to_sets(filename):
"""
Try adding filename to existing sets, or make new one
"""
for set in sets:
if set.add_filename(filename):
log.Debug(_("File %s is part of known set") % (util.ufn(filename),))
break
else:
log.Debug(_("File %s is not part of a known set; creating new set") % (util.ufn(filename),))
new_set = BackupSet(self.backend)
if new_set.add_filename(filename):
sets.append(new_set)
else:
log.Debug(_("Ignoring file (rejected by backup set) '%s'") % util.ufn(filename))
def mktemp(self):
"""
Return a unique filename suitable for use for a temporary
file. The file is not created.
Subsequent calls to this method are guaranteed to never return
the same filename again. As a result, it is safe to use under
concurrent conditions.
NOTE: mkstemp() is greatly preferred.
"""
filename = None
self.__lock.acquire()
try:
self.__tempcount = self.__tempcount + 1
suffix = "-%d" % (self.__tempcount,)
filename = tempfile.mktemp(suffix, "mktemp-", self.__dir)
log.Debug(_("Registering (mktemp) temporary file %s") % util.ufn(filename))
self.__pending[filename] = None
finally:
self.__lock.release()
return filename
def get_fileobj_write(self, filename, parseresults=None, sizelist=None):
"""
Return fileobj opened for writing, which will cause the file
to be written to the backend on close().
The file will be encoded as specified in parseresults (or as
read from the filename), and stored in a temp file until it
can be copied over and deleted.
If sizelist is not None, it should be set to an empty list.
The number of bytes will be inserted into the list.
"""
if not parseresults:
parseresults = file_naming.parse(filename)
assert parseresults, u"Filename %s not correctly parsed" % util.ufn(
filename)
tdp = dup_temp.new_tempduppath(parseresults)
def close_file_hook():
"""This is called when returned fileobj is closed"""
self.put(tdp, filename)
if sizelist is not None:
tdp.setdata()
sizelist.append(tdp.getsize())
tdp.delete()
fh = dup_temp.FileobjHooked(tdp.filtered_open("wb"))
fh.addhook(close_file_hook)
return fh
def get_fileobj_write(self, filename, parseresults=None, sizelist=None):
"""
Return fileobj opened for writing, which will cause the file
to be written to the backend on close().
The file will be encoded as specified in parseresults (or as
read from the filename), and stored in a temp file until it
can be copied over and deleted.
If sizelist is not None, it should be set to an empty list.
The number of bytes will be inserted into the list.
"""
if not parseresults:
parseresults = file_naming.parse(filename)
assert parseresults, u"Filename %s not correctly parsed" % util.ufn(filename)
tdp = dup_temp.new_tempduppath(parseresults)
def close_file_hook():
"""This is called when returned fileobj is closed"""
self.put(tdp, filename)
if sizelist is not None:
tdp.setdata()
sizelist.append(tdp.getsize())
tdp.delete()
fh = dup_temp.FileobjHooked(tdp.filtered_open("wb"))
fh.addhook(close_file_hook)
return fh
def __init__(self, temproot=None):
"""
Create a new TemporaryDirectory backed by a unique and
securely created file system directory.
tempbase - The temp root directory, or None to use system
default (recommended).
"""
if temproot is None:
if globals.temproot:
temproot = globals.temproot
else:
global _initialSystemTempRoot
temproot = _initialSystemTempRoot
self.__dir = tempfile.mkdtemp("-tempdir", "duplicity-", temproot)
log.Info(_("Using temporary directory %s") % util.ufn(self.__dir))
# number of mktemp()/mkstemp() calls served so far
self.__tempcount = 0
# dict of paths pending deletion; use dict even though we are
# not concearned with association, because it is unclear whether
# sets are O(1), while dictionaries are.
self.__pending = {}
self.__lock = threading.Lock() # protect private resources *AND* mktemp/mkstemp calls
def can_fast_process(self, index, ropath):
"""Can fast process (no recursion) if ropath isn't a directory"""
log.Info(_("Writing %s of type %s") %
(util.ufn(ropath.get_relative_path()), ropath.type),
log.InfoCode.patch_file_writing,
"%s %s" % (util.escape(ropath.get_relative_path()), ropath.type))
return not ropath.isdir()
def diryield(path):
"""Generate relevant files in directory path
Returns (path, num) where num == 0 means path should be
generated normally, num == 1 means the path is a directory
and should be included iff something inside is included.
"""
# todo: get around circular dependency issue by importing here
from duplicity import robust #@Reimport
for filename in robust.listpath(path):
new_path = robust.check_common_error(error_handler,
Path.append,
(path, filename))
# make sure file is read accessible
if (new_path and new_path.type in ["reg", "dir"]
and not os.access(new_path.name, os.R_OK)):
log.Warn(
_("Error accessing possibly locked file %s") %
util.ufn(new_path.name), log.WarningCode.cannot_read,
util.escape(new_path.name))
if diffdir.stats:
diffdir.stats.Errors += 1
new_path = None
elif new_path:
s = self.Select(new_path)
if s == 1:
yield (new_path, 0)
elif s == 2 and new_path.isdir():
yield (new_path, 1)
def diryield(path):
"""Generate relevant files in directory path
Returns (path, num) where num == 0 means path should be
generated normally, num == 1 means the path is a directory
and should be included iff something inside is included.
"""
# todo: get around circular dependency issue by importing here
from duplicity import robust #@Reimport
for filename in robust.listpath(path):
new_path = robust.check_common_error(
error_handler, Path.append, (path, filename))
# make sure file is read accessible
if (new_path and new_path.type in ["reg", "dir"]
and not os.access(new_path.name, os.R_OK)):
log.Warn(_("Error accessing possibly locked file %s") % util.ufn(new_path.name),
log.WarningCode.cannot_read,
util.escape(new_path.name))
if diffdir.stats:
diffdir.stats.Errors +=1
new_path = None
elif new_path:
s = self.Select(new_path)
if s == 1:
yield (new_path, 0)
elif s == 2 and new_path.isdir():
yield (new_path, 1)
def Select(self, path):
"""Run through the selection functions and return dominant val 0/1/2"""
# Only used by diryield and tests. Internal.
log.Debug("Selection: examining path %s" % util.ufn(path.name))
if not self.selection_functions:
log.Debug("Selection: + no selection functions found. Including")
return 1
scan_pending = False
for sf in self.selection_functions:
result = sf(path)
log.Debug("Selection: result: %4s from function: %s" % (str(result), sf.name))
if result is 2:
# Selection function says that the path should be scanned for matching files, but keep going
# through the selection functions looking for a real match (0 or 1).
scan_pending = True
elif result == 0 or result == 1:
# A real match found, no need to try other functions.
break
if scan_pending and result != 1:
# A selection function returned 2 and either no real match was
# found or the highest-priority match was 0
result = 2
if result is None:
result = 1
if result == 0:
log.Debug("Selection: - excluding file")
elif result == 1:
log.Debug("Selection: + including file")
else:
assert result == 2
log.Debug("Selection: ? scanning directory for matches")
return result
def delete(self):
"""Remove this file"""
log.Info(_("Deleting %s") % util.ufn(self.name))
if self.isdir():
util.ignore_missing(os.rmdir, self.name)
else:
util.ignore_missing(os.unlink, self.name)
self.setdata()
def error_handler(exc, path, filename):
fullpath = os.path.join(path.name, filename)
try:
mode = os.stat(fullpath)[stat.ST_MODE]
if stat.S_ISSOCK(mode):
log.Info(_("Skipping socket %s") % util.ufn(fullpath),
log.InfoCode.skipping_socket,
util.escape(fullpath))
else:
log.Warn(_("Error initializing file %s") % util.ufn(fullpath),
log.WarningCode.cannot_iterate,
util.escape(fullpath))
except OSError:
log.Warn(_("Error accessing possibly locked file %s") % util.ufn(fullpath),
log.WarningCode.cannot_stat,
util.escape(fullpath))
return None
def can_fast_process(self, index, ropath):
"""Can fast process (no recursion) if ropath isn't a directory"""
log.Info(
_("Writing %s of type %s") %
(util.ufn(ropath.get_relative_path()), ropath.type),
log.InfoCode.patch_file_writing,
"%s %s" % (util.escape(ropath.get_relative_path()), ropath.type))
return not ropath.isdir()
def delete(self):
"""Remove this file"""
log.Info(_("Deleting %s") % util.ufn(self.name))
if self.isdir():
util.ignore_missing(os.rmdir, self.name)
else:
util.ignore_missing(os.unlink, self.name)
self.setdata()
def get_backup_chains(self, filename_list):
"""
Split given filename_list into chains
Return value will be tuple (list of chains, list of sets, list
of incomplete sets), where the list of sets will comprise sets
not fitting into any chain, and the incomplete sets are sets
missing files.
"""
log.Debug(_("Extracting backup chains from list of files: %s")
% [util.ufn(f) for f in filename_list])
# First put filenames in set form
sets = []
def add_to_sets(filename):
"""
Try adding filename to existing sets, or make new one
"""
for set in sets:
if set.add_filename(filename):
log.Debug(_("File %s is part of known set") % (util.ufn(filename),))
break
else:
log.Debug(_("File %s is not part of a known set; creating new set") % (util.ufn(filename),))
new_set = BackupSet(self.backend)
if new_set.add_filename(filename):
sets.append(new_set)
else:
log.Debug(_("Ignoring file (rejected by backup set) '%s'") % util.ufn(filename))
for f in filename_list:
add_to_sets(f)
sets, incomplete_sets = self.get_sorted_sets(sets)
chains, orphaned_sets = [], []
def add_to_chains(set):
"""
Try adding set to existing chains, or make new one
"""
if set.type == "full":
new_chain = BackupChain(self.backend)
new_chain.set_full(set)
chains.append(new_chain)
log.Debug(_("Found backup chain %s") % (new_chain.short_desc()))
else:
assert set.type == "inc"
for chain in chains:
if chain.add_inc(set):
log.Debug(_("Added set %s to pre-existing chain %s") % (set.get_timestr(),
chain.short_desc()))
break
else:
log.Debug(_("Found orphaned set %s") % (set.get_timestr(),))
orphaned_sets.append(set)
for s in sets:
add_to_chains(s)
return (chains, orphaned_sets, incomplete_sets)
def get_backup_chains(self, filename_list):
"""
Split given filename_list into chains
Return value will be tuple (list of chains, list of sets, list
of incomplete sets), where the list of sets will comprise sets
not fitting into any chain, and the incomplete sets are sets
missing files.
"""
log.Debug(_("Extracting backup chains from list of files: %s")
% [util.ufn(f) for f in filename_list])
# First put filenames in set form
sets = []
def add_to_sets(filename):
"""
Try adding filename to existing sets, or make new one
"""
for set in sets:
if set.add_filename(filename):
log.Debug(_("File %s is part of known set") % (util.ufn(filename),))
break
else:
log.Debug(_("File %s is not part of a known set; creating new set") % (util.ufn(filename),))
new_set = BackupSet(self.backend, self.action)
if new_set.add_filename(filename):
sets.append(new_set)
else:
log.Debug(_("Ignoring file (rejected by backup set) '%s'") % util.ufn(filename))
for f in filename_list:
add_to_sets(f)
sets, incomplete_sets = self.get_sorted_sets(sets)
chains, orphaned_sets = [], []
def add_to_chains(set):
"""
Try adding set to existing chains, or make new one
"""
if set.type == "full":
new_chain = BackupChain(self.backend)
new_chain.set_full(set)
chains.append(new_chain)
log.Debug(_("Found backup chain %s") % (new_chain.short_desc()))
else:
assert set.type == "inc"
for chain in chains:
if chain.add_inc(set):
log.Debug(_("Added set %s to pre-existing chain %s") % (set.get_timestr(),
chain.short_desc()))
break
else:
log.Debug(_("Found orphaned set %s") % (set.get_timestr(),))
orphaned_sets.append(set)
for s in sets:
add_to_chains(s)
return (chains, orphaned_sets, incomplete_sets)
def copy_file(self, op, source, target, raise_errors=False):
log.Info(_("Writing %s") % util.ufn(target.get_parse_name()))
try:
source.copy(target,
Gio.FileCopyFlags.OVERWRITE | Gio.FileCopyFlags.NOFOLLOW_SYMLINKS,
None, self.copy_progress, None)
except Exception, e:
self.handle_error(raise_errors, e, op, source.get_parse_name(),
target.get_parse_name())
def log_prev_error(self, index):
"""Call function if no pending exception"""
if not index:
index_str = "."
else:
index_str = os.path.join(*index)
log.Warn(_("Skipping %s because of previous error") % util.ufn(index_str),
log.WarningCode.process_skipped,
util.escape(index_str))
def deltree(self):
"""Remove self by recursively deleting files under it"""
from duplicity import selection # todo: avoid circ. dep. issue
log.Info(_("Deleting tree %s") % util.ufn(self.name))
itr = IterTreeReducer(PathDeleter, [])
for path in selection.Select(self).set_iter():
itr(path.index, path)
itr.Finish()
self.setdata()
def incr_warnings(exc):
"""Warn if prefix is incorrect"""
prefix_warnings[0] += 1
if prefix_warnings[0] < 6:
log.Warn(_("Warning: file specification '%s' in filelist %s\n"
"doesn't start with correct prefix %s. Ignoring.") %
(exc, filelist_name, util.ufn(self.prefix)))
if prefix_warnings[0] == 5:
log.Warn(_("Future prefix errors will not be logged."))
def error_handler(exc, path, filename):
fullpath = os.path.join(path.name, filename)
try:
mode = os.stat(fullpath)[stat.ST_MODE]
if stat.S_ISSOCK(mode):
log.Info(
_("Skipping socket %s") % util.ufn(fullpath),
log.InfoCode.skipping_socket, util.escape(fullpath))
else:
log.Warn(
_("Error initializing file %s") % util.ufn(fullpath),
log.WarningCode.cannot_iterate, util.escape(fullpath))
except OSError:
log.Warn(
_("Error accessing possibly locked file %s") %
util.ufn(fullpath), log.WarningCode.cannot_stat,
util.escape(fullpath))
return None
def read(self, length=-1):
try:
buf = self.infile.read(length)
except IOError as ex:
buf = ""
log.Warn(_("Error %s getting delta for %s") % (str(ex), util.ufn(self.infile.name)))
if stats:
stats.SourceFileSize += len(buf)
return buf
def deltree(self):
"""Remove self by recursively deleting files under it"""
from duplicity import selection # todo: avoid circ. dep. issue
log.Info(_("Deleting tree %s") % util.ufn(self.name))
itr = IterTreeReducer(PathDeleter, [])
for path in selection.Select(self).set_iter():
itr(path.index, path)
itr.Finish()
self.setdata()
def log_delta_path(delta_path, new_path=None, stats=None):
"""
Look at delta path and log delta. Add stats if new_path is set
"""
if delta_path.difftype == "snapshot":
if new_path and stats:
stats.add_new_file(new_path)
log.Info(_("A %s") %
(util.ufn(delta_path.get_relative_path())),
log.InfoCode.diff_file_new,
util.escape(delta_path.get_relative_path()))
else:
if new_path and stats:
stats.add_changed_file(new_path)
log.Info(_("M %s") %
(util.ufn(delta_path.get_relative_path())),
log.InfoCode.diff_file_changed,
util.escape(delta_path.get_relative_path()))
def incr_warnings(exc):
"""Warn if prefix is incorrect"""
prefix_warnings[0] += 1
if prefix_warnings[0] < 6:
log.Warn(
_("Warning: file specification '%s' in filelist %s\n"
"doesn't start with correct prefix %s. Ignoring.") %
(exc, filelist_name, util.ufn(self.prefix)))
if prefix_warnings[0] == 5:
log.Warn(_("Future prefix errors will not be logged."))
def get(self, remote_filename, local_path):
"""Retrieve remote_filename and place in local_path"""
if hasattr(self.backend, '_get'):
self.backend._get(remote_filename, local_path)
local_path.setdata()
if not local_path.exists():
raise BackendException(_("File %s not found locally after get "
"from backend") % util.ufn(local_path.name))
else:
raise NotImplementedError()
def copy_file(self, op, source, target, raise_errors=False):
log.Info(_("Writing %s") % util.ufn(target.get_parse_name()))
try:
source.copy(
target, Gio.FileCopyFlags.OVERWRITE
| Gio.FileCopyFlags.NOFOLLOW_SYMLINKS, None,
self.copy_progress, None)
except Exception, e:
self.handle_error(raise_errors, e, op, source.get_parse_name(),
target.get_parse_name())
def next(self):
try:
res = Block(self.fp.read(self.get_read_size()))
except Exception:
log.FatalError(_("Failed to read %s: %s") %
(util.ufn(self.src.name), sys.exc_info()),
log.ErrorCode.generic)
if not res.data:
self.fp.close()
raise StopIteration
return res
def next(self):
"""Write next file, return filename"""
if self.finished:
raise StopIteration
filename = "%s.%d" % (self.prefix, self.current_index)
log.Info(_("Starting to write %s") % util.ufn(filename))
outfp = open(filename, "wb")
if not self.write_volume(outfp):
# end of input
self.finished = 1
if self.current_index == 1:
# special case first index
log.Notice(_("One only volume required.\n"
"Renaming %s to %s") % (util.ufn(filename), util.ufn(self.prefix)))
os.rename(filename, self.prefix)
return self.prefix
else:
self.current_index += 1
return filename
def next(self):
try:
res = Block(self.fp.read(self.get_read_size()))
except Exception:
log.FatalError(
_("Failed to read %s: %s") %
(util.ufn(self.src.name), sys.exc_info()),
log.ErrorCode.generic)
if not res.data:
self.fp.close()
raise StopIteration
return res
def on_error(self, exc, *args):
"""This is run on any exception in start/end-process"""
self.caught_exception = 1
if args and args[0] and isinstance(args[0], tuple):
filename = os.path.join(*args[0])
elif self.index:
filename = os.path.join(*self.index) # pylint: disable=not-an-iterable
else:
filename = "."
log.Warn(_("Error '%s' processing %s") % (exc, util.ufn(filename)),
log.WarningCode.cannot_process,
util.escape(filename))
def get_temp_in_same_dir(self):
"""Return temp non existent path in same directory as self"""
global _tmp_path_counter
parent_dir = self.get_parent_dir()
while 1:
temp_path = parent_dir.append("duplicity_temp." +
str(_tmp_path_counter))
if not temp_path.type:
return temp_path
_tmp_path_counter += 1
assert _tmp_path_counter < 10000, \
u"Warning too many temp files created for " + util.ufn(self.name)
def get_temp_in_same_dir(self):
"""Return temp non existent path in same directory as self"""
global _tmp_path_counter
parent_dir = self.get_parent_dir()
while 1:
temp_path = parent_dir.append("duplicity_temp." +
str(_tmp_path_counter))
if not temp_path.type:
return temp_path
_tmp_path_counter += 1
assert _tmp_path_counter < 10000, \
u"Warning too many temp files created for " + util.ufn(self.name)
def delta_iter_error_handler(exc, new_path, sig_path, sig_tar=None):
"""
Called by get_delta_iter, report error in getting delta
"""
if new_path:
index_string = new_path.get_relative_path()
elif sig_path:
index_string = sig_path.get_relative_path()
else:
assert 0, "Both new and sig are None for some reason"
log.Warn(_("Error %s getting delta for %s") % (str(exc), util.ufn(index_string)))
return None
def get_delta_iter(new_iter, sig_iter, sig_fileobj=None):
"""
Generate delta iter from new Path iter and sig Path iter.
For each delta path of regular file type, path.difftype with be
set to "snapshot", "diff". sig_iter will probably iterate ROPaths
instead of Paths.
If sig_fileobj is not None, will also write signatures to sig_fileobj.
"""
collated = collate2iters(new_iter, sig_iter)
if sig_fileobj:
sigTarFile = util.make_tarfile("w", sig_fileobj)
else:
sigTarFile = None
for new_path, sig_path in collated:
log.Debug(
_("Comparing %s and %s") %
(new_path and util.uindex(new_path.index), sig_path
and util.uindex(sig_path.index)))
if not new_path or not new_path.type:
# File doesn't exist (but ignore attempts to delete base dir;
# old versions of duplicity could have written out the sigtar in
# such a way as to fool us; LP: #929067)
if sig_path and sig_path.exists() and sig_path.index != ():
# but signature says it did
log.Info(
_("D %s") % (util.ufn(sig_path.get_relative_path())),
log.InfoCode.diff_file_deleted,
util.escape(sig_path.get_relative_path()))
if sigTarFile:
ti = ROPath(sig_path.index).get_tarinfo()
ti.name = "deleted/" + "/".join(sig_path.index)
sigTarFile.addfile(ti)
stats.add_deleted_file(sig_path)
yield ROPath(sig_path.index)
elif not sig_path or new_path != sig_path:
# Must calculate new signature and create delta
delta_path = robust.check_common_error(
delta_iter_error_handler, get_delta_path,
(new_path, sig_path, sigTarFile))
if delta_path:
# log and collect stats
log_delta_path(delta_path, new_path, stats)
yield delta_path
else:
# if not, an error must have occurred
stats.Errors += 1
else:
stats.add_unchanged_file(new_path)
stats.close()
if sigTarFile:
sigTarFile.close()
def set_archive_dir(dirstring):
"""Check archive dir and set global"""
if not os.path.exists(dirstring):
try:
os.makedirs(dirstring)
except Exception:
pass
archive_dir = path.Path(dirstring)
if not archive_dir.isdir():
log.FatalError(_("Specified archive directory '%s' does not exist, "
"or is not a directory") % (util.ufn(archive_dir.name),),
log.ErrorCode.bad_archive_dir)
globals.archive_dir = archive_dir
def set_archive_dir(dirstring):
"""Check archive dir and set global"""
if not os.path.exists(dirstring):
try:
os.makedirs(dirstring)
except Exception:
pass
archive_dir = path.Path(dirstring)
if not archive_dir.isdir():
log.FatalError(_("Specified archive directory '%s' does not exist, "
"or is not a directory") % (util.ufn(archive_dir.name),),
log.ErrorCode.bad_archive_dir)
globals.archive_dir = archive_dir
def init_from_tarinfo(self, tarinfo):
"""Set data from tarinfo object (part of tarfile module)"""
# Set the typepp
type = tarinfo.type
if type == tarfile.REGTYPE or type == tarfile.AREGTYPE:
self.type = "reg"
elif type == tarfile.LNKTYPE:
raise PathException("Hard links not supported yet")
elif type == tarfile.SYMTYPE:
self.type = "sym"
self.symtext = tarinfo.linkname
elif type == tarfile.CHRTYPE:
self.type = "chr"
self.devnums = (tarinfo.devmajor, tarinfo.devminor)
elif type == tarfile.BLKTYPE:
self.type = "blk"
self.devnums = (tarinfo.devmajor, tarinfo.devminor)
elif type == tarfile.DIRTYPE:
self.type = "dir"
elif type == tarfile.FIFOTYPE:
self.type = "fifo"
else:
raise PathException("Unknown tarinfo type %s" % (type,))
self.mode = tarinfo.mode
self.stat = StatResult()
""" Set user and group id
use numeric id if name lookup fails
OR
--numeric-owner is set
"""
try:
if globals.numeric_owner:
raise KeyError
self.stat.st_uid = cached_ops.getpwnam(tarinfo.uname)[2]
except KeyError:
self.stat.st_uid = tarinfo.uid
try:
if globals.numeric_owner:
raise KeyError
self.stat.st_gid = cached_ops.getgrnam(tarinfo.gname)[2]
except KeyError:
self.stat.st_gid = tarinfo.gid
self.stat.st_mtime = int(tarinfo.mtime)
if self.stat.st_mtime < 0:
log.Warn(_("Warning: %s has negative mtime, treating as 0.")
% (util.ufn(tarinfo.name)))
self.stat.st_mtime = 0
self.stat.st_size = tarinfo.size
def get_delta_iter(new_iter, sig_iter, sig_fileobj=None):
"""
Generate delta iter from new Path iter and sig Path iter.
For each delta path of regular file type, path.difftype with be
set to "snapshot", "diff". sig_iter will probably iterate ROPaths
instead of Paths.
If sig_fileobj is not None, will also write signatures to sig_fileobj.
"""
collated = collate2iters(new_iter, sig_iter)
if sig_fileobj:
sigTarFile = util.make_tarfile("w", sig_fileobj)
else:
sigTarFile = None
for new_path, sig_path in collated:
log.Debug(_("Comparing %s and %s") % (new_path and util.uindex(new_path.index),
sig_path and util.uindex(sig_path.index)))
if not new_path or not new_path.type:
# File doesn't exist (but ignore attempts to delete base dir;
# old versions of duplicity could have written out the sigtar in
# such a way as to fool us; LP: #929067)
if sig_path and sig_path.exists() and sig_path.index != ():
# but signature says it did
log.Info(_("D %s") %
(util.ufn(sig_path.get_relative_path())),
log.InfoCode.diff_file_deleted,
util.escape(sig_path.get_relative_path()))
if sigTarFile:
ti = ROPath(sig_path.index).get_tarinfo()
ti.name = "deleted/" + "/".join(sig_path.index)
sigTarFile.addfile(ti)
stats.add_deleted_file(sig_path)
yield ROPath(sig_path.index)
elif not sig_path or new_path != sig_path:
# Must calculate new signature and create delta
delta_path = robust.check_common_error(delta_iter_error_handler,
get_delta_path,
(new_path, sig_path, sigTarFile))
if delta_path:
# log and collect stats
log_delta_path(delta_path, new_path, stats)
yield delta_path
else:
# if not, an error must have occurred
stats.Errors += 1
else:
stats.add_unchanged_file(new_path)
stats.close()
if sigTarFile:
sigTarFile.close()