def recreate_meta(meta_manager):
"""Make regress_time mirror_metadata snapshot by patching
We write to a tempfile first. Otherwise, in case of a crash, it
would seem we would have an intact snapshot and partial diff, not
the reverse.
"""
temprp = [TempFile.new_in_dir(Globals.rbdir)]
def callback(rp):
temprp[0] = rp
writer = metadata.MetadataFile(temprp[0],
'w',
check_path=0,
callback=callback)
for rorp in meta_manager.get_meta_at_time(regress_time, None):
writer.write_object(rorp)
writer.close()
finalrp = Globals.rbdir.append("mirror_metadata.%s.snapshot.gz" %
Time.timetostring(regress_time))
assert not finalrp.lstat(), finalrp
rpath.rename(temprp[0], finalrp)
if Globals.fsync_directories: Globals.rbdir.fsync()
def recreate_meta(meta_manager):
"""Make regress_time mirror_metadata snapshot by patching
We write to a tempfile first. Otherwise, in case of a crash, it
would seem we would have an intact snapshot and partial diff, not
the reverse.
"""
temprp = [TempFile.new_in_dir(Globals.rbdir)]
def callback(rp): temprp[0] = rp
writer = metadata.MetadataFile(temprp[0], 'w', check_path = 0, callback = callback)
for rorp in meta_manager.get_meta_at_time(regress_time, None):
writer.write_object(rorp)
writer.close()
finalrp = Globals.rbdir.append("mirror_metadata.%s.snapshot.gz" %
Time.timetostring(regress_time))
assert not finalrp.lstat(), finalrp
rpath.rename(temprp[0], finalrp)
if Globals.fsync_directories: Globals.rbdir.fsync()
def makesnapshot(mirror, incpref, renameok):
"""Copy mirror to incfile, since new is quite different"""
compress = iscompressed(mirror)
if compress and mirror.isreg():
renameok = False;
snapshotrp = get_inc(incpref, "snapshot.gz")
else: snapshotrp = get_inc(incpref, "snapshot")
if renameok:
rpath.rename(mirror, snapshotrp)
snapshotrp.renamed = True;
elif mirror.isspecial(): # check for errors when creating special increments
eh = robust.get_error_handler("SpecialFileError")
if robust.check_common_error(eh, rpath.copy_with_attribs,
(mirror, snapshotrp, compress)) == 0:
snapshotrp.setdata()
if snapshotrp.lstat(): snapshotrp.delete()
snapshotrp.touch()
else: rpath.copy_with_attribs(mirror, snapshotrp, compress)
return snapshotrp
def restore_orig_regfile(self, rf): """Restore original regular file This is the trickiest case for avoiding information loss, because we don't want to delete the increment before the mirror is fully written. """ assert rf.metadata_rorp.isreg() if rf.mirror_rp.isreg(): tf = TempFile.new(rf.mirror_rp) tf.write_from_fileobj(rf.get_restore_fp()) tf.fsync_with_dir() # make sure tf fully written before move rpath.copy_attribs(rf.metadata_rorp, tf) rpath.rename(tf, rf.mirror_rp) # move is atomic else: if rf.mirror_rp.lstat(): rf.mirror_rp.delete() rf.mirror_rp.write_from_fileobj(rf.get_restore_fp()) rpath.copy_attribs(rf.metadata_rorp, rf.mirror_rp) if Globals.fsync_directories: rf.mirror_rp.get_parent_rp().fsync() # force move before inc delete
def rename(self, rp_dest):
"""Rename temp file to permanent location, possibly overwriting"""
if not self.lstat(): # "Moving" empty file, so just delete
if rp_dest.lstat(): rp_dest.delete()
remove_listing(self)
return
if self.isdir() and not rp_dest.isdir():
# Cannot move a directory directly over another file
rp_dest.delete()
rpath.rename(self, rp_dest)
# Sometimes this just seems to fail silently, as in one
# hardlinked twin is moved over the other. So check to make
# sure below.
self.setdata()
if self.lstat():
rp_dest.delete()
rpath.rename(self, rp_dest)
self.setdata()
if self.lstat(): raise OSError("Cannot rename tmp file correctly")
remove_listing(self)
def restore_orig_regfile(self, rf):
"""Restore original regular file
This is the trickiest case for avoiding information loss,
because we don't want to delete the increment before the
mirror is fully written.
"""
assert rf.metadata_rorp.isreg()
if rf.mirror_rp.isreg():
tf = TempFile.new(rf.mirror_rp)
tf.write_from_fileobj(rf.get_restore_fp())
tf.fsync_with_dir() # make sure tf fully written before move
rpath.copy_attribs(rf.metadata_rorp, tf)
rpath.rename(tf, rf.mirror_rp) # move is atomic
else:
if rf.mirror_rp.lstat(): rf.mirror_rp.delete()
rf.mirror_rp.write_from_fileobj(rf.get_restore_fp())
rpath.copy_attribs(rf.metadata_rorp, rf.mirror_rp)
if Globals.fsync_directories:
rf.mirror_rp.get_parent_rp().fsync(
) # force move before inc delete
def rename(self, rp_dest):
"""Rename temp file to permanent location, possibly overwriting"""
if not self.lstat(): # "Moving" empty file, so just delete
if rp_dest.lstat():
rp_dest.delete()
remove_listing(self)
return
if self.isdir() and not rp_dest.isdir():
# Cannot move a directory directly over another file
rp_dest.delete()
rpath.rename(self, rp_dest)
# Sometimes this just seems to fail silently, as in one
# hardlinked twin is moved over the other. So check to make
# sure below.
self.setdata()
if self.lstat():
rp_dest.delete()
rpath.rename(self, rp_dest)
self.setdata()
if self.lstat():
raise OSError("Cannot rename tmp file correctly")
remove_listing(self)
def write_via_tempfile(fp, rp): """Write fileobj fp to rp by writing to tempfile and renaming""" tf = TempFile.new(rp) retval = tf.write_from_fileobj(fp) rpath.rename(tf, rp) return retval
class PatchITRB(rorpiter.ITRBranch):
"""Patch an rpath with the given diff iters (use with IterTreeReducer)
The main complication here involves directories. We have to
finish processing the directory after what's in the directory, as
the directory may have inappropriate permissions to alter the
contents or the dir's mtime could change as we change the
contents.
"""
def __init__(self, basis_root_rp, CCPP):
"""Set basis_root_rp, the base of the tree to be incremented"""
self.basis_root_rp = basis_root_rp
assert basis_root_rp.conn is Globals.local_connection
self.statfileobj = (statistics.get_active_statfileobj() or
statistics.StatFileObj())
self.dir_replacement, self.dir_update = None, None
self.CCPP = CCPP
self.error_handler = robust.get_error_handler("UpdateError")
def can_fast_process(self, index, diff_rorp):
"""True if diff_rorp and mirror are not directories"""
mirror_rorp = self.CCPP.get_mirror_rorp(index)
return not (diff_rorp.isdir() or (mirror_rorp and mirror_rorp.isdir()))
def fast_process(self, index, diff_rorp):
"""Patch base_rp with diff_rorp (case where neither is directory)"""
mirror_rp, discard = longname.get_mirror_inc_rps(
self.CCPP.get_rorps(index), self.basis_root_rp)
assert not mirror_rp.isdir(), mirror_rp
tf = TempFile.new(mirror_rp)
if self.patch_to_temp(mirror_rp, diff_rorp, tf):
if tf.lstat():
if robust.check_common_error(self.error_handler, rpath.rename,
(tf, mirror_rp)) is None:
self.CCPP.flag_success(index)
else:
tf.delete()
elif mirror_rp and mirror_rp.lstat():
mirror_rp.delete()
self.CCPP.flag_deleted(index)
else:
tf.setdata()
if tf.lstat(): tf.delete()
def patch_to_temp(self, basis_rp, diff_rorp, new):
"""Patch basis_rp, writing output in new, which doesn't exist yet
Returns true if able to write new as desired, false if
UpdateError or similar gets in the way.
"""
if diff_rorp.isflaglinked():
self.patch_hardlink_to_temp(diff_rorp, new)
elif diff_rorp.get_attached_filetype() == 'snapshot':
result = self.patch_snapshot_to_temp(diff_rorp, new)
if not result: return 0
elif result == 2: return 1 # SpecialFile
elif not self.patch_diff_to_temp(basis_rp, diff_rorp, new):
return 0
if new.lstat() and not diff_rorp.isflaglinked():
rpath.copy_attribs(diff_rorp, new)
return self.matches_cached_rorp(diff_rorp, new)
def patch_hardlink_to_temp(self, diff_rorp, new):
"""Hardlink diff_rorp to temp, update hash if necessary"""
Hardlink.link_rp(diff_rorp, new, self.basis_root_rp)
self.CCPP.update_hardlink_hash(diff_rorp)
def patch_snapshot_to_temp(self, diff_rorp, new):
"""Write diff_rorp to new, return true if successful
Returns 1 if normal success, 2 if special file is written,
whether or not it is successful. This is because special
files either fail with a SpecialFileError, or don't need to be
compared.
"""
if diff_rorp.isspecial():
self.write_special(diff_rorp, new)
rpath.copy_attribs(diff_rorp, new)
return 2
report = robust.check_common_error(self.error_handler, rpath.copy,
(diff_rorp, new))
if isinstance(report, hash.Report):
self.CCPP.update_hash(diff_rorp.index, report.sha1_digest)
return 1
return report != 0 # if == 0, error_handler caught something
def patch_diff_to_temp(self, basis_rp, diff_rorp, new):
"""Apply diff_rorp to basis_rp, write output in new"""
assert diff_rorp.get_attached_filetype() == 'diff'
report = robust.check_common_error(self.error_handler,
Rdiff.patch_local, (basis_rp, diff_rorp, new))
if isinstance(report, hash.Report):
self.CCPP.update_hash(diff_rorp.index, report.sha1_digest)
return 1
return report != 0 # if report == 0, error
def matches_cached_rorp(self, diff_rorp, new_rp):
"""Return true if new_rp matches cached src rorp
This is a final check to make sure the temp file just written
matches the stats which we got earlier. If it doesn't it
could confuse the regress operation. This is only necessary
for regular files.
"""
if not new_rp.isreg(): return 1
cached_rorp = self.CCPP.get_source_rorp(diff_rorp.index)
if cached_rorp and cached_rorp.equal_loose(new_rp): return 1
log.ErrorLog.write_if_open("UpdateError", diff_rorp, "Updated mirror "
"temp file %s does not match source" % (new_rp.path,))
return 0
def write_special(self, diff_rorp, new):
"""Write diff_rorp (which holds special file) to new"""
eh = robust.get_error_handler("SpecialFileError")
if robust.check_common_error(eh, rpath.copy, (diff_rorp, new)) == 0:
new.setdata()
if new.lstat(): new.delete()
new.touch()
def start_process(self, index, diff_rorp):
"""Start processing directory - record information for later"""
self.base_rp, discard = longname.get_mirror_inc_rps(
self.CCPP.get_rorps(index), self.basis_root_rp)
if diff_rorp.isdir(): self.prepare_dir(diff_rorp, self.base_rp)
elif self.set_dir_replacement(diff_rorp, self.base_rp):
if diff_rorp.lstat(): self.CCPP.flag_success(index)
else: self.CCPP.flag_deleted(index)
def set_dir_replacement(self, diff_rorp, base_rp):
"""Set self.dir_replacement, which holds data until done with dir
This is used when base_rp is a dir, and diff_rorp is not.
Returns 1 for success or 0 for failure
"""
assert diff_rorp.get_attached_filetype() == 'snapshot'
self.dir_replacement = TempFile.new(base_rp)
if not self.patch_to_temp(None, diff_rorp, self.dir_replacement):
if self.dir_replacement.lstat(): self.dir_replacement.delete()
# Was an error, so now restore original directory
rpath.copy_with_attribs(self.CCPP.get_mirror_rorp(diff_rorp.index),
self.dir_replacement)
return 0
else: return 1
def prepare_dir(self, diff_rorp, base_rp):
"""Prepare base_rp to be a directory"""
self.dir_update = diff_rorp.getRORPath() # make copy in case changes
if not base_rp.isdir():
if base_rp.lstat(): self.base_rp.delete()
base_rp.setdata()
base_rp.mkdir()
self.CCPP.flag_success(diff_rorp.index)
else: # maybe no change, so query CCPP before tagging success
if self.CCPP.in_cache(diff_rorp.index):
self.CCPP.flag_success(diff_rorp.index)
def end_process(self):
"""Finish processing directory"""
if self.dir_update:
assert self.base_rp.isdir()
rpath.copy_attribs(self.dir_update, self.base_rp)
if (Globals.process_uid != 0 and
self.dir_update.getperms() % 01000 < 0700):
# Directory was unreadable at start -- keep it readable
# until the end of the backup process.
self.base_rp.chmod(0700 | self.dir_update.getperms())
elif self.dir_replacement:
self.base_rp.rmdir()
if self.dir_replacement.lstat():
rpath.rename(self.dir_replacement, self.base_rp)