Ejemplo n.º 1
0
def recreate_meta(meta_manager):
    """Make regress_time mirror_metadata snapshot by patching

	We write to a tempfile first.  Otherwise, in case of a crash, it
	would seem we would have an intact snapshot and partial diff, not
	the reverse.

	"""
    temprp = [TempFile.new_in_dir(Globals.rbdir)]

    def callback(rp):
        temprp[0] = rp

    writer = metadata.MetadataFile(temprp[0],
                                   'w',
                                   check_path=0,
                                   callback=callback)
    for rorp in meta_manager.get_meta_at_time(regress_time, None):
        writer.write_object(rorp)
    writer.close()

    finalrp = Globals.rbdir.append("mirror_metadata.%s.snapshot.gz" %
                                   Time.timetostring(regress_time))
    assert not finalrp.lstat(), finalrp
    rpath.rename(temprp[0], finalrp)
    if Globals.fsync_directories: Globals.rbdir.fsync()
Ejemplo n.º 2
0
def recreate_meta(meta_manager):
	"""Make regress_time mirror_metadata snapshot by patching

	We write to a tempfile first.  Otherwise, in case of a crash, it
	would seem we would have an intact snapshot and partial diff, not
	the reverse.

	"""
	temprp = [TempFile.new_in_dir(Globals.rbdir)]
	def callback(rp): temprp[0] = rp
	writer = metadata.MetadataFile(temprp[0], 'w', check_path = 0, callback = callback)
	for rorp in meta_manager.get_meta_at_time(regress_time, None):
		writer.write_object(rorp)
	writer.close()

	finalrp = Globals.rbdir.append("mirror_metadata.%s.snapshot.gz" %
								   Time.timetostring(regress_time))
	assert not finalrp.lstat(), finalrp
	rpath.rename(temprp[0], finalrp)
	if Globals.fsync_directories: Globals.rbdir.fsync()
Ejemplo n.º 3
0
def makesnapshot(mirror, incpref, renameok):
	"""Copy mirror to incfile, since new is quite different"""
	compress = iscompressed(mirror)
	if compress and mirror.isreg():
		renameok = False;
		snapshotrp = get_inc(incpref, "snapshot.gz")
	else: snapshotrp = get_inc(incpref, "snapshot")

	if renameok:
		rpath.rename(mirror, snapshotrp)
		snapshotrp.renamed = True;
	elif mirror.isspecial(): # check for errors when creating special increments
		eh = robust.get_error_handler("SpecialFileError")
		if robust.check_common_error(eh, rpath.copy_with_attribs,
									 (mirror, snapshotrp, compress)) == 0:
			snapshotrp.setdata()
			if snapshotrp.lstat(): snapshotrp.delete()
			snapshotrp.touch()
	else: rpath.copy_with_attribs(mirror, snapshotrp, compress)
	return snapshotrp
Ejemplo n.º 4
0
	def restore_orig_regfile(self, rf):
		"""Restore original regular file

		This is the trickiest case for avoiding information loss,
		because we don't want to delete the increment before the
		mirror is fully written.

		"""
		assert rf.metadata_rorp.isreg()
		if rf.mirror_rp.isreg():
			tf = TempFile.new(rf.mirror_rp)
			tf.write_from_fileobj(rf.get_restore_fp())
			tf.fsync_with_dir() # make sure tf fully written before move
			rpath.copy_attribs(rf.metadata_rorp, tf)
			rpath.rename(tf, rf.mirror_rp) # move is atomic
		else:
			if rf.mirror_rp.lstat(): rf.mirror_rp.delete()
			rf.mirror_rp.write_from_fileobj(rf.get_restore_fp())
			rpath.copy_attribs(rf.metadata_rorp, rf.mirror_rp)
		if Globals.fsync_directories:
			rf.mirror_rp.get_parent_rp().fsync() # force move before inc delete
Ejemplo n.º 5
0
    def rename(self, rp_dest):
        """Rename temp file to permanent location, possibly overwriting"""
        if not self.lstat():  # "Moving" empty file, so just delete
            if rp_dest.lstat(): rp_dest.delete()
            remove_listing(self)
            return

        if self.isdir() and not rp_dest.isdir():
            # Cannot move a directory directly over another file
            rp_dest.delete()
        rpath.rename(self, rp_dest)

        # Sometimes this just seems to fail silently, as in one
        # hardlinked twin is moved over the other.  So check to make
        # sure below.
        self.setdata()
        if self.lstat():
            rp_dest.delete()
            rpath.rename(self, rp_dest)
            self.setdata()
            if self.lstat(): raise OSError("Cannot rename tmp file correctly")
        remove_listing(self)
Ejemplo n.º 6
0
    def restore_orig_regfile(self, rf):
        """Restore original regular file

		This is the trickiest case for avoiding information loss,
		because we don't want to delete the increment before the
		mirror is fully written.

		"""
        assert rf.metadata_rorp.isreg()
        if rf.mirror_rp.isreg():
            tf = TempFile.new(rf.mirror_rp)
            tf.write_from_fileobj(rf.get_restore_fp())
            tf.fsync_with_dir()  # make sure tf fully written before move
            rpath.copy_attribs(rf.metadata_rorp, tf)
            rpath.rename(tf, rf.mirror_rp)  # move is atomic
        else:
            if rf.mirror_rp.lstat(): rf.mirror_rp.delete()
            rf.mirror_rp.write_from_fileobj(rf.get_restore_fp())
            rpath.copy_attribs(rf.metadata_rorp, rf.mirror_rp)
        if Globals.fsync_directories:
            rf.mirror_rp.get_parent_rp().fsync(
            )  # force move before inc delete
Ejemplo n.º 7
0
    def rename(self, rp_dest):
        """Rename temp file to permanent location, possibly overwriting"""
        if not self.lstat():  # "Moving" empty file, so just delete
            if rp_dest.lstat():
                rp_dest.delete()
            remove_listing(self)
            return

        if self.isdir() and not rp_dest.isdir():
            # Cannot move a directory directly over another file
            rp_dest.delete()
        rpath.rename(self, rp_dest)

        # Sometimes this just seems to fail silently, as in one
        # hardlinked twin is moved over the other.  So check to make
        # sure below.
        self.setdata()
        if self.lstat():
            rp_dest.delete()
            rpath.rename(self, rp_dest)
            self.setdata()
            if self.lstat():
                raise OSError("Cannot rename tmp file correctly")
        remove_listing(self)
Ejemplo n.º 8
0
def write_via_tempfile(fp, rp):
	"""Write fileobj fp to rp by writing to tempfile and renaming"""
	tf = TempFile.new(rp)
	retval = tf.write_from_fileobj(fp)
	rpath.rename(tf, rp)
	return retval
Ejemplo n.º 9
0
class PatchITRB(rorpiter.ITRBranch):
	"""Patch an rpath with the given diff iters (use with IterTreeReducer)

	The main complication here involves directories.  We have to
	finish processing the directory after what's in the directory, as
	the directory may have inappropriate permissions to alter the
	contents or the dir's mtime could change as we change the
	contents.

	"""
	def __init__(self, basis_root_rp, CCPP):
		"""Set basis_root_rp, the base of the tree to be incremented"""
		self.basis_root_rp = basis_root_rp
		assert basis_root_rp.conn is Globals.local_connection
		self.statfileobj = (statistics.get_active_statfileobj() or
							statistics.StatFileObj())
		self.dir_replacement, self.dir_update = None, None
		self.CCPP = CCPP
		self.error_handler = robust.get_error_handler("UpdateError")

	def can_fast_process(self, index, diff_rorp):
		"""True if diff_rorp and mirror are not directories"""
		mirror_rorp = self.CCPP.get_mirror_rorp(index)
		return not (diff_rorp.isdir() or (mirror_rorp and mirror_rorp.isdir()))

	def fast_process(self, index, diff_rorp):
		"""Patch base_rp with diff_rorp (case where neither is directory)"""
		mirror_rp, discard = longname.get_mirror_inc_rps(
			self.CCPP.get_rorps(index), self.basis_root_rp)
		assert not mirror_rp.isdir(), mirror_rp
		tf = TempFile.new(mirror_rp)
		if self.patch_to_temp(mirror_rp, diff_rorp, tf):
			if tf.lstat():
				if robust.check_common_error(self.error_handler, rpath.rename,
						(tf, mirror_rp)) is None:
					self.CCPP.flag_success(index)
				else:
					tf.delete()
			elif mirror_rp and mirror_rp.lstat():
				mirror_rp.delete()
				self.CCPP.flag_deleted(index)
		else: 
			tf.setdata()
			if tf.lstat(): tf.delete()

	def patch_to_temp(self, basis_rp, diff_rorp, new):
		"""Patch basis_rp, writing output in new, which doesn't exist yet

		Returns true if able to write new as desired, false if
		UpdateError or similar gets in the way.

		"""
		if diff_rorp.isflaglinked():
			self.patch_hardlink_to_temp(diff_rorp, new)
		elif diff_rorp.get_attached_filetype() == 'snapshot':
			result = self.patch_snapshot_to_temp(diff_rorp, new)
			if not result: return 0
			elif result == 2: return 1 # SpecialFile
		elif not self.patch_diff_to_temp(basis_rp, diff_rorp, new):
			return 0
		if new.lstat() and not diff_rorp.isflaglinked():
			rpath.copy_attribs(diff_rorp, new)
		return self.matches_cached_rorp(diff_rorp, new)

	def patch_hardlink_to_temp(self, diff_rorp, new):
		"""Hardlink diff_rorp to temp, update hash if necessary"""
		Hardlink.link_rp(diff_rorp, new, self.basis_root_rp)
		self.CCPP.update_hardlink_hash(diff_rorp)

	def patch_snapshot_to_temp(self, diff_rorp, new):
		"""Write diff_rorp to new, return true if successful

		Returns 1 if normal success, 2 if special file is written,
		whether or not it is successful.  This is because special
		files either fail with a SpecialFileError, or don't need to be
		compared.

		"""
		if diff_rorp.isspecial():
			self.write_special(diff_rorp, new)
			rpath.copy_attribs(diff_rorp, new)
			return 2
		
		report = robust.check_common_error(self.error_handler, rpath.copy,
										   (diff_rorp, new))
		if isinstance(report, hash.Report):
			self.CCPP.update_hash(diff_rorp.index, report.sha1_digest)
			return 1
		return report != 0 # if == 0, error_handler caught something

	def patch_diff_to_temp(self, basis_rp, diff_rorp, new):
		"""Apply diff_rorp to basis_rp, write output in new"""
		assert diff_rorp.get_attached_filetype() == 'diff'
		report = robust.check_common_error(self.error_handler,
			      Rdiff.patch_local, (basis_rp, diff_rorp, new))
		if isinstance(report, hash.Report):
			self.CCPP.update_hash(diff_rorp.index, report.sha1_digest)
			return 1
		return report != 0 # if report == 0, error

	def matches_cached_rorp(self, diff_rorp, new_rp):
		"""Return true if new_rp matches cached src rorp

		This is a final check to make sure the temp file just written
		matches the stats which we got earlier.  If it doesn't it
		could confuse the regress operation.  This is only necessary
		for regular files.

		"""
		if not new_rp.isreg(): return 1
		cached_rorp = self.CCPP.get_source_rorp(diff_rorp.index)
		if cached_rorp and cached_rorp.equal_loose(new_rp): return 1
		log.ErrorLog.write_if_open("UpdateError", diff_rorp, "Updated mirror "
					  "temp file %s does not match source" % (new_rp.path,))
		return 0

	def write_special(self, diff_rorp, new):
		"""Write diff_rorp (which holds special file) to new"""
		eh = robust.get_error_handler("SpecialFileError")
		if robust.check_common_error(eh, rpath.copy, (diff_rorp, new)) == 0:
			new.setdata()
			if new.lstat(): new.delete()
			new.touch()

	def start_process(self, index, diff_rorp):
		"""Start processing directory - record information for later"""
		self.base_rp, discard = longname.get_mirror_inc_rps(
			self.CCPP.get_rorps(index), self.basis_root_rp)
		if diff_rorp.isdir(): self.prepare_dir(diff_rorp, self.base_rp)
		elif self.set_dir_replacement(diff_rorp, self.base_rp):
			if diff_rorp.lstat(): self.CCPP.flag_success(index)
			else: self.CCPP.flag_deleted(index)

	def set_dir_replacement(self, diff_rorp, base_rp):
		"""Set self.dir_replacement, which holds data until done with dir

		This is used when base_rp is a dir, and diff_rorp is not.
		Returns 1 for success or 0 for failure

		"""
		assert diff_rorp.get_attached_filetype() == 'snapshot'
		self.dir_replacement = TempFile.new(base_rp)
		if not self.patch_to_temp(None, diff_rorp, self.dir_replacement):
			if self.dir_replacement.lstat(): self.dir_replacement.delete()
			# Was an error, so now restore original directory
			rpath.copy_with_attribs(self.CCPP.get_mirror_rorp(diff_rorp.index),
									self.dir_replacement)
			return 0
		else: return 1

	def prepare_dir(self, diff_rorp, base_rp):
		"""Prepare base_rp to be a directory"""
		self.dir_update = diff_rorp.getRORPath() # make copy in case changes
		if not base_rp.isdir():
			if base_rp.lstat(): self.base_rp.delete()
			base_rp.setdata()
			base_rp.mkdir()
			self.CCPP.flag_success(diff_rorp.index)
		else: # maybe no change, so query CCPP before tagging success
			if self.CCPP.in_cache(diff_rorp.index):
				self.CCPP.flag_success(diff_rorp.index)

	def end_process(self):
		"""Finish processing directory"""
		if self.dir_update:
			assert self.base_rp.isdir()
			rpath.copy_attribs(self.dir_update, self.base_rp)

			if (Globals.process_uid != 0 and
					self.dir_update.getperms() % 01000 < 0700):
				# Directory was unreadable at start -- keep it readable
				# until the end of the backup process.
				self.base_rp.chmod(0700 | self.dir_update.getperms())
		elif self.dir_replacement:
			self.base_rp.rmdir()
			if self.dir_replacement.lstat():
				rpath.rename(self.dir_replacement, self.base_rp)