def restore(self, ctx): """ Recreate the data from the information stored in the backup """ logging.info("Restoring " + self.path()) # # Check if the file has already been processed # during this pass # if self.restore_hlink(ctx): return # # No, this file is new. Create it. # packer = PackerStream.PackerIStream(self.backup, self.digest, self.level) file = open(self.path(), "wb") for data in FileIO.read_blocks(packer, Digest.dataDigestSize()): #print "File", self.path(), "reading digest", # base64.b64encode(digest) file.write(data) file.close() self.restore_stats()
def scan(self, ctx, prev_num): self.compute_stats() # # Check if we have encountered this file during this scan already # ctx.num_visited_files_reporter.increment(1) ctx.current_scanned_file_reporter.set(self.path()) if self.scan_hlink(ctx): logging.info("File %s: HLINK" % self.path()) return # # Check if the file is the same as in one of the upper levels # if self.scan_prev(ctx, prev_num): logging.debug("File %s: PREV" % self.path()) ctx.num_prev_files_reporter.increment(1) return # --- File not yet in database, process it file_size = 0 packer = PackerStream.PackerOStream(self.backup, Container.CODE_DATA) handle = open(self.path(), "rb") for data in FileIO.read_blocks(handle, self.backup.get_block_size()): packer.write(data) file_size += len(data) ctx.num_total_blocks_reporter.increment(1) ctx.size_total_blocks_reporter.increment(len(data)) ctx.update_scan_status() handle.close() self.digest = packer.get_digest() self.level = packer.get_level() self.update_hlink(ctx) logging.info("Scanned file %s size:%d new_blocks:%d new_blocks_size:%d" % (self.path(), file_size, packer.get_num_new_blocks(), packer.get_size_new_blocks())) ctx.num_scanned_files_reporter.increment(1) if packer.get_num_new_blocks() != 0: ctx.num_new_blocks_reporter.increment(packer.get_num_new_blocks()) ctx.size_new_blocks_reporter.increment(packer.get_size_new_blocks()) ctx.num_changed_files_reporter.increment(1) ctx.changed_files_reporter.append(self.path()) if file_size > 256 * 1024: logging.debug("File %s is big enough to register in cndb" % self.path()) cndb = self.backup.get_completed_nodes_db() assert self.stats is not None path_digest = Digest.dataDigest(self.path().encode('utf8')) encoded = (self.digest + IntegerEncodings.binary_encode_int_varlen(self.level) + IntegerEncodings.binary_encode_int_varlen(self.get_type()) + serialize_stats(self.get_stats())) if not cndb.has_key(path_digest) or cndb[path_digest] != encoded: cndb[path_digest] = encoded
def retrieve(self, stream): """ Recreate the data from the information stored in the backup into the given stream """ logging.info("Retrieving file " + self.path()) packer = PackerStream.PackerIStream(self.backup, self.digest, self.level) for data in FileIO.read_blocks(packer, Digest.dataDigestSize()): stream.write(data)
def test(self, ctx): """ Test that loading the data from the storages is successful """ logging.info("Testing " + self.path()) packer = PackerStream.PackerIStream(self.backup, self.digest, self.level) for data in FileIO.read_blocks(packer, Digest.dataDigestSize()): # Do nothing with the data, just make sure it got loaded pass