def _cost(self, sink, size, prevNode, height): cost = 0 prevSize = self._totalSize(prevNode) # Transfer if sink != self.dest: cost += size if prevNode is not None and prevNode.intermediate and prevNode.sink != self.dest: cost += prevSize # Storage if sink != self.dest or self.delete: cost += size / 16 # Corruption risk cost += (prevSize + size) * (2**(height - 6)) logger.debug( "_cost=%s (%s %s %s %d)", humanize(cost), sink, humanize(size), humanize(prevSize), height, ) return cost
def _display(self, sent, now, chunk, mbps): """ Display intermediate progress. """ if self.parent is not None: self.parent._display(self.parent.offset + sent, now, chunk, mbps) return elapsed = now - self.startTime if sent > 0 and self.total is not None and sent <= self.total: eta = (self.total - sent) * elapsed.total_seconds() / sent eta = datetime.timedelta(seconds=eta) else: eta = None self.output.write( "\r %s: Sent %s%s%s (%s%s) ETA: %s %20s\r" % ( elapsed, util.humanize(sent), "" if self.total is None else " of %s" % (util.humanize(self.total),), "" if self.total is None else " (%d%%)" % (int(100 * sent / self.total),), chunk or "", "" if mbps is None else " %.3g Mbps" % (mbps,), eta, " ", ) ) self.output.flush()
def _unsafeGetUsage(self): for (header, buf) in self._walkTree(BTRFS_QUOTA_TREE_OBJECTID): # logger.debug("%s %s", objectTypeNames[header.type], header) if header.type == objectTypeKeys['BTRFS_QGROUP_INFO_KEY']: data = btrfs_qgroup_info_item.read(buf) try: vol = self.volumes[header.offset] vol.totalSize = data.referenced vol.exclusiveSize = data.exclusive if (data.referenced < 0 or data.exclusive < 0 or data.referenced < data.exclusive): raise _BtrfsError( "Btrfs returned corrupt size of %s (%s exclusive) for %s" % ( humanize(vol.totalSize or -1), humanize(vol.exclusiveSize or -1), vol.fullPath, )) except KeyError: pass elif header.type == objectTypeKeys['BTRFS_QGROUP_LIMIT_KEY']: data = btrfs_qgroup_limit_item.read(buf) elif header.type == objectTypeKeys['BTRFS_QGROUP_RELATION_KEY']: data = None else: data = None
def __str__(self): """ String representation. """ # logger.debug("%d %d %d", self.gen, self.info.generation, self.info.inode.generation) # logger.debug("%o %o", self.info.flags, self.info.inode.flags) return """%4d '%s' (level:%d gen:%d total:%s exclusive:%s%s) %s (parent:%s/%d received:%s/%d) %s%s""" % ( self.id or -1, # ", ".join([dirPath + name for (dirPath, name) in self.links.values()]), self.fullPath, self.level or -1, self.current_gen or -1, # self.size, humanize(self.totalSize or -1), humanize(self.exclusiveSize or -1), " ro" if self.readOnly else "", self.uuid, self.parent_uuid, self.original_gen, self.received_uuid, self.sent_gen, "\n\t".join(self.linuxPaths), # "\n\t" + pretty(self.__dict__), "", )
def _display(self, sent, now, chunk, mbps): """ Display intermediate progress. """ if self.parent is not None: self.parent._display(self.parent.offset + sent, now, chunk, mbps) return elapsed = now - self.startTime if all([ sent > 0, self.total is not None, self.total != 0, sent <= self.total, ]): eta = (self.total - sent) * elapsed.total_seconds() / sent eta = datetime.timedelta(seconds=eta) else: eta = None self.output.write("\r %s: Sent %s%s%s ETA: %s (%s) %s%20s\r" % ( elapsed, util.humanize(sent), "" if (self.total is None or self.total == 0) else " of %s" % (util.humanize(self.total), ), "" if (self.total is None or self.total == 0) else " (%d%%)" % (int(100 * sent / self.total), ), eta, "" if not mbps else "%.3g Mbps " % (mbps, ), chunk or "", " ", )) self.output.flush()
def _unsafeGetUsage(self): for (header, buf) in self._walkTree(BTRFS_QUOTA_TREE_OBJECTID): # logger.debug("%s %s", objectTypeNames[header.type], header) if header.type == objectTypeKeys['BTRFS_QGROUP_INFO_KEY']: data = btrfs_qgroup_info_item.read(buf) try: vol = self.volumes[header.offset] vol.totalSize = data.referenced vol.exclusiveSize = data.exclusive if ( data.referenced < 0 or data.exclusive < 0 or data.referenced < data.exclusive ): raise _BtrfsError( "Btrfs returned corrupt size of %s (%s exclusive) for %s" % ( humanize(vol.totalSize or -1), humanize(vol.exclusiveSize or -1), vol.fullPath, ) ) except KeyError: pass elif header.type == objectTypeKeys['BTRFS_QGROUP_LIMIT_KEY']: data = btrfs_qgroup_limit_item.read(buf) elif header.type == objectTypeKeys['BTRFS_QGROUP_RELATION_KEY']: data = None else: data = None
def skipChunk(self, chunkSize, checkSum, data=None): part = self.parts[self.chunkCount] if part is None: return False (size, tag) = part tag = tag.strip('"') if size != chunkSize: logger.warning("Unexpected chunk size %d instead of %d", chunkSize, size) # return False if tag != checkSum: logger.warning("Bad check sum %d instead of %d", checkSum, tag) return False self.chunkCount += 1 logger.info( "Skipping already uploaded %s chunk #%d", humanize(chunkSize), self.chunkCount, ) return True
def __init__(self, app_path): self.namespace = app_path self.name = app_path self.verbose_name = util.humanize(self.name) self._module = util.import_from_string(app_path) self._urls_module = util.import_from_string(app_path + '.urls', True) self._models_module = util.import_from_string(app_path + '.models', True) self._views_module = util.import_from_string(app_path + '.views', True) self.models = RozModel.get_models(self, self._models_module) self.routes = RozEngine.Engine.app_routes if hasattr(self._module, 'RozMeta'): meta = getattr(self._module, 'RozMeta') RozMetaDefault.set_defaults(meta) else: meta = RozMetaDefault self._module.RozMeta = meta for key, value in vars(meta).iteritems(): util.safe_setattr(self, key, value) self.register_views()
def deleteUnused(self): """ Delete any old snapshots in path, if not kept. """ (count, size) = (0, 0) for (diff, path) in self.extraKeys.items(): if path.startswith("/"): continue keyName = self._keyName(diff.toUUID, diff.fromUUID, path) count += 1 size += diff.size if self._skipDryRun(logger, 'INFO')("Trash: %s", diff): continue try: self.bucket.copy_key(theTrashPrefix + keyName, self.bucket.name, keyName) self.bucket.delete_key(keyName) except boto.exception.S3ResponseError as error: logger.error("%s: %s", error.code, error.message) try: keyName = os.path.dirname(keyName) + Store.theInfoExtension self.bucket.copy_key(theTrashPrefix + keyName, self.bucket.name, keyName) self.bucket.delete_key(keyName) except boto.exception.S3ResponseError as error: logger.debug("%s: %s", error.code, error.message) logger.info("Trashed %d diffs (%s)", count, humanize(size))
def __str__(self): """ human-readable string. """ return u"%s from %s (%s%s)" % ( self.toVol.display(self.sink), self.fromVol.display(self.sink) if self.fromVol else "None", "~" if self.sizeIsEstimated else "", humanize(self.size), # self.sink, )
def analyze(self, chunkSize, *sinks): """ Figure out the best diffs to use to reach all our required volumes. """ measureSize = False if self.measureSize: for sink in sinks: if sink.isRemote: measureSize = True # Use destination (already uploaded) edges first sinks = list(sinks) sinks.reverse() self.dest = sinks[0] def currentSize(): return sum([ n.diffSize for n in self.nodes.values() if n.diff is not None and n.diff.sink != self.dest ]) while True: self._analyzeDontMeasure(chunkSize, measureSize, *sinks) if not measureSize: return estimatedSize = currentSize() # logger.info("Measuring any estimated diffs") for node in self.nodes.values(): edge = node.diff if edge is not None and edge.sink != self.dest and edge.sizeIsEstimated: edge.sink.measureSize(edge, chunkSize) actualSize = currentSize() logger.info( "measured size (%s), estimated size (%s)", humanize(actualSize), humanize(estimatedSize), ) if actualSize <= 1.2 * estimatedSize: return
def display(self, sink=None, detail='phrase'): """ Friendly string for volume, using sink paths. """ if not isinstance(detail, int): detail = detailNum[detail] if detail >= detailNum['line'] and self.size is not None: size = " (%s%s)" % (humanize( self.size), "" if self.exclusiveSize is None else (" %s exclusive" % (humanize(self.exclusiveSize)))) else: size = "" vol = "%s %s" % ( _printUUID(self._uuid, detail - 1), sink.getSendPath(self) if sink else "", ) return vol + size
def display(self, sink=None, detail='phrase'): """ Friendly string for volume, using sink paths. """ if not isinstance(detail, int): detail = detailNum[detail] if detail >= detailNum['line'] and self.size is not None: size = " (%s%s)" % ( humanize(self.size), "" if self.exclusiveSize is None else ( " %s exclusive" % (humanize(self.exclusiveSize)) ) ) else: size = "" vol = "%s %s" % ( _printUUID(self._uuid, detail - 1), sink.getSendPath(self) if sink else "", ) return vol + size
def __init__(self, app, definition, name): self._app = app self.definition = definition self.name = name self.verbose_name = util.humanize(self.name) self.routes = RozEngine.Engine.model_routes if hasattr(self.definition, 'RozMeta'): meta = getattr(self.definition, 'RozMeta') RozMetaDefault.set_defaults(meta) else: meta = RozMetaDefault self.definition.RozMeta = meta for key, value in vars(meta).iteritems(): util.safe_setattr(self, key, value) for field_name, field in RozModel.get_model_field_types(self.definition).iteritems(): if not getattr(field, 'verbose_name'): setattr(field, 'verbose_name', util.humanize(field_name))
def _cost(self, sink, size, prevNode, height): cost = 0 prevSize = self._totalSize(prevNode) # Transfer if sink != self.dest: cost += size if prevNode is not None and prevNode.intermediate and prevNode.sink != self.dest: cost += prevSize # Storage if sink != self.dest or self.delete: cost += size / 16 # Corruption risk cost += (prevSize + size) * (2 ** (height - 6)) logger.debug( "_cost=%s (%s %s %s %d)", humanize(cost), sink, humanize(size), humanize(prevSize), height, ) return cost
def listContents(self): """ Return list of volumes or diffs in this Store's selected directory. """ items = list(self.extraKeys.items()) items.sort(key=lambda t: t[1]) (count, size) = (0, 0) for (diff, path) in items: if path.startswith("/"): continue yield str(diff) count += 1 size += diff.size yield "TOTAL: %d diffs %s" % (count, humanize(size))
def _analyzeDontMeasure(self, chunkSize, willMeasureLater, *sinks): """ Figure out the best diffs to use to reach all our required volumes. """ nodes = [None] height = 1 def sortKey(node): if node is None: return None return (node.intermediate, self._totalSize(node)) while len(nodes) > 0: logger.debug("Analyzing %d nodes for height %d...", len(nodes), height) nodes.sort(key=sortKey) for fromNode in nodes: if self._height(fromNode) >= height: continue if fromNode is not None and fromNode.diffSize is None: continue fromVol = fromNode.volume if fromNode else None logger.debug("Following edges from %s", fromVol) for sink in sinks: # logger.debug( # "Listing edges in %s", # sink # ) for edge in sink.getEdges(fromVol): toVol = edge.toVol # logger.debug("Edge: %s", edge) # Skip any edges already in the destination if sink != self.dest and self.dest.hasEdge(edge): continue if toVol in self.nodes: toNode = self.nodes[toVol] # Don't transfer any edges we won't need in the destination # elif sink != self.dest: # logger.debug("Won't transfer unnecessary %s", edge) # continue else: toNode = _Node(toVol, True) self.nodes[toVol] = toNode logger.debug("Considering %s", edge) edgeSize = edge.size if edge.sizeIsEstimated: if willMeasureLater: # Slight preference for accurate sizes edgeSize *= 1.2 else: # Large preference for accurate sizes edgeSize *= 2 newCost = self._cost(sink, edgeSize, fromNode, height) if toNode.diff is None: oldCost = None else: oldCost = self._cost( toNode.sink, toNode.diffSize, self._getNode(toNode.previous), self._height(toNode) ) # Don't use a more-expensive path if oldCost is not None and oldCost <= newCost: continue # Don't create circular paths if self._wouldLoop(fromVol, toVol): # logger.debug("Ignoring looping edge: %s", toVol.display(sink)) continue # if measureSize and sink != self.dest and edge.sizeIsEstimated: # sink.measureSize(edge, chunkSize) # newCost = self._cost(sink, edge.size, fromSize, height) # if oldCost is not None and oldCost <= newCost: # continue logger.debug( "Replacing edge (%s -> %s cost)\n%s", humanize(oldCost), humanize(newCost), toNode.display(sink) ) # logger.debug("Cost elements: %s", dict( # sink=str(sink), # edgeSize=humanize(edgeSize), # fromSize=humanize(fromSize), # height=height, # )) toNode.diff = edge nodes = [node for node in self.nodes.values() if self._height(node) == height] height += 1 self._prune() for node in self.nodes.values(): node.height = self._height(node) if node.diff is None: logger.error( "No source diffs for %s", node.volume.display(sinks[-1], detail="line"), )
def main(): """ Main program. """ try: args = command.parse_args() _setupLogging(args.quiet, args.logfile, args.server) logger.debug("Version: %s, Arguments: %s", theVersion, vars(args)) if args.server: server = SSHStore.StoreProxyServer(args.dest, args.mode) return(server.run()) source = parseSink(args.source, False, args.delete, args.dry_run) dest = parseSink(args.dest, source is not None, args.delete, args.dry_run) if source is None: source = dest dest = None if not sys.stderr.isatty(): source.showProgress = dest.showProgress = False elif dest is None or (source.isRemote and not dest.isRemote): source.showProgress = True else: dest.showProgress = True with source: try: next(source.listVolumes()) except StopIteration: logger.warn("No snapshots in source.") path = args.source or args.dest if path.endswith("/"): logger.error( "'%s' does not contain any snapshots. Did you mean to type '%s'?", path, path[0:-1] ) else: logger.error( "'%s' is not a snapshot. Did you mean to type '%s/'?", path, path ) return 1 if dest is None: for item in source.listContents(): print item if args.delete: source.deletePartials() return 0 with dest: best = BestDiffs.BestDiffs(source.listVolumes(), args.delete, not args.estimate) best.analyze(args.part_size << 20, source, dest) summary = best.summary() logger.info("Optimal synchronization:") for sink, values in summary.items(): logger.info("%s from %d diffs in %s", humanize(values.size), values.count, sink or "TOTAL", ) for diff in best.iterDiffs(): if diff is None: raise Exception("Missing diff. Can't fully replicate.") else: diff.sendTo(dest, chunkSize=args.part_size << 20) if args.delete: dest.deleteUnused() logger.debug("Successful exit") return 0 except Exception as error: if ( isinstance(error, IOError) and error.errno == errno.EPERM and os.getuid() != 0 ): logger.error("You must be root to access a btrfs filesystem. Use 'sudo'") else: if not theDebug: logger.debug("Trace information for debugging", exc_info=True) logger.error("ERROR: %s.", error, exc_info=theDebug) return 1
def main(): """ Main program. """ try: args = command.parse_args() # Use btrfs quota information useQuota = args.estimate < 2 _setupLogging(args.quiet, args.logfile, args.server) logger.debug("Version: %s, Arguments: %s", theVersion, vars(args)) if args.server: server = SSHStore.StoreProxyServer(args.dest, args.mode) return (server.run()) logger.info( "Snapshot graph is rebuilt after every transfer completion.") logger.info( "Look at option \033[1m-e\033[0m if you want to speed up the process." ) round = 0 while True: round += 1 logger.info("Iteration number %s", round) source = parseSink(args.source, False, args.delete, args.dry_run) dest = parseSink(args.dest, source is not None, args.delete, args.dry_run) if source is None: source = dest dest = None if not sys.stderr.isatty(): source.showProgress = dest.showProgress = False elif dest is None or (source.isRemote and not dest.isRemote): source.showProgress = True else: dest.showProgress = True with source: if useQuota: source.rescanSizes() try: next(source.listVolumes()) except StopIteration: logger.warn("No snapshots in source.") path = args.source or args.dest if path.endswith("/"): logger.error( "'%s' does not contain any snapshots. Did you mean to type '%s'?", path, path[0:-1]) else: logger.error( "'%s' is not a snapshot. Did you mean to type '%s/'?", path, path) return 1 if dest is None: for item in source.listContents(): print(item) if args.delete: source.deletePartials() return 0 with dest: volumes = list(source.listVolumes()) if args.exclude: def is_excluded(vol): for path in source.getPaths(vol): for pattern in args.exclude: if re.match(pattern, path): return True return False volumes = [ vol for vol in volumes if not is_excluded(vol) ] if useQuota: best = BestDiffs.BestDiffs(volumes, args.delete, not args.estimate) best.analyze(args.part_size << 20, source, dest) else: destVolumes = list(dest.listVolumes()) commonVolumes = list(set(volumes) & set(destVolumes)) missingVolumes = list(set(volumes) - set(destVolumes)) missingVolumeUUIDs = { vol.uuid for vol in missingVolumes } if not missingVolumes: break latestCommonVolume = None commonVolumes.sort(key=lambda v: v.otime) if commonVolumes: latestCommonVolume = commonVolumes[-1] diffs = source.getEdges(latestCommonVolume) missingDiffs = [ diff for diff in diffs if diff.toUUID in missingVolumeUUIDs ] missingDiffs.sort(key=lambda d: d.toVol.otime) oldestMissingDiff = missingDiffs[0] oldestMissingDiff.sendTo( dest, chunkSize=args.part_size << 20) continue summary = best.summary() logger.info("Optimal synchronization:") for sink, values in summary.items(): logger.info( "%s from %d diffs in %s", humanize(values.size), values.count, sink or "TOTAL", ) try: # take only first diff diff = next(best.iterDiffs()) if diff is None: raise Exception( "Missing diff. Can't fully replicate.") else: diff.sendTo(dest, chunkSize=args.part_size << 20) except StopIteration: logger.info("No snapshots left to transfer.") if args.delete: dest.deleteUnused() break logger.debug("Successful exit") return 0 except Exception as error: if (isinstance(error, IOError) and error.errno == errno.EPERM and os.getuid() != 0): logger.error( "You must be root to access a btrfs filesystem. Use 'sudo'") else: if not theDebug: logger.debug("Trace information for debugging", exc_info=True) logger.error("ERROR: %s.", error, exc_info=theDebug) return 1