def _cost(self, sink, size, prevNode, height):
cost = 0
prevSize = self._totalSize(prevNode)
# Transfer
if sink != self.dest:
cost += size
if prevNode is not None and prevNode.intermediate and prevNode.sink != self.dest:
cost += prevSize
# Storage
if sink != self.dest or self.delete:
cost += size / 16
# Corruption risk
cost += (prevSize + size) * (2**(height - 6))
logger.debug(
"_cost=%s (%s %s %s %d)",
humanize(cost),
sink,
humanize(size),
humanize(prevSize),
height,
)
return cost
def _display(self, sent, now, chunk, mbps):
""" Display intermediate progress. """
if self.parent is not None:
self.parent._display(self.parent.offset + sent, now, chunk, mbps)
return
elapsed = now - self.startTime
if sent > 0 and self.total is not None and sent <= self.total:
eta = (self.total - sent) * elapsed.total_seconds() / sent
eta = datetime.timedelta(seconds=eta)
else:
eta = None
self.output.write(
"\r %s: Sent %s%s%s (%s%s) ETA: %s %20s\r" % (
elapsed,
util.humanize(sent),
"" if self.total is None else " of %s" % (util.humanize(self.total),),
"" if self.total is None else " (%d%%)" % (int(100 * sent / self.total),),
chunk or "",
"" if mbps is None else " %.3g Mbps" % (mbps,),
eta,
" ",
)
)
self.output.flush()
def _unsafeGetUsage(self):
for (header, buf) in self._walkTree(BTRFS_QUOTA_TREE_OBJECTID):
# logger.debug("%s %s", objectTypeNames[header.type], header)
if header.type == objectTypeKeys['BTRFS_QGROUP_INFO_KEY']:
data = btrfs_qgroup_info_item.read(buf)
try:
vol = self.volumes[header.offset]
vol.totalSize = data.referenced
vol.exclusiveSize = data.exclusive
if (data.referenced < 0 or data.exclusive < 0
or data.referenced < data.exclusive):
raise _BtrfsError(
"Btrfs returned corrupt size of %s (%s exclusive) for %s"
% (
humanize(vol.totalSize or -1),
humanize(vol.exclusiveSize or -1),
vol.fullPath,
))
except KeyError:
pass
elif header.type == objectTypeKeys['BTRFS_QGROUP_LIMIT_KEY']:
data = btrfs_qgroup_limit_item.read(buf)
elif header.type == objectTypeKeys['BTRFS_QGROUP_RELATION_KEY']:
data = None
else:
data = None
def __str__(self):
""" String representation. """
# logger.debug("%d %d %d", self.gen, self.info.generation, self.info.inode.generation)
# logger.debug("%o %o", self.info.flags, self.info.inode.flags)
return """%4d '%s' (level:%d gen:%d total:%s exclusive:%s%s) %s
(parent:%s/%d received:%s/%d)
%s%s""" % (
self.id or -1,
# ", ".join([dirPath + name for (dirPath, name) in self.links.values()]),
self.fullPath,
self.level or -1,
self.current_gen or -1,
# self.size,
humanize(self.totalSize or -1),
humanize(self.exclusiveSize or -1),
" ro" if self.readOnly else "",
self.uuid,
self.parent_uuid,
self.original_gen,
self.received_uuid,
self.sent_gen,
"\n\t".join(self.linuxPaths),
# "\n\t" + pretty(self.__dict__),
"",
)
def _display(self, sent, now, chunk, mbps):
""" Display intermediate progress. """
if self.parent is not None:
self.parent._display(self.parent.offset + sent, now, chunk, mbps)
return
elapsed = now - self.startTime
if all([
sent > 0,
self.total is not None,
self.total != 0,
sent <= self.total,
]):
eta = (self.total - sent) * elapsed.total_seconds() / sent
eta = datetime.timedelta(seconds=eta)
else:
eta = None
self.output.write("\r %s: Sent %s%s%s ETA: %s (%s) %s%20s\r" % (
elapsed,
util.humanize(sent),
"" if (self.total is None or self.total == 0) else " of %s" %
(util.humanize(self.total), ),
"" if (self.total is None or self.total == 0) else " (%d%%)" %
(int(100 * sent / self.total), ),
eta,
"" if not mbps else "%.3g Mbps " % (mbps, ),
chunk or "",
" ",
))
self.output.flush()
def _unsafeGetUsage(self):
for (header, buf) in self._walkTree(BTRFS_QUOTA_TREE_OBJECTID):
# logger.debug("%s %s", objectTypeNames[header.type], header)
if header.type == objectTypeKeys['BTRFS_QGROUP_INFO_KEY']:
data = btrfs_qgroup_info_item.read(buf)
try:
vol = self.volumes[header.offset]
vol.totalSize = data.referenced
vol.exclusiveSize = data.exclusive
if (
data.referenced < 0 or
data.exclusive < 0 or
data.referenced < data.exclusive
):
raise _BtrfsError(
"Btrfs returned corrupt size of %s (%s exclusive) for %s" %
(
humanize(vol.totalSize or -1),
humanize(vol.exclusiveSize or -1),
vol.fullPath,
)
)
except KeyError:
pass
elif header.type == objectTypeKeys['BTRFS_QGROUP_LIMIT_KEY']:
data = btrfs_qgroup_limit_item.read(buf)
elif header.type == objectTypeKeys['BTRFS_QGROUP_RELATION_KEY']:
data = None
else:
data = None
def skipChunk(self, chunkSize, checkSum, data=None):
part = self.parts[self.chunkCount]
if part is None:
return False
(size, tag) = part
tag = tag.strip('"')
if size != chunkSize:
logger.warning("Unexpected chunk size %d instead of %d", chunkSize,
size)
# return False
if tag != checkSum:
logger.warning("Bad check sum %d instead of %d", checkSum, tag)
return False
self.chunkCount += 1
logger.info(
"Skipping already uploaded %s chunk #%d",
humanize(chunkSize),
self.chunkCount,
)
return True
def __init__(self, app_path):
self.namespace = app_path
self.name = app_path
self.verbose_name = util.humanize(self.name)
self._module = util.import_from_string(app_path)
self._urls_module = util.import_from_string(app_path + '.urls', True)
self._models_module = util.import_from_string(app_path + '.models', True)
self._views_module = util.import_from_string(app_path + '.views', True)
self.models = RozModel.get_models(self, self._models_module)
self.routes = RozEngine.Engine.app_routes
if hasattr(self._module, 'RozMeta'):
meta = getattr(self._module, 'RozMeta')
RozMetaDefault.set_defaults(meta)
else:
meta = RozMetaDefault
self._module.RozMeta = meta
for key, value in vars(meta).iteritems():
util.safe_setattr(self, key, value)
self.register_views()
def deleteUnused(self):
""" Delete any old snapshots in path, if not kept. """
(count, size) = (0, 0)
for (diff, path) in self.extraKeys.items():
if path.startswith("/"):
continue
keyName = self._keyName(diff.toUUID, diff.fromUUID, path)
count += 1
size += diff.size
if self._skipDryRun(logger, 'INFO')("Trash: %s", diff):
continue
try:
self.bucket.copy_key(theTrashPrefix + keyName,
self.bucket.name, keyName)
self.bucket.delete_key(keyName)
except boto.exception.S3ResponseError as error:
logger.error("%s: %s", error.code, error.message)
try:
keyName = os.path.dirname(keyName) + Store.theInfoExtension
self.bucket.copy_key(theTrashPrefix + keyName,
self.bucket.name, keyName)
self.bucket.delete_key(keyName)
except boto.exception.S3ResponseError as error:
logger.debug("%s: %s", error.code, error.message)
logger.info("Trashed %d diffs (%s)", count, humanize(size))
def deleteUnused(self):
""" Delete any old snapshots in path, if not kept. """
(count, size) = (0, 0)
for (diff, path) in self.extraKeys.items():
if path.startswith("/"):
continue
keyName = self._keyName(diff.toUUID, diff.fromUUID, path)
count += 1
size += diff.size
if self._skipDryRun(logger, 'INFO')("Trash: %s", diff):
continue
try:
self.bucket.copy_key(theTrashPrefix + keyName, self.bucket.name, keyName)
self.bucket.delete_key(keyName)
except boto.exception.S3ResponseError as error:
logger.error("%s: %s", error.code, error.message)
try:
keyName = os.path.dirname(keyName) + Store.theInfoExtension
self.bucket.copy_key(theTrashPrefix + keyName, self.bucket.name, keyName)
self.bucket.delete_key(keyName)
except boto.exception.S3ResponseError as error:
logger.debug("%s: %s", error.code, error.message)
logger.info("Trashed %d diffs (%s)", count, humanize(size))
def __str__(self):
""" human-readable string. """
return u"%s from %s (%s%s)" % (
self.toVol.display(self.sink),
self.fromVol.display(self.sink) if self.fromVol else "None",
"~" if self.sizeIsEstimated else "",
humanize(self.size),
# self.sink,
)
def __str__(self):
""" human-readable string. """
return u"%s from %s (%s%s)" % (
self.toVol.display(self.sink),
self.fromVol.display(self.sink) if self.fromVol else "None",
"~" if self.sizeIsEstimated else "",
humanize(self.size),
# self.sink,
)
def analyze(self, chunkSize, *sinks):
""" Figure out the best diffs to use to reach all our required volumes. """
measureSize = False
if self.measureSize:
for sink in sinks:
if sink.isRemote:
measureSize = True
# Use destination (already uploaded) edges first
sinks = list(sinks)
sinks.reverse()
self.dest = sinks[0]
def currentSize():
return sum([
n.diffSize
for n in self.nodes.values()
if n.diff is not None and n.diff.sink != self.dest
])
while True:
self._analyzeDontMeasure(chunkSize, measureSize, *sinks)
if not measureSize:
return
estimatedSize = currentSize()
# logger.info("Measuring any estimated diffs")
for node in self.nodes.values():
edge = node.diff
if edge is not None and edge.sink != self.dest and edge.sizeIsEstimated:
edge.sink.measureSize(edge, chunkSize)
actualSize = currentSize()
logger.info(
"measured size (%s), estimated size (%s)",
humanize(actualSize), humanize(estimatedSize),
)
if actualSize <= 1.2 * estimatedSize:
return
def analyze(self, chunkSize, *sinks):
""" Figure out the best diffs to use to reach all our required volumes. """
measureSize = False
if self.measureSize:
for sink in sinks:
if sink.isRemote:
measureSize = True
# Use destination (already uploaded) edges first
sinks = list(sinks)
sinks.reverse()
self.dest = sinks[0]
def currentSize():
return sum([
n.diffSize
for n in self.nodes.values()
if n.diff is not None and n.diff.sink != self.dest
])
while True:
self._analyzeDontMeasure(chunkSize, measureSize, *sinks)
if not measureSize:
return
estimatedSize = currentSize()
# logger.info("Measuring any estimated diffs")
for node in self.nodes.values():
edge = node.diff
if edge is not None and edge.sink != self.dest and edge.sizeIsEstimated:
edge.sink.measureSize(edge, chunkSize)
actualSize = currentSize()
logger.info(
"measured size (%s), estimated size (%s)",
humanize(actualSize), humanize(estimatedSize),
)
if actualSize <= 1.2 * estimatedSize:
return
def display(self, sink=None, detail='phrase'):
""" Friendly string for volume, using sink paths. """
if not isinstance(detail, int):
detail = detailNum[detail]
if detail >= detailNum['line'] and self.size is not None:
size = " (%s%s)" % (humanize(
self.size), "" if self.exclusiveSize is None else
(" %s exclusive" %
(humanize(self.exclusiveSize))))
else:
size = ""
vol = "%s %s" % (
_printUUID(self._uuid, detail - 1),
sink.getSendPath(self) if sink else "",
)
return vol + size
def display(self, sink=None, detail='phrase'):
""" Friendly string for volume, using sink paths. """
if not isinstance(detail, int):
detail = detailNum[detail]
if detail >= detailNum['line'] and self.size is not None:
size = " (%s%s)" % (
humanize(self.size),
"" if self.exclusiveSize is None else (
" %s exclusive" % (humanize(self.exclusiveSize))
)
)
else:
size = ""
vol = "%s %s" % (
_printUUID(self._uuid, detail - 1),
sink.getSendPath(self) if sink else "",
)
return vol + size
def __init__(self, app, definition, name):
self._app = app
self.definition = definition
self.name = name
self.verbose_name = util.humanize(self.name)
self.routes = RozEngine.Engine.model_routes
if hasattr(self.definition, 'RozMeta'):
meta = getattr(self.definition, 'RozMeta')
RozMetaDefault.set_defaults(meta)
else:
meta = RozMetaDefault
self.definition.RozMeta = meta
for key, value in vars(meta).iteritems():
util.safe_setattr(self, key, value)
for field_name, field in RozModel.get_model_field_types(self.definition).iteritems():
if not getattr(field, 'verbose_name'):
setattr(field, 'verbose_name', util.humanize(field_name))
def __str__(self):
""" String representation. """
# logger.debug("%d %d %d", self.gen, self.info.generation, self.info.inode.generation)
# logger.debug("%o %o", self.info.flags, self.info.inode.flags)
return """%4d '%s' (level:%d gen:%d total:%s exclusive:%s%s) %s
(parent:%s/%d received:%s/%d)
%s%s""" % (
self.id or -1,
# ", ".join([dirPath + name for (dirPath, name) in self.links.values()]),
self.fullPath,
self.level or -1,
self.current_gen or -1,
# self.size,
humanize(self.totalSize or -1),
humanize(self.exclusiveSize or -1),
" ro" if self.readOnly else "",
self.uuid,
self.parent_uuid, self.original_gen,
self.received_uuid, self.sent_gen,
"\n\t".join(self.linuxPaths),
# "\n\t" + pretty(self.__dict__),
"",
)
def _cost(self, sink, size, prevNode, height):
cost = 0
prevSize = self._totalSize(prevNode)
# Transfer
if sink != self.dest:
cost += size
if prevNode is not None and prevNode.intermediate and prevNode.sink != self.dest:
cost += prevSize
# Storage
if sink != self.dest or self.delete:
cost += size / 16
# Corruption risk
cost += (prevSize + size) * (2 ** (height - 6))
logger.debug(
"_cost=%s (%s %s %s %d)",
humanize(cost), sink, humanize(size), humanize(prevSize), height,
)
return cost
def listContents(self):
""" Return list of volumes or diffs in this Store's selected directory. """
items = list(self.extraKeys.items())
items.sort(key=lambda t: t[1])
(count, size) = (0, 0)
for (diff, path) in items:
if path.startswith("/"):
continue
yield str(diff)
count += 1
size += diff.size
yield "TOTAL: %d diffs %s" % (count, humanize(size))
def listContents(self):
""" Return list of volumes or diffs in this Store's selected directory. """
items = list(self.extraKeys.items())
items.sort(key=lambda t: t[1])
(count, size) = (0, 0)
for (diff, path) in items:
if path.startswith("/"):
continue
yield str(diff)
count += 1
size += diff.size
yield "TOTAL: %d diffs %s" % (count, humanize(size))
def skipChunk(self, chunkSize, checkSum, data=None):
part = self.parts[self.chunkCount]
if part is None:
return False
(size, tag) = part
tag = tag.strip('"')
if size != chunkSize:
logger.warning("Unexpected chunk size %d instead of %d", chunkSize, size)
# return False
if tag != checkSum:
logger.warning("Bad check sum %d instead of %d", checkSum, tag)
return False
self.chunkCount += 1
logger.info(
"Skipping already uploaded %s chunk #%d",
humanize(chunkSize),
self.chunkCount,
)
return True
def _analyzeDontMeasure(self, chunkSize, willMeasureLater, *sinks):
""" Figure out the best diffs to use to reach all our required volumes. """
nodes = [None]
height = 1
def sortKey(node):
if node is None:
return None
return (node.intermediate, self._totalSize(node))
while len(nodes) > 0:
logger.debug("Analyzing %d nodes for height %d...", len(nodes), height)
nodes.sort(key=sortKey)
for fromNode in nodes:
if self._height(fromNode) >= height:
continue
if fromNode is not None and fromNode.diffSize is None:
continue
fromVol = fromNode.volume if fromNode else None
logger.debug("Following edges from %s", fromVol)
for sink in sinks:
# logger.debug(
# "Listing edges in %s",
# sink
# )
for edge in sink.getEdges(fromVol):
toVol = edge.toVol
# logger.debug("Edge: %s", edge)
# Skip any edges already in the destination
if sink != self.dest and self.dest.hasEdge(edge):
continue
if toVol in self.nodes:
toNode = self.nodes[toVol]
# Don't transfer any edges we won't need in the destination
# elif sink != self.dest:
# logger.debug("Won't transfer unnecessary %s", edge)
# continue
else:
toNode = _Node(toVol, True)
self.nodes[toVol] = toNode
logger.debug("Considering %s", edge)
edgeSize = edge.size
if edge.sizeIsEstimated:
if willMeasureLater:
# Slight preference for accurate sizes
edgeSize *= 1.2
else:
# Large preference for accurate sizes
edgeSize *= 2
newCost = self._cost(sink, edgeSize, fromNode, height)
if toNode.diff is None:
oldCost = None
else:
oldCost = self._cost(
toNode.sink,
toNode.diffSize,
self._getNode(toNode.previous),
self._height(toNode)
)
# Don't use a more-expensive path
if oldCost is not None and oldCost <= newCost:
continue
# Don't create circular paths
if self._wouldLoop(fromVol, toVol):
# logger.debug("Ignoring looping edge: %s", toVol.display(sink))
continue
# if measureSize and sink != self.dest and edge.sizeIsEstimated:
# sink.measureSize(edge, chunkSize)
# newCost = self._cost(sink, edge.size, fromSize, height)
# if oldCost is not None and oldCost <= newCost:
# continue
logger.debug(
"Replacing edge (%s -> %s cost)\n%s",
humanize(oldCost),
humanize(newCost),
toNode.display(sink)
)
# logger.debug("Cost elements: %s", dict(
# sink=str(sink),
# edgeSize=humanize(edgeSize),
# fromSize=humanize(fromSize),
# height=height,
# ))
toNode.diff = edge
nodes = [node for node in self.nodes.values() if self._height(node) == height]
height += 1
self._prune()
for node in self.nodes.values():
node.height = self._height(node)
if node.diff is None:
logger.error(
"No source diffs for %s",
node.volume.display(sinks[-1], detail="line"),
)
def main():
""" Main program. """
try:
args = command.parse_args()
_setupLogging(args.quiet, args.logfile, args.server)
logger.debug("Version: %s, Arguments: %s", theVersion, vars(args))
if args.server:
server = SSHStore.StoreProxyServer(args.dest, args.mode)
return(server.run())
source = parseSink(args.source, False, args.delete, args.dry_run)
dest = parseSink(args.dest, source is not None, args.delete, args.dry_run)
if source is None:
source = dest
dest = None
if not sys.stderr.isatty():
source.showProgress = dest.showProgress = False
elif dest is None or (source.isRemote and not dest.isRemote):
source.showProgress = True
else:
dest.showProgress = True
with source:
try:
next(source.listVolumes())
except StopIteration:
logger.warn("No snapshots in source.")
path = args.source or args.dest
if path.endswith("/"):
logger.error(
"'%s' does not contain any snapshots. Did you mean to type '%s'?",
path, path[0:-1]
)
else:
logger.error(
"'%s' is not a snapshot. Did you mean to type '%s/'?",
path, path
)
return 1
if dest is None:
for item in source.listContents():
print item
if args.delete:
source.deletePartials()
return 0
with dest:
best = BestDiffs.BestDiffs(source.listVolumes(), args.delete, not args.estimate)
best.analyze(args.part_size << 20, source, dest)
summary = best.summary()
logger.info("Optimal synchronization:")
for sink, values in summary.items():
logger.info("%s from %d diffs in %s",
humanize(values.size),
values.count,
sink or "TOTAL",
)
for diff in best.iterDiffs():
if diff is None:
raise Exception("Missing diff. Can't fully replicate.")
else:
diff.sendTo(dest, chunkSize=args.part_size << 20)
if args.delete:
dest.deleteUnused()
logger.debug("Successful exit")
return 0
except Exception as error:
if (
isinstance(error, IOError) and
error.errno == errno.EPERM and
os.getuid() != 0
):
logger.error("You must be root to access a btrfs filesystem. Use 'sudo'")
else:
if not theDebug:
logger.debug("Trace information for debugging", exc_info=True)
logger.error("ERROR: %s.", error, exc_info=theDebug)
return 1
def _analyzeDontMeasure(self, chunkSize, willMeasureLater, *sinks):
""" Figure out the best diffs to use to reach all our required volumes. """
nodes = [None]
height = 1
def sortKey(node):
if node is None:
return None
return (node.intermediate, self._totalSize(node))
while len(nodes) > 0:
logger.debug("Analyzing %d nodes for height %d...", len(nodes), height)
nodes.sort(key=sortKey)
for fromNode in nodes:
if self._height(fromNode) >= height:
continue
if fromNode is not None and fromNode.diffSize is None:
continue
fromVol = fromNode.volume if fromNode else None
logger.debug("Following edges from %s", fromVol)
for sink in sinks:
# logger.debug(
# "Listing edges in %s",
# sink
# )
for edge in sink.getEdges(fromVol):
toVol = edge.toVol
# logger.debug("Edge: %s", edge)
# Skip any edges already in the destination
if sink != self.dest and self.dest.hasEdge(edge):
continue
if toVol in self.nodes:
toNode = self.nodes[toVol]
# Don't transfer any edges we won't need in the destination
# elif sink != self.dest:
# logger.debug("Won't transfer unnecessary %s", edge)
# continue
else:
toNode = _Node(toVol, True)
self.nodes[toVol] = toNode
logger.debug("Considering %s", edge)
edgeSize = edge.size
if edge.sizeIsEstimated:
if willMeasureLater:
# Slight preference for accurate sizes
edgeSize *= 1.2
else:
# Large preference for accurate sizes
edgeSize *= 2
newCost = self._cost(sink, edgeSize, fromNode, height)
if toNode.diff is None:
oldCost = None
else:
oldCost = self._cost(
toNode.sink,
toNode.diffSize,
self._getNode(toNode.previous),
self._height(toNode)
)
# Don't use a more-expensive path
if oldCost is not None and oldCost <= newCost:
continue
# Don't create circular paths
if self._wouldLoop(fromVol, toVol):
# logger.debug("Ignoring looping edge: %s", toVol.display(sink))
continue
# if measureSize and sink != self.dest and edge.sizeIsEstimated:
# sink.measureSize(edge, chunkSize)
# newCost = self._cost(sink, edge.size, fromSize, height)
# if oldCost is not None and oldCost <= newCost:
# continue
logger.debug(
"Replacing edge (%s -> %s cost)\n%s",
humanize(oldCost),
humanize(newCost),
toNode.display(sink)
)
# logger.debug("Cost elements: %s", dict(
# sink=str(sink),
# edgeSize=humanize(edgeSize),
# fromSize=humanize(fromSize),
# height=height,
# ))
toNode.diff = edge
nodes = [node for node in self.nodes.values() if self._height(node) == height]
height += 1
self._prune()
for node in self.nodes.values():
node.height = self._height(node)
if node.diff is None:
logger.error(
"No source diffs for %s",
node.volume.display(sinks[-1], detail="line"),
)
def main():
""" Main program. """
try:
args = command.parse_args()
# Use btrfs quota information
useQuota = args.estimate < 2
_setupLogging(args.quiet, args.logfile, args.server)
logger.debug("Version: %s, Arguments: %s", theVersion, vars(args))
if args.server:
server = SSHStore.StoreProxyServer(args.dest, args.mode)
return (server.run())
logger.info(
"Snapshot graph is rebuilt after every transfer completion.")
logger.info(
"Look at option \033[1m-e\033[0m if you want to speed up the process."
)
round = 0
while True:
round += 1
logger.info("Iteration number %s", round)
source = parseSink(args.source, False, args.delete, args.dry_run)
dest = parseSink(args.dest, source is not None, args.delete,
args.dry_run)
if source is None:
source = dest
dest = None
if not sys.stderr.isatty():
source.showProgress = dest.showProgress = False
elif dest is None or (source.isRemote and not dest.isRemote):
source.showProgress = True
else:
dest.showProgress = True
with source:
if useQuota:
source.rescanSizes()
try:
next(source.listVolumes())
except StopIteration:
logger.warn("No snapshots in source.")
path = args.source or args.dest
if path.endswith("/"):
logger.error(
"'%s' does not contain any snapshots. Did you mean to type '%s'?",
path, path[0:-1])
else:
logger.error(
"'%s' is not a snapshot. Did you mean to type '%s/'?",
path, path)
return 1
if dest is None:
for item in source.listContents():
print(item)
if args.delete:
source.deletePartials()
return 0
with dest:
volumes = list(source.listVolumes())
if args.exclude:
def is_excluded(vol):
for path in source.getPaths(vol):
for pattern in args.exclude:
if re.match(pattern, path):
return True
return False
volumes = [
vol for vol in volumes if not is_excluded(vol)
]
if useQuota:
best = BestDiffs.BestDiffs(volumes, args.delete,
not args.estimate)
best.analyze(args.part_size << 20, source, dest)
else:
destVolumes = list(dest.listVolumes())
commonVolumes = list(set(volumes) & set(destVolumes))
missingVolumes = list(set(volumes) - set(destVolumes))
missingVolumeUUIDs = {
vol.uuid
for vol in missingVolumes
}
if not missingVolumes:
break
latestCommonVolume = None
commonVolumes.sort(key=lambda v: v.otime)
if commonVolumes:
latestCommonVolume = commonVolumes[-1]
diffs = source.getEdges(latestCommonVolume)
missingDiffs = [
diff for diff in diffs
if diff.toUUID in missingVolumeUUIDs
]
missingDiffs.sort(key=lambda d: d.toVol.otime)
oldestMissingDiff = missingDiffs[0]
oldestMissingDiff.sendTo(
dest, chunkSize=args.part_size << 20)
continue
summary = best.summary()
logger.info("Optimal synchronization:")
for sink, values in summary.items():
logger.info(
"%s from %d diffs in %s",
humanize(values.size),
values.count,
sink or "TOTAL",
)
try:
# take only first diff
diff = next(best.iterDiffs())
if diff is None:
raise Exception(
"Missing diff. Can't fully replicate.")
else:
diff.sendTo(dest, chunkSize=args.part_size << 20)
except StopIteration:
logger.info("No snapshots left to transfer.")
if args.delete:
dest.deleteUnused()
break
logger.debug("Successful exit")
return 0
except Exception as error:
if (isinstance(error, IOError) and error.errno == errno.EPERM
and os.getuid() != 0):
logger.error(
"You must be root to access a btrfs filesystem. Use 'sudo'")
else:
if not theDebug:
logger.debug("Trace information for debugging", exc_info=True)
logger.error("ERROR: %s.", error, exc_info=theDebug)
return 1
