Exemple #1
0
def cmd_list_snapshot_sizes(args):
    """ List size of data needed for each snapshot.
        Syntax: $0 --data=DATADIR list-snapshot-sizes
    """
    store = cumulus.CumulusStore(options.store)
    backend = store.backend
    backend.prefetch_generic()
    previous = set()
    size = 0

    def get_size(segment):
        return backend.stat_generic(segment + ".tar", "segments")["size"]

    for s in sorted(store.list_snapshots()):
        d = cumulus.parse_full(store.load_snapshot(s))
        check_version(d['Format'])

        segments = set(d['Segments'].split())
        (added, removed, addcount, remcount) = (0, 0, 0, 0)
        for seg in segments.difference(previous):
            added += get_size(seg)
            addcount += 1
        for seg in previous.difference(segments):
            removed += get_size(seg)
            remcount += 1
        size += added - removed
        previous = segments
        print("%s: %.3f +%.3f -%.3f (+%d/-%d segments)" %
              (s, size / 1024.0**2, added / 1024.0**2, removed / 1024.0**2,
               addcount, remcount))
Exemple #2
0
def cmd_list_snapshot_sizes(args):
    """ List size of data needed for each snapshot.
        Syntax: $0 --data=DATADIR list-snapshot-sizes
    """
    store = cumulus.CumulusStore(options.store)
    backend = store.backend
    backend.prefetch_generic()
    previous = set()
    size = 0
    def get_size(segment):
        return backend.stat_generic(segment + ".tar", "segments")["size"]
    for s in sorted(store.list_snapshots()):
        d = cumulus.parse_full(store.load_snapshot(s))
        check_version(d['Format'])

        segments = set(d['Segments'].split())
        (added, removed, addcount, remcount) = (0, 0, 0, 0)
        for seg in segments.difference(previous):
            added += get_size(seg)
            addcount += 1
        for seg in previous.difference(segments):
            removed += get_size(seg)
            remcount += 1
        size += added - removed
        previous = segments
        print("%s: %.3f +%.3f -%.3f (+%d/-%d segments)" % (s, size / 1024.0**2, added / 1024.0**2, removed / 1024.0**2, addcount, remcount))
Exemple #3
0
def cmd_garbage_collect(args):
    """ Search for any files which are not needed by any current
        snapshots and offer to delete them.
        Syntax: $0 --store=DATADIR gc
    """
    store = cumulus.CumulusStore(options.store)
    backend = store.backend
    referenced = set()
    for s in store.list_snapshots():
        d = cumulus.parse_full(store.load_snapshot(s))
        check_version(d['Format'])
        referenced.add(s)
        referenced.update(d['Segments'].split())

    print(referenced)

    to_delete = []
    to_preserve = []
    for filetype in cumulus.SEARCH_PATHS:
        for (name, path) in store.backend.list_generic(filetype):
            if name in referenced:
                to_preserve.append(path)
            else:
                to_delete.append(path)

    print(to_preserve)
    print(to_delete)

    raw_backend = backend.raw_backend
    for f in to_delete:
        print("Delete:", f)
        if not options.dry_run:
            raw_backend.delete(f)
Exemple #4
0
def cmd_garbage_collect(args):
    """ Search for any files which are not needed by any current
        snapshots and offer to delete them.
        Syntax: $0 --store=DATADIR gc
    """
    store = cumulus.CumulusStore(options.store)
    backend = store.backend
    referenced = set()
    for s in store.list_snapshots():
        d = cumulus.parse_full(store.load_snapshot(s))
        check_version(d['Format'])
        referenced.add(s)
        referenced.update(d['Segments'].split())

    print(referenced)

    to_delete = []
    to_preserve = []
    for filetype in cumulus.SEARCH_PATHS:
        for (name, path) in store.backend.list_generic(filetype):
            if name in referenced:
                to_preserve.append(path)
            else:
                to_delete.append(path)

    print(to_preserve)
    print(to_delete)

    raw_backend = backend.raw_backend
    for f in to_delete:
        print("Delete:", f)
        if not options.dry_run:
            raw_backend.delete(f)
Exemple #5
0
def cmd_read_snapshots(snapshots):
    """ Read a snapshot file
    """
    get_passphrase()
    store = cumulus.CumulusStore(options.store)
    for s in snapshots:
        d = cumulus.parse_full(store.load_snapshot(s))
        check_version(d['Format'])
        print(d)
        print(d['Segments'].split())
    store.cleanup()
Exemple #6
0
def cmd_read_snapshots(snapshots):
    """ Read a snapshot file
    """
    get_passphrase()
    store = cumulus.CumulusStore(options.store)
    for s in snapshots:
        d = cumulus.parse_full(store.load_snapshot(s))
        check_version(d['Format'])
        print(d)
        print(d['Segments'].split())
    store.cleanup()
Exemple #7
0
def cmd_read_metadata(args):
    """ Produce a flattened metadata dump from a snapshot
    """
    snapshot = args[0]
    get_passphrase()
    store = cumulus.CumulusStore(options.store)
    d = cumulus.parse_full(store.load_snapshot(snapshot))
    check_version(d['Format'])
    metadata = cumulus.read_metadata(store, d['Root'])
    blank = True
    for l in metadata:
        if l == '\n':
            if blank: continue
            blank = True
        else:
            blank = False
        sys.stdout.write(l)
    store.cleanup()
Exemple #8
0
def cmd_read_metadata(args):
    """ Produce a flattened metadata dump from a snapshot
    """
    snapshot = args [0]
    get_passphrase()
    store = cumulus.CumulusStore(options.store)
    d = cumulus.parse_full(store.load_snapshot(snapshot))
    check_version(d['Format'])
    metadata = cumulus.read_metadata(store, d['Root'])
    blank = True
    for l in metadata:
        if l == '\n':
            if blank: continue
            blank = True
        else:
            blank = False
        sys.stdout.write(l)
    store.cleanup()
Exemple #9
0
def cmd_verify_snapshots(snapshots):
    """ Verify snapshot integrity
    """
    get_passphrase()
    store = cumulus.CumulusStore(options.store)
    for s in snapshots:
        cumulus.accessed_segments.clear()
        print("#### Snapshot", s)
        d = cumulus.parse_full(store.load_snapshot(s))
        check_version(d['Format'])
        print("## Root:", d['Root'])
        metadata = cumulus.iterate_metadata(store, d['Root'])
        for m in metadata:
            if m.fields['type'] not in ('-', 'f'): continue
            print("%s [%d bytes]" % (m.fields['name'], int(m.fields['size'])))
            verifier = cumulus.ChecksumVerifier(m.fields['checksum'])
            size = 0
            for block in m.data():
                data = store.get(block)
                verifier.update(data)
                size += len(data)
            if int(m.fields['size']) != size:
                raise ValueError("File size does not match!")
            if not verifier.valid():
                raise ValueError("Bad checksum found")

        # Verify that the list of segments included with the snapshot was
        # actually accurate: covered all segments that were really read, and
        # doesn't contain duplicates.
        listed_segments = set(d['Segments'].split())
        if cumulus.accessed_segments - listed_segments:
            print("Error: Some segments not listed in descriptor!")
            print(sorted(list(cumulus.accessed_segments - listed_segments)))
        if listed_segments - cumulus.accessed_segments:
            print("Warning: Extra unused segments listed in descriptor!")
            print(sorted(list(listed_segments - cumulus.accessed_segments)))
    store.cleanup()
Exemple #10
0
def cmd_verify_snapshots(snapshots):
    """ Verify snapshot integrity
    """
    get_passphrase()
    store = cumulus.CumulusStore(options.store)
    for s in snapshots:
        cumulus.accessed_segments.clear()
        print("#### Snapshot", s)
        d = cumulus.parse_full(store.load_snapshot(s))
        check_version(d['Format'])
        print("## Root:", d['Root'])
        metadata = cumulus.iterate_metadata(store, d['Root'])
        for m in metadata:
            if m.fields['type'] not in ('-', 'f'): continue
            print("%s [%d bytes]" % (m.fields['name'], int(m.fields['size'])))
            verifier = cumulus.ChecksumVerifier(m.fields['checksum'])
            size = 0
            for block in m.data():
                data = store.get(block)
                verifier.update(data)
                size += len(data)
            if int(m.fields['size']) != size:
                raise ValueError("File size does not match!")
            if not verifier.valid():
                raise ValueError("Bad checksum found")

        # Verify that the list of segments included with the snapshot was
        # actually accurate: covered all segments that were really read, and
        # doesn't contain duplicates.
        listed_segments = set(d['Segments'].split())
        if cumulus.accessed_segments - listed_segments:
            print("Error: Some segments not listed in descriptor!")
            print(sorted(list(cumulus.accessed_segments - listed_segments)))
        if listed_segments - cumulus.accessed_segments :
            print("Warning: Extra unused segments listed in descriptor!")
            print(sorted(list(listed_segments - cumulus.accessed_segments)))
    store.cleanup()
Exemple #11
0
def cmd_restore_snapshot(args):
    """ Restore a snapshot, or some subset of files from it
    """
    get_passphrase()
    store = cumulus.CumulusStore(options.store)
    snapshot = cumulus.parse_full(store.load_snapshot(args[0]))
    check_version(snapshot['Format'])
    destdir = args[1]
    paths = args[2:]

    def matchpath(path):
        "Return true if the specified path should be included in the restore."

        # No specification of what to restore => restore everything
        if len(paths) == 0: return True

        for p in paths:
            if path == p: return True
            if path.startswith(p + "/"): return True
        return False

    def warn(m, msg):
        print("Warning: %s: %s" % (m.items.name, msg))

    # Phase 1: Read the complete metadata log and create directory structure.
    metadata_items = []
    metadata_paths = {}
    metadata_segments = {}
    for m in cumulus.iterate_metadata(store, snapshot['Root']):
        pathname = os.path.normpath(m.items.name)
        while os.path.isabs(pathname):
            pathname = pathname[1:]
        if not matchpath(pathname): continue

        destpath = os.path.join(destdir, pathname)
        if m.items.type == 'd':
            path = destpath
        else:
            (path, filename) = os.path.split(destpath)

        metadata_items.append((pathname, m))
        if m.items.type in ('-', 'f'):
            metadata_paths[pathname] = m
            for block in m.data():
                (segment, object, checksum, slice) \
                    = cumulus.CumulusStore.parse_ref(block)
                if segment not in metadata_segments:
                    metadata_segments[segment] = set()
                metadata_segments[segment].add(pathname)

        try:
            if not os.path.isdir(path):
                print("mkdir:", path)
                os.makedirs(path)
        except Exception as e:
            warn(m, "Error creating directory structure: %s" % (e, ))
            continue

    # Phase 2: Restore files, ordered by how data is stored in segments.
    def restore_file(pathname, m):
        assert m.items.type in ('-', 'f')
        print("extract:", pathname)
        destpath = os.path.join(destdir, pathname)

        file = open(destpath, 'wb')
        verifier = cumulus.ChecksumVerifier(m.items.checksum)
        size = 0
        for block in m.data():
            data = store.get(block)
            verifier.update(data)
            size += len(data)
            file.write(data)
        file.close()
        if int(m.fields['size']) != size:
            raise ValueError("File size does not match!")
        if not verifier.valid():
            raise ValueError("Bad checksum found")

    while metadata_segments:
        (segment, items) = metadata_segments.popitem()
        print("+ Segment", segment)
        for pathname in sorted(items):
            if pathname in metadata_paths:
                restore_file(pathname, metadata_paths[pathname])
                del metadata_paths[pathname]

    print("+ Remaining files")
    while metadata_paths:
        (pathname, m) = metadata_paths.popitem()
        restore_file(pathname, m)

    # Phase 3: Restore special files (symlinks, devices).
    # Phase 4: Restore directory permissions and modification times.
    for (pathname, m) in reversed(metadata_items):
        print("permissions:", pathname)
        destpath = os.path.join(destdir, pathname)
        (path, filename) = os.path.split(destpath)

        # TODO: Check for ../../../paths that might attempt to write outside
        # the destination directory.  Maybe also check attempts to follow
        # symlinks pointing outside?

        try:
            if m.items.type in ('-', 'f', 'd'):
                pass
            elif m.items.type == 'l':
                try:
                    target = m.items.target
                except:
                    # Old (v0.2 format) name for 'target'
                    target = m.items.contents
                os.symlink(target, destpath)
            elif m.items.type == 'p':
                os.mkfifo(destpath)
            elif m.items.type in ('c', 'b'):
                if m.items.type == 'c':
                    mode = 0o600 | stat.S_IFCHR
                else:
                    mode = 0o600 | stat.S_IFBLK
                os.mknod(destpath, mode, os.makedev(*m.items.device))
            elif m.items.type == 's':
                pass  # TODO: Implement
            else:
                warn(m, "Unknown type code: " + m.items.type)
                continue

        except Exception as e:
            warn(m, "Error restoring: %s" % (e, ))
            continue

        try:
            uid = m.items.user[0]
            gid = m.items.group[0]
            os.lchown(destpath, uid, gid)
        except Exception as e:
            warn(m, "Error restoring file ownership: %s" % (e, ))

        if m.items.type == 'l':
            continue

        try:
            os.chmod(destpath, m.items.mode)
        except Exception as e:
            warn(m, "Error restoring file permissions: %s" % (e, ))

        try:
            os.utime(destpath, (time.time(), m.items.mtime))
        except Exception as e:
            warn(m, "Error restoring file timestamps: %s" % (e, ))

    store.cleanup()
Exemple #12
0
def cmd_restore_snapshot(args):
    """ Restore a snapshot, or some subset of files from it
    """
    get_passphrase()
    store = cumulus.CumulusStore(options.store)
    snapshot = cumulus.parse_full(store.load_snapshot(args[0]))
    check_version(snapshot['Format'])
    destdir = args[1]
    paths = args[2:]

    def matchpath(path):
        "Return true if the specified path should be included in the restore."

        # No specification of what to restore => restore everything
        if len(paths) == 0: return True

        for p in paths:
            if path == p: return True
            if path.startswith(p + "/"): return True
        return False

    def warn(m, msg):
        print("Warning: %s: %s" % (m.items.name, msg))

    # Phase 1: Read the complete metadata log and create directory structure.
    metadata_items = []
    metadata_paths = {}
    metadata_segments = {}
    for m in cumulus.iterate_metadata(store, snapshot['Root']):
        pathname = os.path.normpath(m.items.name)
        while os.path.isabs(pathname):
            pathname = pathname[1:]
        if not matchpath(pathname): continue

        destpath = os.path.join(destdir, pathname)
        if m.items.type == 'd':
            path = destpath
        else:
            (path, filename) = os.path.split(destpath)

        metadata_items.append((pathname, m))
        if m.items.type in ('-', 'f'):
            metadata_paths[pathname] = m
            for block in m.data():
                (segment, object, checksum, slice) \
                    = cumulus.CumulusStore.parse_ref(block)
                if segment not in metadata_segments:
                    metadata_segments[segment] = set()
                metadata_segments[segment].add(pathname)

        try:
            if not os.path.isdir(path):
                print("mkdir:", path)
                os.makedirs(path)
        except Exception as e:
            warn(m, "Error creating directory structure: %s" % (e,))
            continue

    # Phase 2: Restore files, ordered by how data is stored in segments.
    def restore_file(pathname, m):
        assert m.items.type in ('-', 'f')
        print("extract:", pathname)
        destpath = os.path.join(destdir, pathname)

        file = open(destpath, 'wb')
        verifier = cumulus.ChecksumVerifier(m.items.checksum)
        size = 0
        for block in m.data():
            data = store.get(block)
            verifier.update(data)
            size += len(data)
            file.write(data)
        file.close()
        if int(m.fields['size']) != size:
            raise ValueError("File size does not match!")
        if not verifier.valid():
            raise ValueError("Bad checksum found")

    while metadata_segments:
        (segment, items) = metadata_segments.popitem()
        print("+ Segment", segment)
        for pathname in sorted(items):
            if pathname in metadata_paths:
                restore_file(pathname, metadata_paths[pathname])
                del metadata_paths[pathname]

    print("+ Remaining files")
    while metadata_paths:
        (pathname, m) = metadata_paths.popitem()
        restore_file(pathname, m)

    # Phase 3: Restore special files (symlinks, devices).
    # Phase 4: Restore directory permissions and modification times.
    for (pathname, m) in reversed(metadata_items):
        print("permissions:", pathname)
        destpath = os.path.join(destdir, pathname)
        (path, filename) = os.path.split(destpath)

        # TODO: Check for ../../../paths that might attempt to write outside
        # the destination directory.  Maybe also check attempts to follow
        # symlinks pointing outside?

        try:
            if m.items.type in ('-', 'f', 'd'):
                pass
            elif m.items.type == 'l':
                try:
                    target = m.items.target
                except:
                    # Old (v0.2 format) name for 'target'
                    target = m.items.contents
                os.symlink(target, destpath)
            elif m.items.type == 'p':
                os.mkfifo(destpath)
            elif m.items.type in ('c', 'b'):
                if m.items.type == 'c':
                    mode = 0o600 | stat.S_IFCHR
                else:
                    mode = 0o600 | stat.S_IFBLK
                os.mknod(destpath, mode, os.makedev(*m.items.device))
            elif m.items.type == 's':
                pass        # TODO: Implement
            else:
                warn(m, "Unknown type code: " + m.items.type)
                continue

        except Exception as e:
            warn(m, "Error restoring: %s" % (e,))
            continue

        try:
            uid = m.items.user[0]
            gid = m.items.group[0]
            os.lchown(destpath, uid, gid)
        except Exception as e:
            warn(m, "Error restoring file ownership: %s" % (e,))

        if m.items.type == 'l':
            continue

        try:
            os.chmod(destpath, m.items.mode)
        except Exception as e:
            warn(m, "Error restoring file permissions: %s" % (e,))

        try:
            os.utime(destpath, (time.time(), m.items.mtime))
        except Exception as e:
            warn(m, "Error restoring file timestamps: %s" % (e,))

    store.cleanup()