def _read(self, ptr):
"""Parse the metadata item at the given address and return it."""
if ptr is None or ptr == []: return {}
(ref, n) = ptr[-1]
lines = self._load(ref)[n:]
try:
return next(cumulus.parse(lines, lambda l: len(l) == 0))
except StopIteration:
return {}
def reload_segment_metadata(self, segment_metadata):
"""Read a segment metadata (.meta) file into the local database.
Updates the segments table in the local database with information from
a a segment metadata backup file. Old data is not overwritten, so
loading a .meta file with partial information is fine.
"""
for info in cumulus.parse(segment_metadata,
terminate=lambda l: len(l) == 0):
segment = info.pop("segment")
self.insert_segment_info(segment, info)
self.database.commit()
def reload_segment_metadata(self, segment_metadata):
"""Read a segment metadata (.meta) file into the local database.
Updates the segments table in the local database with information from
a a segment metadata backup file. Old data is not overwritten, so
loading a .meta file with partial information is fine.
"""
for info in cumulus.parse(segment_metadata,
terminate=lambda l: len(l) == 0):
segment = info.pop("segment")
self.insert_segment_info(segment, info)
self.database.commit()
def rebuild(self, metadata, reference_path):
"""Iterate through old metadata and use it to rebuild the database.
Args:
metadata: An iterable containing lines of the metadata log.
reference_path: Path to the root of a file system tree which may be
similar to data in the metadata log (used to recompute block
signatures).
"""
for fields in cumulus.parse(metadata, lambda l: len(l) == 0):
metadata = cumulus.MetadataItem(fields, None)
# Only process regular files; skip over other types (directories,
# symlinks, etc.)
if metadata.items.type not in ("-", "f"): continue
try:
path = os.path.join(reference_path, metadata.items.name)
print("Path:", path)
# TODO: Check file size for early abort if different
self.rebuild_file(open(path), metadata)
except IOError as e:
print(e)
pass # Ignore the file
self.database.commit()
def rebuild(self, metadata, reference_path):
"""Iterate through old metadata and use it to rebuild the database.
Args:
metadata: An iterable containing lines of the metadata log.
reference_path: Path to the root of a file system tree which may be
similar to data in the metadata log (used to recompute block
signatures).
"""
for fields in cumulus.parse(metadata, lambda l: len(l) == 0):
metadata = cumulus.MetadataItem(fields, None)
# Only process regular files; skip over other types (directories,
# symlinks, etc.)
if metadata.items.type not in ("-", "f"): continue
try:
path = os.path.join(reference_path, metadata.items.name)
print("Path:", path)
# TODO: Check file size for early abort if different
self.rebuild_file(open(path), metadata)
except IOError as e:
print(e)
pass # Ignore the file
self.database.commit()
