def pgfid_to_path(brick, changelog_data):
"""
For all the pgfids in table, converts into path using recursive
readlink.
"""
# pgfid1 to path1 in case of CREATE/MKNOD/MKDIR/LINK/SYMLINK
for row in changelog_data.gfidpath_get_distinct("pgfid1", {"path1": ""}):
# In case of Data/Metadata only, pgfid1 will not be their
if row[0] == "":
continue
try:
path = symlink_gfid_to_path(brick, row[0])
path = output_path_prepare(path, args)
changelog_data.gfidpath_set_path1(path, row[0])
except (IOError, OSError) as e:
logger.warn("Error converting to path: %s" % e)
continue
# pgfid2 to path2 in case of RENAME
for row in changelog_data.gfidpath_get_distinct("pgfid2", {
"type": "RENAME",
"path2": ""
}):
# Only in case of Rename pgfid2 exists
if row[0] == "":
continue
try:
path = symlink_gfid_to_path(brick, row[0])
path = output_path_prepare(path, args)
changelog_data.gfidpath_set_path2(path, row[0])
except (IOError, OSError) as e:
logger.warn("Error converting to path: %s" % e)
continue
def pgfid_to_path(brick, changelog_data):
"""
For all the pgfids in table, converts into path using recursive
readlink.
"""
# pgfid1 to path1 in case of CREATE/MKNOD/MKDIR/LINK/SYMLINK
for row in changelog_data.gfidpath_get_distinct("pgfid1", {"path1": ""}):
# In case of Data/Metadata only, pgfid1 will not be their
if row[0] == "":
continue
try:
path = symlink_gfid_to_path(brick, row[0])
path = output_path_prepare(path, args)
changelog_data.gfidpath_set_path1(path, row[0])
except (IOError, OSError) as e:
logger.warn("Error converting to path: %s" % e)
continue
# pgfid2 to path2 in case of RENAME
for row in changelog_data.gfidpath_get_distinct("pgfid2",
{"type": "RENAME",
"path2": ""}):
# Only in case of Rename pgfid2 exists
if row[0] == "":
continue
try:
path = symlink_gfid_to_path(brick, row[0])
path = output_path_prepare(path, args)
changelog_data.gfidpath_set_path2(path, row[0])
except (IOError, OSError) as e:
logger.warn("Error converting to path: %s" % e)
continue
def populate_pgfid_and_inodegfid(brick, changelog_data):
"""
For all the DATA/METADATA modifications GFID,
If symlink, directly convert to Path using Readlink.
If not symlink, try to get PGFIDs via xattr query and populate it
to pgfid table, collect inodes in inodegfid table
"""
for row in changelog_data.gfidpath_get({"path1": "", "type": "MODIFY"}):
gfid = row[3].strip()
p = os.path.join(brick, ".glusterfs", gfid[0:2], gfid[2:4], gfid)
if os.path.islink(p):
# It is a Directory if GFID backend path is symlink
try:
path = symlink_gfid_to_path(brick, gfid)
path = output_path_prepare(path, args)
changelog_data.gfidpath_update({"path1": path}, {"gfid": gfid})
except (IOError, OSError) as e:
logger.warn("Error converting to path: %s" % e)
continue
else:
try:
# INODE and GFID to inodegfid table
changelog_data.inodegfid_add(os.stat(p).st_ino, gfid)
file_xattrs = xattr.list(p)
for x in file_xattrs:
if x.startswith("trusted.pgfid."):
# PGFID in pgfid table
changelog_data.pgfid_add(x.split(".")[-1])
except (IOError, OSError):
# All OS Errors ignored, since failures will be logged
# in End. All GFIDs present in gfidpath table
continue
def enum_hard_links_using_gfid2path(brick, gfid, args):
hardlinks = []
p = os.path.join(brick, ".glusterfs", gfid[0:2], gfid[2:4], gfid)
if not os.path.isdir(p):
# we have a symlink or a normal file
try:
file_xattrs = xattr.list(p)
for x in file_xattrs:
x_str = bytearray_to_str(x)
if x_str.startswith("trusted.gfid2path."):
# get the value for the xattr i.e. <PGFID>/<BN>
v = xattr.getxattr(p, x_str)
v_str = bytearray_to_str(v)
pgfid, bn = v_str.split(os.sep)
try:
path = symlink_gfid_to_path(brick, pgfid)
fullpath = os.path.join(path, bn)
fullpath = output_path_prepare(fullpath, args)
hardlinks.append(fullpath)
except (IOError, OSError) as e:
logger.warn("Error converting to path: %s" % e)
continue
except (IOError, OSError):
pass
return hardlinks
def populate_pgfid_and_inodegfid(brick, changelog_data):
"""
For all the DATA/METADATA modifications GFID,
If symlink, directly convert to Path using Readlink.
If not symlink, try to get PGFIDs via xattr query and populate it
to pgfid table, collect inodes in inodegfid table
"""
for row in changelog_data.gfidpath_get({"path1": "", "type": "MODIFY"}):
gfid = row[3].strip()
p = os.path.join(brick, ".glusterfs", gfid[0:2], gfid[2:4], gfid)
if os.path.islink(p):
# It is a Directory if GFID backend path is symlink
try:
path = symlink_gfid_to_path(brick, gfid)
path = output_path_prepare(path, args)
changelog_data.gfidpath_update({"path1": path},
{"gfid": gfid})
except (IOError, OSError) as e:
logger.warn("Error converting to path: %s" % e)
continue
else:
try:
# INODE and GFID to inodegfid table
changelog_data.inodegfid_add(os.stat(p).st_ino, gfid)
file_xattrs = xattr.list(p)
for x in file_xattrs:
if x.startswith("trusted.pgfid."):
# PGFID in pgfid table
changelog_data.pgfid_add(x.split(".")[-1])
except (IOError, OSError):
# All OS Errors ignored, since failures will be logged
# in End. All GFIDs present in gfidpath table
continue
def when_unlink_rmdir(self, changelogfile, data, args):
# E <GFID> <UNLINK|RMDIR> <PGFID>/<BASENAME>
pgfid1, bn1 = urllib.unquote_plus(data[3]).split("/", 1)
# Quote again the basename
bn1 = urllib.quote_plus(bn1.strip())
deleted_path = data[4] if len(data) == 5 else ""
if deleted_path != "":
deleted_path = output_path_prepare(deleted_path, args.output_prefix)
if self.gfidpath_exists({"gfid": data[1], "type": "NEW", "pgfid1": pgfid1, "bn1": bn1}):
# If path exists in table as NEW with same GFID
# Delete that row
self.gfidpath_delete({"gfid": data[1], "type": "NEW", "pgfid1": pgfid1, "bn1": bn1})
else:
# Else Record as DELETE
self.gfidpath_add(changelogfile, RecordType.DELETE, data[1], pgfid1, bn1, path1=deleted_path)
# Update path1 as deleted_path if pgfid1 and bn1 is same as deleted
self.gfidpath_update({"path1": deleted_path}, {"gfid": data[1], "pgfid1": pgfid1, "bn1": bn1})
# Update path2 as deleted_path if pgfid2 and bn2 is same as deleted
self.gfidpath_update(
{"path2": deleted_path}, {"type": RecordType.RENAME, "gfid": data[1], "pgfid2": pgfid1, "bn2": bn1}
)
# If deleted directory is parent for somebody
query1 = """UPDATE gfidpath SET path1 = ? || '%2F' || bn1
WHERE pgfid1 = ? AND path1 != ''"""
self.cursor.execute(query1, (deleted_path, data[1]))
query1 = """UPDATE gfidpath SET path2 = ? || '%2F' || bn1
WHERE pgfid2 = ? AND path2 != ''"""
self.cursor.execute(query1, (deleted_path, data[1]))
def output_callback(path, inode):
# For each path found, encodes it and updates path1
# Also updates converted flag in inodegfid table as 1
path = path.strip()
path = path[brick_path_len + 1:]
path = output_path_prepare(path, args)
changelog_data.append_path1(path, inode)
def output_callback(path, inode):
# For each path found, encodes it and updates path1
# Also updates converted flag in inodegfid table as 1
path = path.strip()
path = path[brick_path_len+1:]
path = output_path_prepare(path, args)
changelog_data.append_path1(path, inode)
def when_unlink_rmdir(self, changelogfile, data):
# E <GFID> <UNLINK|RMDIR> <PGFID>/<BASENAME>
pgfid1, bn1 = data[3].split("/", 1)
if self.args.no_encode:
bn1 = unquote_plus_space_newline(bn1).strip()
deleted_path = data[4] if len(data) == 5 else ""
if deleted_path != "":
deleted_path = unquote_plus_space_newline(deleted_path)
deleted_path = output_path_prepare(deleted_path, self.args)
if self.gfidpath_exists({
"gfid": data[1],
"type": "NEW",
"pgfid1": pgfid1,
"bn1": bn1
}):
# If path exists in table as NEW with same GFID
# Delete that row
self.gfidpath_delete({
"gfid": data[1],
"type": "NEW",
"pgfid1": pgfid1,
"bn1": bn1
})
else:
# Else Record as DELETE
self.gfidpath_add(changelogfile,
RecordType.DELETE,
data[1],
pgfid1,
bn1,
path1=deleted_path)
# Update path1 as deleted_path if pgfid1 and bn1 is same as deleted
self.gfidpath_update({"path1": deleted_path}, {
"gfid": data[1],
"pgfid1": pgfid1,
"bn1": bn1
})
# Update path2 as deleted_path if pgfid2 and bn2 is same as deleted
self.gfidpath_update({"path2": deleted_path}, {
"type": RecordType.RENAME,
"gfid": data[1],
"pgfid2": pgfid1,
"bn2": bn1
})
# If deleted directory is parent for somebody
query1 = """UPDATE gfidpath SET path1 = ? || '{0}' || bn1
WHERE pgfid1 = ? AND path1 != ''""".format(self.path_sep)
self.cursor.execute(query1, (deleted_path, data[1]))
query1 = """UPDATE gfidpath SET path2 = ? || '{0}' || bn1
WHERE pgfid2 = ? AND path2 != ''""".format(self.path_sep)
self.cursor.execute(query1, (deleted_path, data[1]))
