def pgfid_to_path(brick, changelog_data): """ For all the pgfids in table, converts into path using recursive readlink. """ # pgfid1 to path1 in case of CREATE/MKNOD/MKDIR/LINK/SYMLINK for row in changelog_data.gfidpath_get_distinct("pgfid1", {"path1": ""}): # In case of Data/Metadata only, pgfid1 will not be their if row[0] == "": continue try: path = symlink_gfid_to_path(brick, row[0]) path = output_path_prepare(path, args) changelog_data.gfidpath_set_path1(path, row[0]) except (IOError, OSError) as e: logger.warn("Error converting to path: %s" % e) continue # pgfid2 to path2 in case of RENAME for row in changelog_data.gfidpath_get_distinct("pgfid2", { "type": "RENAME", "path2": "" }): # Only in case of Rename pgfid2 exists if row[0] == "": continue try: path = symlink_gfid_to_path(brick, row[0]) path = output_path_prepare(path, args) changelog_data.gfidpath_set_path2(path, row[0]) except (IOError, OSError) as e: logger.warn("Error converting to path: %s" % e) continue
def pgfid_to_path(brick, changelog_data): """ For all the pgfids in table, converts into path using recursive readlink. """ # pgfid1 to path1 in case of CREATE/MKNOD/MKDIR/LINK/SYMLINK for row in changelog_data.gfidpath_get_distinct("pgfid1", {"path1": ""}): # In case of Data/Metadata only, pgfid1 will not be their if row[0] == "": continue try: path = symlink_gfid_to_path(brick, row[0]) path = output_path_prepare(path, args) changelog_data.gfidpath_set_path1(path, row[0]) except (IOError, OSError) as e: logger.warn("Error converting to path: %s" % e) continue # pgfid2 to path2 in case of RENAME for row in changelog_data.gfidpath_get_distinct("pgfid2", {"type": "RENAME", "path2": ""}): # Only in case of Rename pgfid2 exists if row[0] == "": continue try: path = symlink_gfid_to_path(brick, row[0]) path = output_path_prepare(path, args) changelog_data.gfidpath_set_path2(path, row[0]) except (IOError, OSError) as e: logger.warn("Error converting to path: %s" % e) continue
def populate_pgfid_and_inodegfid(brick, changelog_data): """ For all the DATA/METADATA modifications GFID, If symlink, directly convert to Path using Readlink. If not symlink, try to get PGFIDs via xattr query and populate it to pgfid table, collect inodes in inodegfid table """ for row in changelog_data.gfidpath_get({"path1": "", "type": "MODIFY"}): gfid = row[3].strip() p = os.path.join(brick, ".glusterfs", gfid[0:2], gfid[2:4], gfid) if os.path.islink(p): # It is a Directory if GFID backend path is symlink try: path = symlink_gfid_to_path(brick, gfid) path = output_path_prepare(path, args) changelog_data.gfidpath_update({"path1": path}, {"gfid": gfid}) except (IOError, OSError) as e: logger.warn("Error converting to path: %s" % e) continue else: try: # INODE and GFID to inodegfid table changelog_data.inodegfid_add(os.stat(p).st_ino, gfid) file_xattrs = xattr.list(p) for x in file_xattrs: if x.startswith("trusted.pgfid."): # PGFID in pgfid table changelog_data.pgfid_add(x.split(".")[-1]) except (IOError, OSError): # All OS Errors ignored, since failures will be logged # in End. All GFIDs present in gfidpath table continue
def enum_hard_links_using_gfid2path(brick, gfid, args): hardlinks = [] p = os.path.join(brick, ".glusterfs", gfid[0:2], gfid[2:4], gfid) if not os.path.isdir(p): # we have a symlink or a normal file try: file_xattrs = xattr.list(p) for x in file_xattrs: x_str = bytearray_to_str(x) if x_str.startswith("trusted.gfid2path."): # get the value for the xattr i.e. <PGFID>/<BN> v = xattr.getxattr(p, x_str) v_str = bytearray_to_str(v) pgfid, bn = v_str.split(os.sep) try: path = symlink_gfid_to_path(brick, pgfid) fullpath = os.path.join(path, bn) fullpath = output_path_prepare(fullpath, args) hardlinks.append(fullpath) except (IOError, OSError) as e: logger.warn("Error converting to path: %s" % e) continue except (IOError, OSError): pass return hardlinks
def when_unlink_rmdir(self, changelogfile, data, args): # E <GFID> <UNLINK|RMDIR> <PGFID>/<BASENAME> pgfid1, bn1 = urllib.unquote_plus(data[3]).split("/", 1) # Quote again the basename bn1 = urllib.quote_plus(bn1.strip()) deleted_path = data[4] if len(data) == 5 else "" if deleted_path != "": deleted_path = output_path_prepare(deleted_path, args.output_prefix) if self.gfidpath_exists({"gfid": data[1], "type": "NEW", "pgfid1": pgfid1, "bn1": bn1}): # If path exists in table as NEW with same GFID # Delete that row self.gfidpath_delete({"gfid": data[1], "type": "NEW", "pgfid1": pgfid1, "bn1": bn1}) else: # Else Record as DELETE self.gfidpath_add(changelogfile, RecordType.DELETE, data[1], pgfid1, bn1, path1=deleted_path) # Update path1 as deleted_path if pgfid1 and bn1 is same as deleted self.gfidpath_update({"path1": deleted_path}, {"gfid": data[1], "pgfid1": pgfid1, "bn1": bn1}) # Update path2 as deleted_path if pgfid2 and bn2 is same as deleted self.gfidpath_update( {"path2": deleted_path}, {"type": RecordType.RENAME, "gfid": data[1], "pgfid2": pgfid1, "bn2": bn1} ) # If deleted directory is parent for somebody query1 = """UPDATE gfidpath SET path1 = ? || '%2F' || bn1 WHERE pgfid1 = ? AND path1 != ''""" self.cursor.execute(query1, (deleted_path, data[1])) query1 = """UPDATE gfidpath SET path2 = ? || '%2F' || bn1 WHERE pgfid2 = ? AND path2 != ''""" self.cursor.execute(query1, (deleted_path, data[1]))
def output_callback(path, inode): # For each path found, encodes it and updates path1 # Also updates converted flag in inodegfid table as 1 path = path.strip() path = path[brick_path_len + 1:] path = output_path_prepare(path, args) changelog_data.append_path1(path, inode)
def output_callback(path, inode): # For each path found, encodes it and updates path1 # Also updates converted flag in inodegfid table as 1 path = path.strip() path = path[brick_path_len+1:] path = output_path_prepare(path, args) changelog_data.append_path1(path, inode)
def when_unlink_rmdir(self, changelogfile, data): # E <GFID> <UNLINK|RMDIR> <PGFID>/<BASENAME> pgfid1, bn1 = data[3].split("/", 1) if self.args.no_encode: bn1 = unquote_plus_space_newline(bn1).strip() deleted_path = data[4] if len(data) == 5 else "" if deleted_path != "": deleted_path = unquote_plus_space_newline(deleted_path) deleted_path = output_path_prepare(deleted_path, self.args) if self.gfidpath_exists({ "gfid": data[1], "type": "NEW", "pgfid1": pgfid1, "bn1": bn1 }): # If path exists in table as NEW with same GFID # Delete that row self.gfidpath_delete({ "gfid": data[1], "type": "NEW", "pgfid1": pgfid1, "bn1": bn1 }) else: # Else Record as DELETE self.gfidpath_add(changelogfile, RecordType.DELETE, data[1], pgfid1, bn1, path1=deleted_path) # Update path1 as deleted_path if pgfid1 and bn1 is same as deleted self.gfidpath_update({"path1": deleted_path}, { "gfid": data[1], "pgfid1": pgfid1, "bn1": bn1 }) # Update path2 as deleted_path if pgfid2 and bn2 is same as deleted self.gfidpath_update({"path2": deleted_path}, { "type": RecordType.RENAME, "gfid": data[1], "pgfid2": pgfid1, "bn2": bn1 }) # If deleted directory is parent for somebody query1 = """UPDATE gfidpath SET path1 = ? || '{0}' || bn1 WHERE pgfid1 = ? AND path1 != ''""".format(self.path_sep) self.cursor.execute(query1, (deleted_path, data[1])) query1 = """UPDATE gfidpath SET path2 = ? || '{0}' || bn1 WHERE pgfid2 = ? AND path2 != ''""".format(self.path_sep) self.cursor.execute(query1, (deleted_path, data[1]))