def get_file_atts_read(file: str):
# reading all at once to minimize data access
xattr_list = os.listxattr(file)
file_attributes_sum = [None]*Backup.att_number
file_attributes_time = [None]*Backup.att_number
for xattr in xattr_list:
for att in range(Backup.att_number):
att_sum_cur = Backup.att_sum + '_' + str(att)
att_time_cur = Backup.att_time + '_' + str(att)
if xattr == att_sum_cur:
raw_attribute = os.getxattr(file, att_sum_cur)
file_attributes_sum[att] = \
raw_attribute.decode(encoding='utf-8')
elif xattr == att_time_cur:
raw_attribute = os.getxattr(file, att_time_cur)
file_attributes_time[att] = \
raw_attribute.decode(encoding='utf-8')
else:
pass
sum_most_common, error_level = \
Backup.get_most_common_att(file_attributes_sum)
time_most_common, error_level = \
Backup.get_most_common_att(file_attributes_time)
return (sum_most_common, time_most_common, error_level)
def test_removexattr(setup_unreliablefs, symlink):
mnt_dir, src_dir = setup_unreliablefs
name = name_generator()
src_name = pjoin(src_dir, name)
mnt_name = pjoin(mnt_dir, name)
os_create(mnt_name)
linkname = name_generator()
link_path = os.path.join(mnt_dir, linkname)
os.symlink(mnt_dir, link_path)
if symlink:
target = link_path
else:
target = mnt_name
attr_value = b"unreliablefs"
attr_name = b"user.fsname"
os.setxattr(target, attr_name, attr_value)
assert attr_name.decode("utf-8") in os.listxattr(target)
assert os.getxattr(target, attr_name) == attr_value
os.removexattr(target, attr_name)
try:
assert os.getxattr(target, attr_name) == None
except OSError:
pass
def test_get_sha1_xattr_succeeds_after_querying_xattr_on_dir(self) -> None:
with self.assertRaises(OSError):
os.getxattr(self.mount, "does_not_exist")
filename = os.path.join(self.mount, "hello")
xattr = os.getxattr(filename, "user.sha1")
contents = open(filename, "rb").read()
expected_sha1 = sha1(contents)
self.assertEqual(expected_sha1, xattr)
def getxattr(self, path, name, *args):
logging.info("API: getxattr " + path + ", " + str(name) + ", " +
str(args))
if path == '/' or path.split('/')[-1].strip('.') in self._tags():
if set(dir_tags(path)).issubset(set(self._tags())):
return os.getxattr(self.store, name, *args)
else:
raise FuseOSError(errno.ENOENT)
if not self._consistent_file_path(path):
logging.debug(path + " deemed inconsistent")
raise FuseOSError(errno.ENOENT)
return os.getxattr(self._store_path(path), name, *args)
def update_permissions(path: Path, valid_users: T.Set[str]) -> T.Set[str]:
ret = set(valid_users)
for attr in os.listxattr(path):
if attr == "user.access":
ret = set(os.getxattr(str(path), attr).decode().split(":"))
elif attr == "user.access_add":
ret = ret.union(os.getxattr(str(path), attr).decode().split(":"))
elif attr == "user.access_del":
ret = ret.difference(
os.getxattr(str(path), attr).decode().split(":")
)
return ret
def _copyxattr(src, dst, *, follow_symlinks=True):
"""Copy extended filesystem attributes from `src` to `dst`.
Overwrite existing attributes.
If `follow_symlinks` is false, symlinks won't be followed.
"""
try:
names = os.listxattr(src, follow_symlinks=follow_symlinks)
except OSError as e:
if e.errno not in (errno.ENOTSUP, errno.ENODATA):
raise
return
for name in names:
if name == 'system.wsl_case_sensitive':
# 这个属性是在 WSL Linux 下访问 windows 下的目录会有这个扩展属性
# 在 WSL Linux 下的目录也没有这个属性
# https://blogs.msdn.microsoft.com/commandline/2018/06/14/improved-per-directory-case-sensitivity-support-in-wsl/
continue
try:
value = os.getxattr(src, name, follow_symlinks=follow_symlinks)
os.setxattr(dst, name, value, follow_symlinks=follow_symlinks)
except OSError as e:
if e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA):
raise
def checkUbridgePermission(self):
"""Check if ubridge has the correct permission"""
if not sys.platform.startswith("win") and os.geteuid() == 0:
# we are root, so we should have privileged access.
return (0, None)
path = LocalServer.instance().localServerSettings().get("ubridge_path")
if path is None:
return (0, None)
if not os.path.exists(path):
return (2, "Ubridge path {path} doesn't exists".format(path=path))
request_setuid = False
if sys.platform.startswith("linux"):
try:
if "security.capability" in os.listxattr(path):
caps = os.getxattr(path, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if not struct.unpack("<IIIII", caps)[1] & 1 << 13:
return (2, "Ubridge requires CAP_NET_RAW. Run sudo setcap cap_net_admin,cap_net_raw=ep {path}".format(path=path))
else:
# capabilities not supported
request_setuid = True
except AttributeError:
# Due to a Python bug, os.listxattr could be missing: https://github.com/GNS3/gns3-gui/issues/2010
return (1, "Could not determine if CAP_NET_RAW capability is set for uBridge (Python bug)".format(path=path))
if sys.platform.startswith("darwin") or request_setuid:
if os.stat(path).st_uid != 0 or not os.stat(path).st_mode & stat.S_ISUID:
return (2, "Ubridge should be setuid. Run sudo chown root:admin {path} and sudo chmod 4750 {path}".format(path=path))
return (0, None)
def process_file(brick_path, path):
if args.debug:
print(f"Processing file: {path}")
# Try to get a string representation of the xattr from hex bytes. We need
# to try/except because os.getxattr() returns FileNotFoundError when the
# file is a broken symlink, and OSError when the link goes somewhere that
# does not have GlusterFS metadata. This happens in the case of eg Python
# virtual environments in user's home directories and it is not a problem
# to remove it and return without trying to remove a .glusterfs hard link.
try:
file_gfid = os.getxattr(path, "trusted.gfid").hex()
except (FileNotFoundError, OSError):
if not args.dry_run:
os.remove(path)
print(f'{"(DRY RUN) " if args.dry_run else ""}Removed symlink: {path}')
return
file_glusterfs_path = dot_glusterfs_path(brick_path, file_gfid)
if Path(path).exists():
if not args.dry_run:
os.remove(path)
print(f'{"(DRY RUN) " if args.dry_run else ""}Removed file: {path}')
if Path(file_glusterfs_path).exists():
if not args.dry_run:
os.remove(file_glusterfs_path)
print(
f'{"(DRY RUN) " if args.dry_run else ""}Removed file hardlink: {file_glusterfs_path}'
)
def _copyxattr(self, dst, *, follow_symlinks=True):
"""Copy extended filesystem attributes from `self.source_path` to `dst`.
Overwrite existing attributes.
If `follow_symlinks` is false, symlinks won't be followed.
"""
try:
names = os.listxattr(self.source_path,
follow_symlinks=follow_symlinks)
except OSError as e:
if e.errno not in (errno.ENOTSUP, errno.ENODATA, errno.EINVAL):
raise
return
for name in names:
try:
value = os.getxattr(self.source_path,
name,
follow_symlinks=follow_symlinks)
os.setxattr(dst,
name,
value,
follow_symlinks=follow_symlinks)
except OSError as e:
if e.errno not in (errno.EPERM, errno.ENOTSUP,
errno.ENODATA, errno.EINVAL):
raise
def getxattr(self, path, name, position=0):
full_path = self._full_path(path)
print("getxattr "+str(self)+" path="+full_path+" name="+name)
if os.path.isfile(full_path):
if name == "user.owncloud.virtual":
return b"maybe"
return os.getxattr(full_path, name)
def _copyxattr(src, dest, exclude=None):
try:
attrs = _os.listxattr(src)
except OSError as e:
if e.errno != OperationNotSupported.errno:
raise
attrs = ()
if attrs:
if exclude is not None and isinstance(attrs[0], bytes):
exclude = exclude.encode(_encodings['fs'])
exclude = _get_xattr_excluder(exclude)
for attr in attrs:
if exclude(attr):
continue
try:
_os.setxattr(dest, attr, _os.getxattr(src, attr))
raise_exception = False
except OSError:
raise_exception = True
if raise_exception:
raise OperationNotSupported(_("Filesystem containing file '%s' "
"does not support extended attribute '%s'") %
(_unicode_decode(dest), _unicode_decode(attr)))
def _has_privileged_access(executable):
"""
Check if an executable can access Ethernet and TAP devices in
RAW mode.
:param executable: executable path
:returns: True or False
"""
if sys.platform.startswith("win"):
# do not check anything on Windows
return True
if os.geteuid() == 0:
# we are root, so we should have privileged access.
return True
if os.stat(executable).st_mode & stat.S_ISUID or os.stat(executable).st_mode & stat.S_ISGID:
# the executable has set UID bit.
return True
# test if the executable has the CAP_NET_RAW capability (Linux only)
if sys.platform.startswith("linux") and "security.capability" in os.listxattr(executable):
try:
caps = os.getxattr(executable, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if struct.unpack("<IIIII", caps)[1] & 1 << 13:
return True
except Exception as e:
log.error("could not determine if CAP_NET_RAW capability is set for {}: {}".format(executable, e))
return False
def verify(self, target_dir):
""""""
with pushd(target_dir):
for f in self.source_files.keys():
fp = os.path.join(target_dir, f)
attrs = os.listxattr(path=fp, follow_symlinks=False)
assert len(attrs) == self.xattr_pairs
for k in self.source_xattrs[f].keys():
v = os.getxattr(fp, k.encode(self.encoding)).decode(
self.encoding)
assert v == self.source_xattrs[f][k]
attrs = os.listxattr(fp, follow_symlinks=False)
if self.encoding != "gb2312":
for attr in attrs:
v = xattr.getxattr(f, attr)
assert attr in self.source_xattrs[f].keys()
assert v.decode(
self.encoding) == self.source_xattrs[f][attr]
with pushd(target_dir):
for d in self.source_dirs.keys():
dp = os.path.join(target_dir, d)
attrs = xattr.listxattr(dp)
assert len(attrs) == self.xattr_pairs
for attr in attrs:
v = xattr.getxattr(d, attr)
assert attr in self.source_dirs_xattrs[d].keys()
assert v.decode(
self.encoding) == self.source_dirs_xattrs[d][attr]
def getxattr(self, path, name, position=0):
# getxattr called multiple times for security.capability
# always precedes a write operation
# from https://sourceforge.net/p/fuse/mailman/fuse-devel/thread/0DEBAD03-97FD-4A55-B953-75964DEBE226%40caringo.com/
# disable CONFIG_SECURITY_FILE_CAPABILITIES in kernel configuration to avoid this.
p = self.realpath(path)
return os.getxattr(p, name)
def _copyxattr(src, dst, *, follow_symlinks=True):
"""Copy extended filesystem attributes from `src` to `dst`.
Overwrite existing attributes.
If `follow_symlinks` is false, symlinks won't be followed.
"""
try:
names = os.listxattr(src, follow_symlinks=follow_symlinks)
except OSError as e:
try:
if e.errno not in (errno.ENOTSUP, errno.ENODATA):
raise
return
finally:
e = None
del e
for name in names:
try:
value = os.getxattr(src, name, follow_symlinks=follow_symlinks)
os.setxattr(dst, name, value, follow_symlinks=follow_symlinks)
except OSError as e:
try:
if e.errno not in (errno.EPERM, errno.ENOTSUP,
errno.ENODATA):
raise
finally:
e = None
del e
def _checkUbridgePermissions(self):
"""
Checks that uBridge can interact with network interfaces.
"""
path = self._settings["local_server"]["ubridge_path"]
if not path or len(path) == 0 or not os.path.exists(path):
return False
if sys.platform.startswith("win"):
# do not check anything on Windows
return True
if os.geteuid() == 0:
# we are root, so we should have privileged access.
return True
from .main_window import MainWindow
main_window = MainWindow.instance()
request_setuid = False
if sys.platform.startswith("linux"):
# test if the executable has the CAP_NET_RAW capability (Linux only)
try:
if "security.capability" in os.listxattr(path):
caps = os.getxattr(path, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if not struct.unpack("<IIIII", caps)[1] & 1 << 13:
proceed = QtWidgets.QMessageBox.question(
main_window,
"uBridge",
"uBridge requires CAP_NET_RAW capability to interact with network interfaces. Set the capability to uBridge?",
QtWidgets.QMessageBox.Yes,
QtWidgets.QMessageBox.No)
if proceed == QtWidgets.QMessageBox.Yes:
sudo(["setcap", "cap_net_admin,cap_net_raw=ep"])
else:
# capabilities not supported
request_setuid = True
except OSError as e:
QtWidgets.QMessageBox.critical(main_window, "uBridge", "Can't set CAP_NET_RAW capability to uBridge {}: {}".format(path, str(e)))
return False
if sys.platform.startswith("darwin") or request_setuid:
try:
if os.stat(path).st_uid != 0 or not os.stat(path).st_mode & stat.S_ISUID:
proceed = QtWidgets.QMessageBox.question(
main_window,
"uBridge",
"uBridge requires root permissions to interact with network interfaces. Set root permissions to uBridge?",
QtWidgets.QMessageBox.Yes,
QtWidgets.QMessageBox.No)
if proceed == QtWidgets.QMessageBox.Yes:
sudo(["chmod", "4755", path])
sudo(["chown", "root", path])
except OSError as e:
QtWidgets.QMessageBox.critical(main_window, "uBridge", "Can't set root permissions to uBridge {}: {}".format(path, str(e)))
return False
return True
def test_upload(self):
self.gen_files(lock=False)
self.start_thread()
# allow enough time that files could upload twice if there is a bug in the logic
time.sleep(5)
self.join_thread()
exp_order = self.gen_order([self.seg_num], [])
self.assertTrue(
len(log_handler.upload_ignored) == 0, "Some files were ignored")
self.assertFalse(
len(log_handler.upload_order) < len(exp_order),
"Some files failed to upload")
self.assertFalse(
len(log_handler.upload_order) > len(exp_order),
"Some files were uploaded twice")
for f_path in exp_order:
self.assertTrue(
os.getxattr(
os.path.join(self.root, f_path.replace('.bz2', '')),
uploader.UPLOAD_ATTR_NAME), "All files not uploaded")
self.assertTrue(log_handler.upload_order == exp_order,
"Files uploaded in wrong order")
def _copyxattr(src, dest, exclude=None):
try:
attrs = _os.listxattr(src)
except OSError as e:
if e.errno != OperationNotSupported.errno:
raise
attrs = ()
if attrs:
if exclude is not None and isinstance(attrs[0], bytes):
exclude = exclude.encode(_encodings['fs'])
exclude = _get_xattr_excluder(exclude)
for attr in attrs:
if exclude(attr):
continue
try:
_os.setxattr(dest, attr, _os.getxattr(src, attr))
raise_exception = False
except OSError:
raise_exception = True
if raise_exception:
raise OperationNotSupported(
_("Filesystem containing file '%s' "
"does not support extended attribute '%s'") %
(_unicode_decode(dest), _unicode_decode(attr)))
def _get_xattrs(fd, follow_symlinks=True):
ret = []
for name in os.listxattr(fd, follow_symlinks=follow_symlinks):
name = name.encode()
ret.append(
(name, os.getxattr(fd, name, follow_symlinks=follow_symlinks)))
return ret
class LocalClient(LocalClientMixin):
"""GNU/Linux client API implementation for the local file system."""
shared_icons = Path.home() / ".local/share/icons"
def has_folder_icon(self, ref: Path, /) -> bool:
"""Check if the folder icon is set."""
emblem = self.shared_icons / "emblem-nuxeo.svg"
if not emblem.is_file():
return False
folder = self.abspath(ref)
cmd = ["gio", "info", "-a", "metadata", str(folder)]
try:
output = subprocess.check_output(cmd, encoding="utf-8")
except Exception:
log.warning(f"Could not check the metadata of {folder!r}",
exc_info=True)
return False
matcher = re.compile(r"metadata::emblems: \[.*emblem-nuxeo.*\]")
return bool(matcher.findall(output))
@staticmethod
def get_path_remote_id(path: Path, /, *, name: str = "ndrive") -> str:
"""Get a given extended attribute from a file/folder."""
try:
return (os.getxattr(path, f"user.{name}").decode(
"utf-8", errors="ignore") # type: ignore
or "")
except OSError:
return ""
def checkUbridgePermission(self):
"""Check if ubridge has the correct permission"""
if not sys.platform.startswith("win") and os.geteuid() == 0:
# we are root, so we should have privileged access.
return (0, None)
path = Servers.instance().localServerSettings().get("ubridge_path")
if path is None:
return (0, None)
if not os.path.exists(path):
return (2, "Ubridge path {path} doesn't exists".format(path=path))
request_setuid = False
if sys.platform.startswith("linux"):
if "security.capability" in os.listxattr(path):
caps = os.getxattr(path, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if not struct.unpack("<IIIII", caps)[1] & 1 << 13:
return(2, "Ubridge require CAP_NET_RAW. Run sudo setcap cap_net_admin,cap_net_raw=ep {path}".format(path=path))
else:
# capabilities not supported
request_setuid = True
if sys.platform.startswith("darwin") or request_setuid:
if os.stat(path).st_uid != 0 or not os.stat(path).st_mode & stat.S_ISUID:
return (2, "Ubridge should be setuid. Run sudo chown root {path} and sudo chmod 4755 {path}".format(path=path))
return (0, None)
def getxattr(_endpoint, filename, _ruid, _rgid, key):
'''Get the extended attribute <key> on behalf of the given uid, gid. Do not raise exceptions'''
try:
return os.getxattr(_getfilename(filename), key)
except (FileNotFoundError, PermissionError) as e:
log.warning('msg="Failed to getxattr" filename="%s" key="%s" exception="%s"' % (filename, key, e))
return None
def checkDynamipsPermission(self):
"""Check if dynamips has the correct permission"""
if not sys.platform.startswith("win") and os.geteuid() == 0:
# we are root, so we should have privileged access.
return (0, None)
path = LocalServer.instance().localServerSettings().get(
"dynamips_path")
if path is None:
return (0, None)
if not os.path.exists(path):
return (2, "Dynamips path {path} doesn't exists".format(path=path))
try:
if sys.platform.startswith(
"linux") and "security.capability" in os.listxattr(path):
caps = os.getxattr(path, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if not struct.unpack("<IIIII", caps)[1] & 1 << 13:
return (
2,
"Dynamips requires CAP_NET_RAW. Run sudo setcap cap_net_raw,cap_net_admin+eip {path}"
.format(path=path))
except AttributeError:
# Due to a Python bug, os.listxattr could be missing: https://github.com/GNS3/gns3-gui/issues/2010
return (
1,
"Could not determine if CAP_NET_RAW capability is set for Dynamips (Python bug)"
.format(path=path))
return (0, None)
def _checkUbridgePermissions(self):
"""
Checks that uBridge can interact with network interfaces.
"""
path = os.path.abspath(self._settings["ubridge_path"])
if not path or len(path) == 0 or not os.path.exists(path):
return False
if sys.platform.startswith("win"):
# do not check anything on Windows
return True
if os.geteuid() == 0:
# we are root, so we should have privileged access.
return True
request_setuid = False
if sys.platform.startswith("linux"):
# test if the executable has the CAP_NET_RAW capability (Linux only)
try:
if "security.capability" in os.listxattr(path):
caps = os.getxattr(path, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if not struct.unpack("<IIIII", caps)[1] & 1 << 13:
proceed = QtWidgets.QMessageBox.question(
self.parent(),
"uBridge",
"uBridge requires CAP_NET_RAW capability to interact with network interfaces. Set the capability to uBridge? All users on the system will be able to read packet from the network interfaces.",
QtWidgets.QMessageBox.Yes,
QtWidgets.QMessageBox.No)
if proceed == QtWidgets.QMessageBox.Yes:
sudo(["setcap", "cap_net_admin,cap_net_raw=ep"])
else:
# capabilities not supported
request_setuid = True
except AttributeError:
# Due to a Python bug, os.listxattr could be missing: https://github.com/GNS3/gns3-gui/issues/2010
log.warning("Could not determine if CAP_NET_RAW capability is set for uBridge (Python bug)")
return True
except OSError as e:
QtWidgets.QMessageBox.critical(self.parent(), "uBridge", "Can't set CAP_NET_RAW capability to uBridge {}: {}".format(path, str(e)))
return False
if sys.platform.startswith("darwin") or request_setuid:
try:
if os.stat(path).st_uid != 0 or not os.stat(path).st_mode & stat.S_ISUID:
proceed = QtWidgets.QMessageBox.question(
self.parent(),
"uBridge",
"uBridge requires root permissions to interact with network interfaces. Set root permissions to uBridge? All admin users on the system will be able to read packet from the network interfaces.",
QtWidgets.QMessageBox.Yes,
QtWidgets.QMessageBox.No)
if proceed == QtWidgets.QMessageBox.Yes:
sudo(["chown", "root:admin", path], ["chmod", "4750", path])
except OSError as e:
QtWidgets.QMessageBox.critical(self.parent(), "uBridge", "Can't set root permissions to uBridge {}: {}".format(path, str(e)))
return False
return True
def test_upload_files_in_create_order(self):
seg1_nums = [0, 1, 2, 10, 20]
for i in seg1_nums:
self.seg_dir = self.seg_format.format(i)
self.gen_files(boot=False)
seg2_nums = [5, 50, 51]
for i in seg2_nums:
self.seg_dir = self.seg_format2.format(i)
self.gen_files(boot=False)
exp_order = self.gen_order(seg1_nums, seg2_nums, boot=False)
self.start_thread()
# allow enough time that files could upload twice if there is a bug in the logic
time.sleep(5)
self.join_thread()
self.assertTrue(
len(log_handler.upload_ignored) == 0, "Some files were ignored")
self.assertFalse(
len(log_handler.upload_order) < len(exp_order),
"Some files failed to upload")
self.assertFalse(
len(log_handler.upload_order) > len(exp_order),
"Some files were uploaded twice")
for f_path in exp_order:
self.assertTrue(
os.getxattr(
os.path.join(self.root, f_path.replace('.bz2', '')),
uploader.UPLOAD_ATTR_NAME), "All files not uploaded")
self.assertTrue(log_handler.upload_order == exp_order,
"Files uploaded in wrong order")
def has_privileged_access(executable):
"""
Check if an executable can access Ethernet and TAP devices in
RAW mode.
:param executable: executable path
:returns: True or False
"""
if sys.platform.startswith("win"):
# do not check anything on Windows
return True
if os.geteuid() == 0:
# we are root, so we should have privileged access.
return True
if os.stat(executable).st_mode & stat.S_ISVTX == stat.S_ISVTX:
# the executable has a sticky bit.
return True
# test if the executable has the CAP_NET_RAW capability (Linux only)
if sys.platform.startswith(
"linux") and "security.capability" in os.listxattr(executable):
try:
caps = os.getxattr(executable, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if struct.unpack("<IIIII", caps)[1] & 1 << 13:
return True
except Exception as e:
log.error(
"could not determine if CAP_NET_RAW capability is set for {}: {}"
.format(executable, e))
return False
def get_path_remote_id(path: Path, name: str = "ndrive") -> str:
"""Get a given extended attribute from a file/folder."""
try:
return (os.getxattr(path, f"user.{name}").decode(
"utf-8", errors="ignore") # type: ignore
or "")
except OSError:
return ""
def GetMaxRevisionAge (fname):
s_max_revision_age = os.getxattr (fname, xattr_max_revision_age, follow_symlinks=False)
max_revision_age = 185
try:
max_revision_age = int (s_max_revision_age)
except ValueError:
pass
return max_revision_age
def GetMinRevisionsAge (fname):
s_min_revisions_age = os.getxattr (fname, xattr_min_revisions_age, follow_symlinks=False)
min_revisions_age = 1
try:
min_revisions_age = int (s_min_revisions_age)
except ValueError:
pass
return min_revisions_age
def GetMaxRevisions (fname):
s_max_revisions = os.getxattr (fname, xattr_max_revisions_name, follow_symlinks=False)
max_revisions = 10
try:
max_revisions = int (s_max_revisions)
except ValueError:
pass
return max_revisions
def load_id_from_metadata(path) -> Optional[str]:
path = Path(path)
try:
return os.getxattr(str(path), XATTR_ONEDRIVE_ID).decode()
except OSError as error:
if error.errno != errno.ENODATA:
raise
else:
return None
def get_metadata_from_file(path, attr_key):
"""Get metadata from key attr_key in file at given path."""
try:
mdbytes = os.getxattr(path, attr_key)
except OSError as e:
if e.errno == errno.ENODATA:
return None
raise
return os.fsdecode(mdbytes)
def assertNoCaps(self, path): try: caps = os.getxattr(path, "security.capability") if caps: self.printError(path, "has capabilities despite --no-fscaps") except OSError as e: if e.errno != errno.ENODATA: raise
def _copyxattr(src, dest):
for attr in _os.listxattr(src):
try:
_os.setxattr(dest, attr, _os.getxattr(src, attr))
raise_exception = False
except OSError:
raise_exception = True
if raise_exception:
raise OperationNotSupported("Filesystem containing file '%s' does not support extended attributes" % dest)
async def getxattr(self, vnode, name_enced, ctx):
name = os.fsdecode(name_enced)
vinfo = self.vm[vnode]
if vinfo.virtual:
raise pyfuse3.FUSEError(errno.ENODATA) # No data available
else:
try:
return os.getxattr(vinfo.path, name)
except OSError as exc:
raise pyfuse3.FUSEError(exc.errno)
def copyxattr(src, dest):
"""
Copy the extended attributes (xattr) from `src` to `dest`.
NOTE: xattr only available on Linux.
"""
if not hasattr(os, "listxattr"):
return
for name in os.listxattr(src):
value = os.getxattr(src, name)
os.setxattr(dest, name, value)
def get_file_attr(path, attr):
try:
if hasattr(path, "fileno"):
path = path.fileno()
value = os.getxattr(path, "user.%s" % attr)
try:
return value.decode("utf-8")
except UnicodeDecodeError:
return value
except FileNotFoundError:
raise
except OSError:
return None
def patch_node(p):
# getxattr/setxattr work weirdly for broken symlinks
# glusterd produces these sometimes, seem to be expendable
try: bugged = p.is_symlink() and not p.exists()
except OSError: bugged = True
if bugged:
if not dry_run: p.unlink()
return
for k in os.listxattr(p, follow_symlinks=False):
if not k.startswith('trusted.'): continue
k_user = '******'.format(k[8:])
v = os.getxattr(p, k, follow_symlinks=False)
log.debug(f'patch: {p} :: {k} -> {k_user} [{v!r}]')
if not dry_run: os.setxattr(p, k_user, v, follow_symlinks=False)
def _copyxattr(src, dst, *, follow_symlinks=True):
try:
names = os.listxattr(src, follow_symlinks=follow_symlinks)
except OSError as e:
if e.errno not in (errno.ENOTSUP, errno.ENODATA):
raise
return
for name in names:
try:
value = os.getxattr(src, name, follow_symlinks=follow_symlinks)
os.setxattr(dst, name, value, follow_symlinks=follow_symlinks)
except OSError as e:
while e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA):
raise
def _scan(self):
for image in self.runtime.iterdir():
name = image.name
self.images[name] = attrs = {}
if not image.exists():
continue
for attr in os.listxattr(str(image)):
if not attr.startswith("user."):
continue
val = os.getxattr(str(image), attr)
val = val.decode()
key = attr[5:]
attrs[key] = val
def _copyxattr(src, dst, *, follow_symlinks=True):
"""Copy extended filesystem attributes from `src` to `dst`.
Overwrite existing attributes.
If `follow_symlinks` is false, symlinks won't be followed.
"""
for name in os.listxattr(src, follow_symlinks=follow_symlinks):
try:
value = os.getxattr(src, name, follow_symlinks=follow_symlinks)
os.setxattr(dst, name, value, follow_symlinks=follow_symlinks)
except OSError as e:
if e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA):
raise
def scan(self):
if exists("save.p"):
self.cache = pickle.load(open( "save.p", "rb" ))
for root, dirs, files in os.walk('/home/mikael/Music/Ablaze my sorrow'):
for name in files:
if name.startswith('.'):
continue
with open(join(root, name),'rb') as fp:
print(fp.name)
#if 'user.sha256' not in attrs:
# h = hashlib.sha256(fp.read())
# sum = h.digest()
# attrs['user.sha256'] = sum
modified = os.stat(fp.fileno()).st_mtime
if fp.name in self.cache:
if 'modified' in self.cache[fp.name]:
print(modified, self.cache[fp.name]['modified'])
if modified <= self.cache[fp.name]['modified']:
print("Optimized!")
continue
try:
rid = os.getxattr(fp.fileno(),'user.musicbrainz.recordingid').decode('utf-8')
except Exception as e:
rid = False
# #import pdb;pdb.set_trace()
if rid:
self.cache["mbid:"+rid] = fp.name
self.cache[fp.name] = {}
self.cache[fp.name]['modified'] = modified
continue
else:
try:
metadata = mutagen.File(fp.name,easy=True)
except Exception as e:
metadata = False
if not metadata:
print("Missing!")
continue
if 'musicbrainz_trackid' in metadata:
os.setxattr(fp.fileno(),'user.musicbrainz.recordingid',metadata['musicbrainz_trackid'][0].encode('utf-8'))
self.cache[metadata['musicbrainz_trackid'][0]] = fp.name
if not fp.name in self.cache:
self.cache[fp.name] = {}
self.cache[fp.name]['modified'] = int(time.time())
#import pdb;pdb.set_trace()
for key in cache:
self.dht.set(key,"http://185.97.32.250:8468/by-mbid/"+key)
def checkDynamipsPermission(self):
"""Check if dynamips has the correct permission"""
if not sys.platform.startswith("win") and os.geteuid() == 0:
# we are root, so we should have privileged access.
return (0, None)
path = Servers.instance().localServerSettings().get("dynamips_path")
if path is None:
return (0, None)
if not os.path.exists(path):
return (2, "Dynamips path {path} doesn't exists".format(path=path))
if sys.platform.startswith("linux") and "security.capability" in os.listxattr(path):
caps = os.getxattr(path, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if not struct.unpack("<IIIII", caps)[1] & 1 << 13:
return(2, "Dynamips require CAP_NET_RAW. Run sudo setcap cap_net_raw,cap_net_admin+eip {path}".format(path=path))
return (0, None)
def _getxattr_helper(path, name):
try:
value = llfuse.getxattr(path, name)
except OSError as exc:
errno = exc.errno
value = None
if not hasattr(os, 'getxattr'):
return value
try:
value2 = os.getxattr(path, name)
except OSError as exc:
assert exc.errno == errno
else:
assert value2 is not None
assert value2 == value
return value
def checkDynamipsPermission(self):
"""Check if dynamips has the correct permission"""
if not sys.platform.startswith("win") and os.geteuid() == 0:
# we are root, so we should have privileged access.
return (0, None)
path = LocalServer.instance().localServerSettings().get("dynamips_path")
if path is None:
return (0, None)
if not os.path.exists(path):
return (2, "Dynamips path {path} doesn't exists".format(path=path))
try:
if sys.platform.startswith("linux") and "security.capability" in os.listxattr(path):
caps = os.getxattr(path, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if not struct.unpack("<IIIII", caps)[1] & 1 << 13:
return (2, "Dynamips requires CAP_NET_RAW. Run sudo setcap cap_net_raw,cap_net_admin+eip {path}".format(path=path))
except AttributeError:
# Due to a Python bug, os.listxattr could be missing: https://github.com/GNS3/gns3-gui/issues/2010
return (1, "Could not determine if CAP_NET_RAW capability is set for Dynamips (Python bug)".format(path=path))
return (0, None)
def _check_for_privileged_access(self, device):
"""
Check if iouyap can access Ethernet and TAP devices.
:param device: device name
"""
# we are root, so iouyap should have privileged access too
if os.geteuid() == 0:
return
# test if iouyap has the CAP_NET_RAW capability
if "security.capability" in os.listxattr(self._iouyap):
try:
caps = os.getxattr(self._iouyap, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if struct.unpack("<IIIII", caps)[1] & 1 << 13:
return
except Exception as e:
log.error("could not determine if CAP_NET_RAW capability is set for {}: {}".format(self._iouyap, e))
return
raise IOUError("{} has no privileged access to {}.".format(self._iouyap, device))
def get(item, name, nofollow=False, namespace=None): return os.getxattr(item, name, follow_symlinks=not nofollow)
def build(args):
context = Path(args.path).resolve()
dockerfile = Path(args.file)
if not dockerfile.is_absolute():
dockerfile = context / dockerfile
dockerfile = dockerfile.resolve()
if not dockerfile.is_file():
raise FileNotFoundError("{} does not exist".format(dockerfile))
runtime = Path(args.runtime).resolve()
r = ImageStorage(runtime)
df = DockerfileParser(dockerfile)
if not df.build_commands:
print("Nothing to do")
return
if args.env:
df.add_env_variables(args.env)
# Locate base image for this dockerfile
parent_hash = ""
if df.from_image != "scratch":
parent_hash = r.find_last_build_by_name(df.from_image)
if not parent_hash:
raise FileNotFoundError("Image with name {} not found".format(df.from_image))
print("Using parent image {}".format(parent_hash[:16]))
# Update build hashes based on base image
df.calc_build_hashes(parent_hash=parent_hash)
total_build_steps = len(df.build_commands)
# Early exit if image is already built
if not args.no_cache and args.rm:
if (runtime / df.build_hashes[-1]).exists():
print("==> Already built {}".format(df.build_hashes[-1][:16]))
return
# Do the build
for current_build_step, (cmd, cmdargs) in enumerate(df.build_commands):
build_step_hash = df.build_hashes[current_build_step]
print("==> Building step {}/{} {}".format(current_build_step + 1, total_build_steps, build_step_hash[:16]))
target = runtime / (build_step_hash + "-init")
final_target = runtime / build_step_hash
host_env = {
"TARGET": str(target),
"CONTEXT": str(context)
}
host_env.update(df.env)
## parent image checks
if final_target.exists():
if args.no_cache:
btrfs_subvol_delete(final_target)
else:
previous_parent_hash = ""
try:
previous_parent_hash = os.getxattr(str(final_target), b"user.parent_hash").decode()
except:
pass
if parent_hash and parent_hash != previous_parent_hash:
print(" -> parent image hash changed")
btrfs_subvol_delete(final_target)
else:
print(" -> Using cached image")
parent_hash = build_step_hash
continue
if target.exists():
print(" -> Deleting incomplete image")
btrfs_subvol_delete(target)
if parent_hash:
btrfs_subvol_snapshot(runtime / parent_hash, target)
else:
btrfs_subvol_create(target)
## Run build step
if cmd == "host":
print(' -> HOST {}'.format(cmdargs))
subprocess.run(cmdargs, cwd=str(context), check=True, shell=True, env=host_env)
elif cmd == "run":
print(' -> RUN {}'.format(cmdargs))
nspawn_cmd = ['systemd-nspawn', '--quiet']
for key, val in df.env.items():
nspawn_cmd.extend(('--setenv', '{}={}'.format(key, val)))
nspawn_cmd.extend(('--register=no', '-D', str(target), '/bin/sh', '-c', cmdargs))
subprocess.run(nspawn_cmd, cwd=str(target), check=True, shell=False, env=df.env)
elif cmd == "copy":
print(" -> COPY {}".format(cmdargs))
*srcs, dest = shlex.split(cmdargs)
if Path(dest).is_absolute():
dest = target / dest[1:]
else:
dest = target / dest
if len(srcs) > 1 and not dest.is_dir():
raise NotADirectoryError("Destination must be a directory")
cmd = ['cp', '-rv']
cmd.extend(srcs)
cmd.append(str(dest))
subprocess.run(cmd, cwd=str(context), check=True, shell=False, env=host_env)
## Seal build image
os.setxattr(str(target), b"user.parent_hash", parent_hash.encode())
for attr in ("user.cmd.host", "user.cmd.run"):
try:
os.removexattr(str(target), attr.encode())
except:
pass
os.setxattr(str(target), "user.cmd.{}".format(cmd).encode(), cmdargs.encode())
btrfs_subvol_snapshot(target, final_target, readonly=True)
btrfs_subvol_delete(target)
parent_hash = build_step_hash
# After build cleanup
if args.rm:
print("==> Cleanup")
for build_hash in df.build_hashes[:-1]:
target = runtime / build_hash
if target.exists():
print(" -> Remove intermediate image {}".format(build_hash[:16]))
btrfs_subvol_delete(target)
print("==> Successfully built {}".format(parent_hash[:16]))
if args.tag:
tmp_tag = runtime / ("tag-" + args.tag + "-tmp")
if tmp_tag.exists():
os.unlink(str(tmp_tag))
os.symlink(str(runtime / parent_hash), str(tmp_tag))
os.replace(str(tmp_tag), str(runtime / ("tag-" + args.tag)))
print("==> Tagged image {} as {}".format(parent_hash[:16], args.tag))
def copy_all(self):
"""Core copy process. This is the most important step of this
stage. It clones live filesystem into a local partition in the
selected hard disk."""
self.db.progress('START', 0, 100, 'ubiquity/install/title')
self.db.progress('INFO', 'ubiquity/install/copying')
fs_size = os.path.join(self.casper_path, 'filesystem.size')
if os.path.exists(fs_size):
with open(fs_size) as total_size_fp:
total_size = int(total_size_fp.readline())
else:
# Fallback in case an Linux Mint derivative forgets to put
# /casper/filesystem.size on the CD, or to account for things
# like CD->USB transformation tools that don't copy this file.
# This is slower than just reading the size from a file, but
# better than crashing.
#
# Obviously doing os.walk() twice is inefficient, but I'd rather
# not suck the list into ubiquity's memory, and I'm guessing
# that the kernel's dentry cache will avoid most of the slowness
# anyway.
total_size = 0
for dirpath, dirnames, filenames in os.walk(self.source):
for name in dirnames + filenames:
fqpath = os.path.join(dirpath, name)
total_size += os.lstat(fqpath).st_size
# Progress bar handling:
# We sample progress every half-second (assuming time.time() gives
# us sufficiently good granularity) and use the average of progress
# over the last minute or so to decide how much time remains. We
# don't bother displaying any progress for the first ten seconds in
# order to allow things to settle down, and we only update the "time
# remaining" indicator at most every two seconds after that.
copy_progress = 0
copied_size = 0
directory_times = []
time_start = time.time()
times = [(time_start, copied_size)]
long_enough = False
time_last_update = time_start
debug = 'UBIQUITY_DEBUG' in os.environ
if self.db.get('ubiquity/install/md5_check') == 'false':
md5_check = False
else:
md5_check = True
# Increase kernel flush times during bulk data copying to make it
# more likely that small files are packed contiguously, which should
# speed up initial boot times.
dirty_writeback_centisecs = None
dirty_expire_centisecs = None
if os.path.exists('/proc/sys/vm/dirty_writeback_centisecs'):
with open('/proc/sys/vm/dirty_writeback_centisecs') as dwc:
dirty_writeback_centisecs = int(dwc.readline())
with open('/proc/sys/vm/dirty_writeback_centisecs', 'w') as dwc:
print('3000\n', file=dwc)
if os.path.exists('/proc/sys/vm/dirty_expire_centisecs'):
with open('/proc/sys/vm/dirty_expire_centisecs') as dec:
dirty_expire_centisecs = int(dec.readline())
with open('/proc/sys/vm/dirty_expire_centisecs', 'w') as dec:
print('6000\n', file=dec)
old_umask = os.umask(0)
for dirpath, dirnames, filenames in os.walk(self.source):
sp = dirpath[len(self.source) + 1:]
for name in dirnames + filenames:
relpath = os.path.join(sp, name)
# /etc/fstab was legitimately created by partman, and
# shouldn't be copied again. Similarly, /etc/crypttab may
# have been legitimately created by the user-setup plugin.
if relpath in ("etc/fstab", "etc/crypttab"):
continue
sourcepath = os.path.join(self.source, relpath)
targetpath = os.path.join(self.target, relpath)
st = os.lstat(sourcepath)
# Is the path blacklisted?
if (not stat.S_ISDIR(st.st_mode) and
'/%s' % relpath in self.blacklist):
if debug:
syslog.syslog('Not copying %s' % relpath)
continue
# Remove the target if necessary and if we can.
install_misc.remove_target(
self.source, self.target, relpath, st)
# Now actually copy source to target.
mode = stat.S_IMODE(st.st_mode)
if stat.S_ISLNK(st.st_mode):
linkto = os.readlink(sourcepath)
os.symlink(linkto, targetpath)
elif stat.S_ISDIR(st.st_mode):
if not os.path.isdir(targetpath):
try:
os.mkdir(targetpath, mode)
except OSError as e:
# there is a small window where update-apt-cache
# can race with us since it creates
# "/target/var/cache/apt/...". Hence, ignore
# failure if the directory does now exist where
# brief moments before it didn't.
if e.errno != errno.EEXIST:
raise
elif stat.S_ISCHR(st.st_mode):
os.mknod(targetpath, stat.S_IFCHR | mode, st.st_rdev)
elif stat.S_ISBLK(st.st_mode):
os.mknod(targetpath, stat.S_IFBLK | mode, st.st_rdev)
elif stat.S_ISFIFO(st.st_mode):
os.mknod(targetpath, stat.S_IFIFO | mode)
elif stat.S_ISSOCK(st.st_mode):
os.mknod(targetpath, stat.S_IFSOCK | mode)
elif stat.S_ISREG(st.st_mode):
install_misc.copy_file(
self.db, sourcepath, targetpath, md5_check)
# Copy metadata.
copied_size += st.st_size
os.lchown(targetpath, st.st_uid, st.st_gid)
if not stat.S_ISLNK(st.st_mode):
os.chmod(targetpath, mode)
if stat.S_ISDIR(st.st_mode):
directory_times.append(
(targetpath, st.st_atime, st.st_mtime))
# os.utime() sets timestamp of target, not link
elif not stat.S_ISLNK(st.st_mode):
try:
os.utime(targetpath, (st.st_atime, st.st_mtime))
except Exception:
# We can live with timestamps being wrong.
pass
if (hasattr(os, "listxattr") and
hasattr(os, "supports_follow_symlinks") and
os.supports_follow_symlinks):
try:
attrnames = os.listxattr(
sourcepath, follow_symlinks=False)
for attrname in attrnames:
attrvalue = os.getxattr(
sourcepath, attrname, follow_symlinks=False)
os.setxattr(
targetpath, attrname, attrvalue,
follow_symlinks=False)
except OSError as e:
if e.errno not in (
errno.EPERM, errno.ENOTSUP, errno.ENODATA):
raise
if int((copied_size * 90) / total_size) != copy_progress:
copy_progress = int((copied_size * 90) / total_size)
self.db.progress('SET', 10 + copy_progress)
time_now = time.time()
if (time_now - times[-1][0]) >= 0.5:
times.append((time_now, copied_size))
if not long_enough and time_now - times[0][0] >= 10:
long_enough = True
if long_enough and time_now - time_last_update >= 2:
time_last_update = time_now
while (time_now - times[0][0] > 60 and
time_now - times[1][0] >= 60):
times.pop(0)
speed = ((times[-1][1] - times[0][1]) /
(times[-1][0] - times[0][0]))
if speed != 0:
time_remaining = (
int((total_size - copied_size) / speed))
if time_remaining < 60:
self.db.progress(
'INFO', 'ubiquity/install/copying_minute')
# Apply timestamps to all directories now that the items within them
# have been copied.
for dirtime in directory_times:
(directory, atime, mtime) = dirtime
try:
os.utime(directory, (atime, mtime))
except Exception:
# I have no idea why I've been getting lots of bug reports
# about this failing, but I really don't care. Ignore it.
pass
# Revert to previous kernel flush times.
if dirty_writeback_centisecs is not None:
with open('/proc/sys/vm/dirty_writeback_centisecs', 'w') as dwc:
print(dirty_writeback_centisecs, file=dwc)
if dirty_expire_centisecs is not None:
with open('/proc/sys/vm/dirty_expire_centisecs', 'w') as dec:
print(dirty_expire_centisecs, file=dec)
# Try some possible locations for the kernel we used to boot. This
# lets us save a couple of megabytes of CD space.
bootdir = self.target_file('boot')
kernel = self.find_cd_kernel()
if kernel:
prefix = os.path.basename(kernel).split('-', 1)[0]
release = os.uname()[2]
target_kernel = os.path.join(bootdir, '%s-%s' % (prefix, release))
copies = []
# ISO9660 images may have to use .efi rather than .efi.signed in
# order to support being booted using isolinux, which must abide
# by archaic 8.3 restrictions.
for suffix in (".efi", ".efi.signed"):
if os.path.exists(kernel + suffix):
signed_kernel = kernel + suffix
break
else:
signed_kernel = None
if os.path.exists(kernel):
copies.append((kernel, target_kernel))
elif signed_kernel is not None:
# No unsigned kernel. We'll construct it using sbsigntool.
copies.append((signed_kernel, target_kernel))
if signed_kernel is not None:
copies.append((signed_kernel, "%s.efi.signed" % target_kernel))
for source, target in copies:
osextras.unlink_force(target)
install_misc.copy_file(self.db, source, target, md5_check)
os.lchown(target, 0, 0)
os.chmod(target, 0o644)
st = os.lstat(source)
try:
os.utime(target, (st.st_atime, st.st_mtime))
except Exception:
# We can live with timestamps being wrong.
pass
if not os.path.exists(kernel) and signed_kernel is not None:
# Construct the unsigned kernel.
subprocess.check_call(["sbattach", "--remove", target_kernel])
os.umask(old_umask)
self.db.progress('SET', 100)
self.db.progress('STOP')
def get_attribute(self, path, attr_name):
return os.getxattr(self.full_path(path), attr_name)
def getxattr(self): return os.getxattr(self.abspath(), SIG_VAR)