def test_listxattr(setup_unreliablefs, symlink):
mnt_dir, src_dir = setup_unreliablefs
name = name_generator()
src_name = pjoin(src_dir, name)
mnt_name = pjoin(src_dir, name)
os_create(mnt_name)
linkname = name_generator()
link_path = os.path.join(mnt_dir, linkname)
os.symlink(mnt_dir, link_path)
if symlink:
target = link_path
else:
target = mnt_name
attr1_name = b"user.aa"
attr1_value = b"a"
attr2_name = b"user.bb"
attr2_value = b"b"
num_attrs = len(os.listxattr(target))
os.setxattr(target, attr1_name, attr1_value)
assert attr1_name.decode("utf-8") in os.listxattr(target)
os.setxattr(target, attr2_name, attr2_value)
assert attr2_name.decode("utf-8") in os.listxattr(target)
assert num_attrs + 2 == len(os.listxattr(target))
def verify(self, target_dir):
""""""
with pushd(target_dir):
for f in self.source_files.keys():
fp = os.path.join(target_dir, f)
attrs = os.listxattr(path=fp, follow_symlinks=False)
assert len(attrs) == self.xattr_pairs
for k in self.source_xattrs[f].keys():
v = os.getxattr(fp, k.encode(self.encoding)).decode(
self.encoding)
assert v == self.source_xattrs[f][k]
attrs = os.listxattr(fp, follow_symlinks=False)
if self.encoding != "gb2312":
for attr in attrs:
v = xattr.getxattr(f, attr)
assert attr in self.source_xattrs[f].keys()
assert v.decode(
self.encoding) == self.source_xattrs[f][attr]
with pushd(target_dir):
for d in self.source_dirs.keys():
dp = os.path.join(target_dir, d)
attrs = xattr.listxattr(dp)
assert len(attrs) == self.xattr_pairs
for attr in attrs:
v = xattr.getxattr(d, attr)
assert attr in self.source_dirs_xattrs[d].keys()
assert v.decode(
self.encoding) == self.source_dirs_xattrs[d][attr]
def __init__(self, path: str, lock: typing.Any = None):
"""Initialize the POSIX compatible datastore.
The POSIX cache store uses xattrs to store metadata about stored
objects. Metadata is used to store the key and expiry information
which is used to ensure we're maintaining a POSIX compliant data
store which leverages simple file hashing. If xattrs are not
availble on the filesystem, the cache method will fallback to a
standard string encoding, and rely on in file information for
expiry times.
> If a lock object is not provided, a multiprocessing lock will
be used.
:param path: Storage path
:type path: String
:param lock: Lock type object
:type lock: Object
"""
if not lock:
lock = multiprocessing.Lock()
self._lock = lock
self._db_path = os.path.abspath(os.path.expanduser(path))
_makedirs(path=self._db_path)
try:
os.listxattr(self._db_path)
except Exception:
self._encoder = str
else:
self._encoder = utils.object_sha3_224
def _get_xattrs(fd, follow_symlinks=True):
ret = []
for name in os.listxattr(fd, follow_symlinks=follow_symlinks):
name = name.encode()
ret.append(
(name, os.getxattr(fd, name, follow_symlinks=follow_symlinks)))
return ret
def _copyxattr(src, dest, exclude=None):
try:
attrs = _os.listxattr(src)
except OSError as e:
if e.errno != OperationNotSupported.errno:
raise
attrs = ()
if attrs:
if exclude is not None and isinstance(attrs[0], bytes):
exclude = exclude.encode(_encodings['fs'])
exclude = _get_xattr_excluder(exclude)
for attr in attrs:
if exclude(attr):
continue
try:
_os.setxattr(dest, attr, _os.getxattr(src, attr))
raise_exception = False
except OSError:
raise_exception = True
if raise_exception:
raise OperationNotSupported(
_("Filesystem containing file '%s' "
"does not support extended attribute '%s'") %
(_unicode_decode(dest), _unicode_decode(attr)))
async def listxattr(self, vnode, ctx):
path = self.vm[vnode].path
try:
xattrs = os.listxattr(path)
except OSError as exc:
raise pyfuse3.FUSEError(exc.errno)
return list(map(os.fsencode, xattrs))
def _copyxattr(src, dest, exclude=None):
try:
attrs = _os.listxattr(src)
except OSError as e:
if e.errno != OperationNotSupported.errno:
raise
attrs = ()
if attrs:
if exclude is not None and isinstance(attrs[0], bytes):
exclude = exclude.encode(_encodings['fs'])
exclude = _get_xattr_excluder(exclude)
for attr in attrs:
if exclude(attr):
continue
try:
_os.setxattr(dest, attr, _os.getxattr(src, attr))
raise_exception = False
except OSError:
raise_exception = True
if raise_exception:
raise OperationNotSupported(_("Filesystem containing file '%s' "
"does not support extended attribute '%s'") %
(_unicode_decode(dest), _unicode_decode(attr)))
def test_removexattr(setup_unreliablefs, symlink):
mnt_dir, src_dir = setup_unreliablefs
name = name_generator()
src_name = pjoin(src_dir, name)
mnt_name = pjoin(mnt_dir, name)
os_create(mnt_name)
linkname = name_generator()
link_path = os.path.join(mnt_dir, linkname)
os.symlink(mnt_dir, link_path)
if symlink:
target = link_path
else:
target = mnt_name
attr_value = b"unreliablefs"
attr_name = b"user.fsname"
os.setxattr(target, attr_name, attr_value)
assert attr_name.decode("utf-8") in os.listxattr(target)
assert os.getxattr(target, attr_name) == attr_value
os.removexattr(target, attr_name)
try:
assert os.getxattr(target, attr_name) == None
except OSError:
pass
def checkUbridgePermission(self):
"""Check if ubridge has the correct permission"""
if not sys.platform.startswith("win") and os.geteuid() == 0:
# we are root, so we should have privileged access.
return (0, None)
path = Servers.instance().localServerSettings().get("ubridge_path")
if path is None:
return (0, None)
if not os.path.exists(path):
return (2, "Ubridge path {path} doesn't exists".format(path=path))
request_setuid = False
if sys.platform.startswith("linux"):
if "security.capability" in os.listxattr(path):
caps = os.getxattr(path, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if not struct.unpack("<IIIII", caps)[1] & 1 << 13:
return(2, "Ubridge require CAP_NET_RAW. Run sudo setcap cap_net_admin,cap_net_raw=ep {path}".format(path=path))
else:
# capabilities not supported
request_setuid = True
if sys.platform.startswith("darwin") or request_setuid:
if os.stat(path).st_uid != 0 or not os.stat(path).st_mode & stat.S_ISUID:
return (2, "Ubridge should be setuid. Run sudo chown root {path} and sudo chmod 4755 {path}".format(path=path))
return (0, None)
def _copyxattr(src, dst, *, follow_symlinks=True):
"""Copy extended filesystem attributes from `src` to `dst`.
Overwrite existing attributes.
If `follow_symlinks` is false, symlinks won't be followed.
"""
try:
names = os.listxattr(src, follow_symlinks=follow_symlinks)
except OSError as e:
if e.errno not in (errno.ENOTSUP, errno.ENODATA):
raise
return
for name in names:
if name == 'system.wsl_case_sensitive':
# 这个属性是在 WSL Linux 下访问 windows 下的目录会有这个扩展属性
# 在 WSL Linux 下的目录也没有这个属性
# https://blogs.msdn.microsoft.com/commandline/2018/06/14/improved-per-directory-case-sensitivity-support-in-wsl/
continue
try:
value = os.getxattr(src, name, follow_symlinks=follow_symlinks)
os.setxattr(dst, name, value, follow_symlinks=follow_symlinks)
except OSError as e:
if e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA):
raise
def has_privileged_access(executable):
"""
Check if an executable can access Ethernet and TAP devices in
RAW mode.
:param executable: executable path
:returns: True or False
"""
if sys.platform.startswith("win"):
# do not check anything on Windows
return True
if os.geteuid() == 0:
# we are root, so we should have privileged access.
return True
if os.stat(executable).st_mode & stat.S_ISVTX == stat.S_ISVTX:
# the executable has a sticky bit.
return True
# test if the executable has the CAP_NET_RAW capability (Linux only)
if sys.platform.startswith(
"linux") and "security.capability" in os.listxattr(executable):
try:
caps = os.getxattr(executable, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if struct.unpack("<IIIII", caps)[1] & 1 << 13:
return True
except Exception as e:
log.error(
"could not determine if CAP_NET_RAW capability is set for {}: {}"
.format(executable, e))
return False
def get_file_atts_read(file: str):
# reading all at once to minimize data access
xattr_list = os.listxattr(file)
file_attributes_sum = [None]*Backup.att_number
file_attributes_time = [None]*Backup.att_number
for xattr in xattr_list:
for att in range(Backup.att_number):
att_sum_cur = Backup.att_sum + '_' + str(att)
att_time_cur = Backup.att_time + '_' + str(att)
if xattr == att_sum_cur:
raw_attribute = os.getxattr(file, att_sum_cur)
file_attributes_sum[att] = \
raw_attribute.decode(encoding='utf-8')
elif xattr == att_time_cur:
raw_attribute = os.getxattr(file, att_time_cur)
file_attributes_time[att] = \
raw_attribute.decode(encoding='utf-8')
else:
pass
sum_most_common, error_level = \
Backup.get_most_common_att(file_attributes_sum)
time_most_common, error_level = \
Backup.get_most_common_att(file_attributes_time)
return (sum_most_common, time_most_common, error_level)
def checkUbridgePermission(self):
"""Check if ubridge has the correct permission"""
if not sys.platform.startswith("win") and os.geteuid() == 0:
# we are root, so we should have privileged access.
return (0, None)
path = LocalServer.instance().localServerSettings().get("ubridge_path")
if path is None:
return (0, None)
if not os.path.exists(path):
return (2, "Ubridge path {path} doesn't exists".format(path=path))
request_setuid = False
if sys.platform.startswith("linux"):
try:
if "security.capability" in os.listxattr(path):
caps = os.getxattr(path, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if not struct.unpack("<IIIII", caps)[1] & 1 << 13:
return (2, "Ubridge requires CAP_NET_RAW. Run sudo setcap cap_net_admin,cap_net_raw=ep {path}".format(path=path))
else:
# capabilities not supported
request_setuid = True
except AttributeError:
# Due to a Python bug, os.listxattr could be missing: https://github.com/GNS3/gns3-gui/issues/2010
return (1, "Could not determine if CAP_NET_RAW capability is set for uBridge (Python bug)".format(path=path))
if sys.platform.startswith("darwin") or request_setuid:
if os.stat(path).st_uid != 0 or not os.stat(path).st_mode & stat.S_ISUID:
return (2, "Ubridge should be setuid. Run sudo chown root:admin {path} and sudo chmod 4750 {path}".format(path=path))
return (0, None)
def _checkUbridgePermissions(self):
"""
Checks that uBridge can interact with network interfaces.
"""
path = os.path.abspath(self._settings["ubridge_path"])
if not path or len(path) == 0 or not os.path.exists(path):
return False
if sys.platform.startswith("win"):
# do not check anything on Windows
return True
if os.geteuid() == 0:
# we are root, so we should have privileged access.
return True
request_setuid = False
if sys.platform.startswith("linux"):
# test if the executable has the CAP_NET_RAW capability (Linux only)
try:
if "security.capability" in os.listxattr(path):
caps = os.getxattr(path, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if not struct.unpack("<IIIII", caps)[1] & 1 << 13:
proceed = QtWidgets.QMessageBox.question(
self.parent(),
"uBridge",
"uBridge requires CAP_NET_RAW capability to interact with network interfaces. Set the capability to uBridge? All users on the system will be able to read packet from the network interfaces.",
QtWidgets.QMessageBox.Yes,
QtWidgets.QMessageBox.No)
if proceed == QtWidgets.QMessageBox.Yes:
sudo(["setcap", "cap_net_admin,cap_net_raw=ep"])
else:
# capabilities not supported
request_setuid = True
except AttributeError:
# Due to a Python bug, os.listxattr could be missing: https://github.com/GNS3/gns3-gui/issues/2010
log.warning("Could not determine if CAP_NET_RAW capability is set for uBridge (Python bug)")
return True
except OSError as e:
QtWidgets.QMessageBox.critical(self.parent(), "uBridge", "Can't set CAP_NET_RAW capability to uBridge {}: {}".format(path, str(e)))
return False
if sys.platform.startswith("darwin") or request_setuid:
try:
if os.stat(path).st_uid != 0 or not os.stat(path).st_mode & stat.S_ISUID:
proceed = QtWidgets.QMessageBox.question(
self.parent(),
"uBridge",
"uBridge requires root permissions to interact with network interfaces. Set root permissions to uBridge? All admin users on the system will be able to read packet from the network interfaces.",
QtWidgets.QMessageBox.Yes,
QtWidgets.QMessageBox.No)
if proceed == QtWidgets.QMessageBox.Yes:
sudo(["chown", "root:admin", path], ["chmod", "4750", path])
except OSError as e:
QtWidgets.QMessageBox.critical(self.parent(), "uBridge", "Can't set root permissions to uBridge {}: {}".format(path, str(e)))
return False
return True
def _copyxattr(src, dst, *, follow_symlinks=True):
"""Copy extended filesystem attributes from `src` to `dst`.
Overwrite existing attributes.
If `follow_symlinks` is false, symlinks won't be followed.
"""
try:
names = os.listxattr(src, follow_symlinks=follow_symlinks)
except OSError as e:
try:
if e.errno not in (errno.ENOTSUP, errno.ENODATA):
raise
return
finally:
e = None
del e
for name in names:
try:
value = os.getxattr(src, name, follow_symlinks=follow_symlinks)
os.setxattr(dst, name, value, follow_symlinks=follow_symlinks)
except OSError as e:
try:
if e.errno not in (errno.EPERM, errno.ENOTSUP,
errno.ENODATA):
raise
finally:
e = None
del e
def _copyxattr(self, dst, *, follow_symlinks=True):
"""Copy extended filesystem attributes from `self.source_path` to `dst`.
Overwrite existing attributes.
If `follow_symlinks` is false, symlinks won't be followed.
"""
try:
names = os.listxattr(self.source_path,
follow_symlinks=follow_symlinks)
except OSError as e:
if e.errno not in (errno.ENOTSUP, errno.ENODATA, errno.EINVAL):
raise
return
for name in names:
try:
value = os.getxattr(self.source_path,
name,
follow_symlinks=follow_symlinks)
os.setxattr(dst,
name,
value,
follow_symlinks=follow_symlinks)
except OSError as e:
if e.errno not in (errno.EPERM, errno.ENOTSUP,
errno.ENODATA, errno.EINVAL):
raise
def get_file_xatts(file: str):
# this method splits the file's extended attribute list into the following:
# - valid attributes - split into sum and time
# - surplus attributes - starting with 'user.backup' but not valid
# - other attributes - everything else -> not needed, skipping checks
#
# surplus attributes should be cleared by the end of the backup cycle
# reading all at once to minimize data access
xattr_list = os.listxattr(file)
valid_sum_xatts = []
valid_time_xatts = []
surplus_xatts = []
for xattr in xattr_list:
if xattr in Backup.valid_sum_atts:
valid_sum_xatts.append(xattr)
elif xattr in Backup.valid_time_atts:
valid_time_xatts.append(xattr)
elif xattr.startswith('user.backup.'):
surplus_xatts.append(xattr)
return valid_sum_xatts, valid_time_xatts, surplus_xatts
def _has_privileged_access(executable):
"""
Check if an executable can access Ethernet and TAP devices in
RAW mode.
:param executable: executable path
:returns: True or False
"""
if sys.platform.startswith("win"):
# do not check anything on Windows
return True
if os.geteuid() == 0:
# we are root, so we should have privileged access.
return True
if os.stat(executable).st_mode & stat.S_ISUID or os.stat(executable).st_mode & stat.S_ISGID:
# the executable has set UID bit.
return True
# test if the executable has the CAP_NET_RAW capability (Linux only)
if sys.platform.startswith("linux") and "security.capability" in os.listxattr(executable):
try:
caps = os.getxattr(executable, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if struct.unpack("<IIIII", caps)[1] & 1 << 13:
return True
except Exception as e:
log.error("could not determine if CAP_NET_RAW capability is set for {}: {}".format(executable, e))
return False
def _checkUbridgePermissions(self):
"""
Checks that uBridge can interact with network interfaces.
"""
path = self._settings["local_server"]["ubridge_path"]
if not path or len(path) == 0 or not os.path.exists(path):
return False
if sys.platform.startswith("win"):
# do not check anything on Windows
return True
if os.geteuid() == 0:
# we are root, so we should have privileged access.
return True
from .main_window import MainWindow
main_window = MainWindow.instance()
request_setuid = False
if sys.platform.startswith("linux"):
# test if the executable has the CAP_NET_RAW capability (Linux only)
try:
if "security.capability" in os.listxattr(path):
caps = os.getxattr(path, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if not struct.unpack("<IIIII", caps)[1] & 1 << 13:
proceed = QtWidgets.QMessageBox.question(
main_window,
"uBridge",
"uBridge requires CAP_NET_RAW capability to interact with network interfaces. Set the capability to uBridge?",
QtWidgets.QMessageBox.Yes,
QtWidgets.QMessageBox.No)
if proceed == QtWidgets.QMessageBox.Yes:
sudo(["setcap", "cap_net_admin,cap_net_raw=ep"])
else:
# capabilities not supported
request_setuid = True
except OSError as e:
QtWidgets.QMessageBox.critical(main_window, "uBridge", "Can't set CAP_NET_RAW capability to uBridge {}: {}".format(path, str(e)))
return False
if sys.platform.startswith("darwin") or request_setuid:
try:
if os.stat(path).st_uid != 0 or not os.stat(path).st_mode & stat.S_ISUID:
proceed = QtWidgets.QMessageBox.question(
main_window,
"uBridge",
"uBridge requires root permissions to interact with network interfaces. Set root permissions to uBridge?",
QtWidgets.QMessageBox.Yes,
QtWidgets.QMessageBox.No)
if proceed == QtWidgets.QMessageBox.Yes:
sudo(["chmod", "4755", path])
sudo(["chown", "root", path])
except OSError as e:
QtWidgets.QMessageBox.critical(main_window, "uBridge", "Can't set root permissions to uBridge {}: {}".format(path, str(e)))
return False
return True
def checkDynamipsPermission(self):
"""Check if dynamips has the correct permission"""
if not sys.platform.startswith("win") and os.geteuid() == 0:
# we are root, so we should have privileged access.
return (0, None)
path = LocalServer.instance().localServerSettings().get(
"dynamips_path")
if path is None:
return (0, None)
if not os.path.exists(path):
return (2, "Dynamips path {path} doesn't exists".format(path=path))
try:
if sys.platform.startswith(
"linux") and "security.capability" in os.listxattr(path):
caps = os.getxattr(path, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if not struct.unpack("<IIIII", caps)[1] & 1 << 13:
return (
2,
"Dynamips requires CAP_NET_RAW. Run sudo setcap cap_net_raw,cap_net_admin+eip {path}"
.format(path=path))
except AttributeError:
# Due to a Python bug, os.listxattr could be missing: https://github.com/GNS3/gns3-gui/issues/2010
return (
1,
"Could not determine if CAP_NET_RAW capability is set for Dynamips (Python bug)"
.format(path=path))
return (0, None)
def _strip_acl_posix1e(self, path):
posix_xattrs = ['system.posix_acl_access', 'system.posix_acl_default']
for xat in os.listxattr(path):
if xat not in posix_xattrs:
continue
os.removexattr(path, xat)
def getactiveevents():
"Return list of active events"
event_list = []
for l in os.listdir("/dev/input/"):
if l.startswith("event"):
if "system.posix_acl_access" in os.listxattr("/dev/input/" + l):
event_list.append("/dev/input/" + l)
return event_list
def test_listxattr(self) -> None:
# Assert that listxattr on a directory is empty and does not break
# future listxattr calls.
self.assertEqual([], os.listxattr(os.path.join(self.mount, "subdir")))
filename = os.path.join(self.mount, "hello")
xattrs = getallxattrs(filename)
self.assertEqual({}, xattrs)
def apply_xattr(path_to_file, url):
existing_attr = os.listxattr(path_to_file)
if not existing_attr:
print(f"{BColors.LIGHTGRAY}Setting {url} as user.xdg.origin.url for \
{path_to_file}{BColors.ENDC}")
try:
os.setxattr(path_to_file, r"user.xdg.origin.url", url.encode(), flags=os.XATTR_CREATE)
except BaseException as e:
print(f"{BColors.FAIL}Exception setting xattr for {path_to_file}: {e}{BColors.ENDC}")
def list_file_attrs(path):
try:
if hasattr(path, "fileno"):
path = path.fileno()
return [attr[5:] for attr in os.listxattr(path) if attr.startswith("user.")]
except FileNotFoundError:
raise
except OSError:
return []
def _copyxattr(src, dest):
for attr in _os.listxattr(src):
try:
_os.setxattr(dest, attr, _os.getxattr(src, attr))
raise_exception = False
except OSError:
raise_exception = True
if raise_exception:
raise OperationNotSupported("Filesystem containing file '%s' does not support extended attributes" % dest)
def list_file_attrs(path):
try:
if hasattr(path, "fileno"):
path = path.fileno()
return [attr[5:] for attr in os.listxattr(path) if attr.startswith("user.")]
except FileNotFoundError:
raise
except OSError:
return []
async def listxattr(self, inode: INode,
ctx: RequestContext) -> Sequence[bytes]:
try:
return [
_str2bytes(attr)
for attr in os.listxattr(path=self.paths[inode],
follow_symlinks=False)
]
except OSError as exc:
raise FUSEError(exc.errno) from None
def test_listxattr(self) -> None:
# Assert that listxattr on a directory is empty and does not break
# future listxattr calls.
self.assertEqual([], os.listxattr(os.path.join(self.mount, "subdir")))
filename = os.path.join(self.mount, "hello")
xattrs = getallxattrs(filename)
contents = open(filename, "rb").read()
expected_sha1 = sha1(contents)
self.assertEqual({}, xattrs)
def copyxattr(src, dest):
"""
Copy the extended attributes (xattr) from `src` to `dest`.
NOTE: xattr only available on Linux.
"""
if not hasattr(os, "listxattr"):
return
for name in os.listxattr(src):
value = os.getxattr(src, name)
os.setxattr(dest, name, value)
def copyxattr(src, dest):
"""
Copy the extended attributes (xattr) from `src` to `dest`.
NOTE: xattr only available on Linux.
"""
if not hasattr(os, "listxattr"):
return
for name in os.listxattr(src):
value = os.getxattr(src, name)
os.setxattr(dest, name, value)
def get_file_xatts(file: str):
# this method splits the file's extended attribute list into the following:
# - valid attributes - split into sum and time
# - surplus attributes - starting with 'user.backup' but not valid
# - other attributes - everything else -> not needed, skipping checks
#
# surplus attributes should be cleared by the end of the backup cycle
# reading all at once to minimize data access
xattr_list = os.listxattr(file)
return xattr_list
def _copyxattr(src, dest):
for attr in _os.listxattr(src):
try:
_os.setxattr(dest, attr, _os.getxattr(src, attr))
raise_exception = False
except OSError:
raise_exception = True
if raise_exception:
raise OperationNotSupported(
"Filesystem containing file '%s' does not support extended attributes"
% dest)
def _scan(self):
for image in self.runtime.iterdir():
name = image.name
self.images[name] = attrs = {}
for attr in os.listxattr(str(image)):
if not attr.startswith("user."):
continue
val = os.getxattr(str(image), attr)
val = val.decode()
key = attr[5:]
attrs[key] = val
def update_permissions(path: Path, valid_users: T.Set[str]) -> T.Set[str]:
ret = set(valid_users)
for attr in os.listxattr(path):
if attr == "user.access":
ret = set(os.getxattr(str(path), attr).decode().split(":"))
elif attr == "user.access_add":
ret = ret.union(os.getxattr(str(path), attr).decode().split(":"))
elif attr == "user.access_del":
ret = ret.difference(
os.getxattr(str(path), attr).decode().split(":")
)
return ret
def test_no_upload_with_lock_file(self):
self.start_thread()
time.sleep(0.25)
f_paths = self.gen_files(lock=True, boot=False)
# allow enough time that files should have been uploaded if they would be uploaded
time.sleep(5)
self.join_thread()
for f_path in f_paths:
uploaded = uploader.UPLOAD_ATTR_NAME in os.listxattr(f_path.replace('.bz2', ''))
self.assertFalse(uploaded, "File upload when locked")
def patch_node(p):
# getxattr/setxattr work weirdly for broken symlinks
# glusterd produces these sometimes, seem to be expendable
try: bugged = p.is_symlink() and not p.exists()
except OSError: bugged = True
if bugged:
if not dry_run: p.unlink()
return
for k in os.listxattr(p, follow_symlinks=False):
if not k.startswith('trusted.'): continue
k_user = '******'.format(k[8:])
v = os.getxattr(p, k, follow_symlinks=False)
log.debug(f'patch: {p} :: {k} -> {k_user} [{v!r}]')
if not dry_run: os.setxattr(p, k_user, v, follow_symlinks=False)
def _copyxattr(src, dst, *, follow_symlinks=True):
try:
names = os.listxattr(src, follow_symlinks=follow_symlinks)
except OSError as e:
if e.errno not in (errno.ENOTSUP, errno.ENODATA):
raise
return
for name in names:
try:
value = os.getxattr(src, name, follow_symlinks=follow_symlinks)
os.setxattr(dst, name, value, follow_symlinks=follow_symlinks)
except OSError as e:
while e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA):
raise
def process(self, entry, cached):
xattrs = []
try:
for key in os.listxattr(entry.path):
value = os.getxattr(entry.path, key)
try:
value = os.fsdecode(value)
except UnicodeDecodeError:
value = value.decode(sys.getfilesystemencoding(),
"backslashreplace")
xattrs.append(f"{key}={value}")
except OSError:
pass
yield "xattrs", xattrs
def _scan(self):
for image in self.runtime.iterdir():
name = image.name
self.images[name] = attrs = {}
if not image.exists():
continue
for attr in os.listxattr(str(image)):
if not attr.startswith("user."):
continue
val = os.getxattr(str(image), attr)
val = val.decode()
key = attr[5:]
attrs[key] = val
def _copyxattr(src, dst, *, follow_symlinks=True):
"""Copy extended filesystem attributes from `src` to `dst`.
Overwrite existing attributes.
If `follow_symlinks` is false, symlinks won't be followed.
"""
for name in os.listxattr(src, follow_symlinks=follow_symlinks):
try:
value = os.getxattr(src, name, follow_symlinks=follow_symlinks)
os.setxattr(dst, name, value, follow_symlinks=follow_symlinks)
except OSError as e:
if e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA):
raise
def _copyxattr(src, dst, symlinks=False):
"""Copy extended filesystem attributes from `src` to `dst`.
Overwrite existing attributes.
If the optional flag `symlinks` is set, symlinks won't be followed.
"""
for name in os.listxattr(src, follow_symlinks=symlinks):
try:
value = os.getxattr(src, name, follow_symlinks=symlinks)
os.setxattr(dst, name, value, follow_symlinks=symlinks)
except OSError as e:
if e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA):
raise
def checkDynamipsPermission(self):
"""Check if dynamips has the correct permission"""
if not sys.platform.startswith("win") and os.geteuid() == 0:
# we are root, so we should have privileged access.
return (0, None)
path = Servers.instance().localServerSettings().get("dynamips_path")
if path is None:
return (0, None)
if not os.path.exists(path):
return (2, "Dynamips path {path} doesn't exists".format(path=path))
if sys.platform.startswith("linux") and "security.capability" in os.listxattr(path):
caps = os.getxattr(path, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if not struct.unpack("<IIIII", caps)[1] & 1 << 13:
return(2, "Dynamips require CAP_NET_RAW. Run sudo setcap cap_net_raw,cap_net_admin+eip {path}".format(path=path))
return (0, None)
def checkDynamipsPermission(self):
"""Check if dynamips has the correct permission"""
if not sys.platform.startswith("win") and os.geteuid() == 0:
# we are root, so we should have privileged access.
return (0, None)
path = LocalServer.instance().localServerSettings().get("dynamips_path")
if path is None:
return (0, None)
if not os.path.exists(path):
return (2, "Dynamips path {path} doesn't exists".format(path=path))
try:
if sys.platform.startswith("linux") and "security.capability" in os.listxattr(path):
caps = os.getxattr(path, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if not struct.unpack("<IIIII", caps)[1] & 1 << 13:
return (2, "Dynamips requires CAP_NET_RAW. Run sudo setcap cap_net_raw,cap_net_admin+eip {path}".format(path=path))
except AttributeError:
# Due to a Python bug, os.listxattr could be missing: https://github.com/GNS3/gns3-gui/issues/2010
return (1, "Could not determine if CAP_NET_RAW capability is set for Dynamips (Python bug)".format(path=path))
return (0, None)
def _check_for_privileged_access(self, device):
"""
Check if iouyap can access Ethernet and TAP devices.
:param device: device name
"""
# we are root, so iouyap should have privileged access too
if os.geteuid() == 0:
return
# test if iouyap has the CAP_NET_RAW capability
if "security.capability" in os.listxattr(self._iouyap):
try:
caps = os.getxattr(self._iouyap, "security.capability")
# test the 2nd byte and check if the 13th bit (CAP_NET_RAW) is set
if struct.unpack("<IIIII", caps)[1] & 1 << 13:
return
except Exception as e:
log.error("could not determine if CAP_NET_RAW capability is set for {}: {}".format(self._iouyap, e))
return
raise IOUError("{} has no privileged access to {}.".format(self._iouyap, device))
def get_file_atts_read(file: str):
# reading all at once to minimize data access
xattr_list = os.listxattr(file)
return xattr_list
def list(item, nofollow=False, namespace=None): return os.listxattr(item, follow_symlinks=not nofollow)
def copy_all(self):
"""Core copy process. This is the most important step of this
stage. It clones live filesystem into a local partition in the
selected hard disk."""
self.db.progress('START', 0, 100, 'ubiquity/install/title')
self.db.progress('INFO', 'ubiquity/install/copying')
fs_size = os.path.join(self.casper_path, 'filesystem.size')
if os.path.exists(fs_size):
with open(fs_size) as total_size_fp:
total_size = int(total_size_fp.readline())
else:
# Fallback in case an Linux Mint derivative forgets to put
# /casper/filesystem.size on the CD, or to account for things
# like CD->USB transformation tools that don't copy this file.
# This is slower than just reading the size from a file, but
# better than crashing.
#
# Obviously doing os.walk() twice is inefficient, but I'd rather
# not suck the list into ubiquity's memory, and I'm guessing
# that the kernel's dentry cache will avoid most of the slowness
# anyway.
total_size = 0
for dirpath, dirnames, filenames in os.walk(self.source):
for name in dirnames + filenames:
fqpath = os.path.join(dirpath, name)
total_size += os.lstat(fqpath).st_size
# Progress bar handling:
# We sample progress every half-second (assuming time.time() gives
# us sufficiently good granularity) and use the average of progress
# over the last minute or so to decide how much time remains. We
# don't bother displaying any progress for the first ten seconds in
# order to allow things to settle down, and we only update the "time
# remaining" indicator at most every two seconds after that.
copy_progress = 0
copied_size = 0
directory_times = []
time_start = time.time()
times = [(time_start, copied_size)]
long_enough = False
time_last_update = time_start
debug = 'UBIQUITY_DEBUG' in os.environ
if self.db.get('ubiquity/install/md5_check') == 'false':
md5_check = False
else:
md5_check = True
# Increase kernel flush times during bulk data copying to make it
# more likely that small files are packed contiguously, which should
# speed up initial boot times.
dirty_writeback_centisecs = None
dirty_expire_centisecs = None
if os.path.exists('/proc/sys/vm/dirty_writeback_centisecs'):
with open('/proc/sys/vm/dirty_writeback_centisecs') as dwc:
dirty_writeback_centisecs = int(dwc.readline())
with open('/proc/sys/vm/dirty_writeback_centisecs', 'w') as dwc:
print('3000\n', file=dwc)
if os.path.exists('/proc/sys/vm/dirty_expire_centisecs'):
with open('/proc/sys/vm/dirty_expire_centisecs') as dec:
dirty_expire_centisecs = int(dec.readline())
with open('/proc/sys/vm/dirty_expire_centisecs', 'w') as dec:
print('6000\n', file=dec)
old_umask = os.umask(0)
for dirpath, dirnames, filenames in os.walk(self.source):
sp = dirpath[len(self.source) + 1:]
for name in dirnames + filenames:
relpath = os.path.join(sp, name)
# /etc/fstab was legitimately created by partman, and
# shouldn't be copied again. Similarly, /etc/crypttab may
# have been legitimately created by the user-setup plugin.
if relpath in ("etc/fstab", "etc/crypttab"):
continue
sourcepath = os.path.join(self.source, relpath)
targetpath = os.path.join(self.target, relpath)
st = os.lstat(sourcepath)
# Is the path blacklisted?
if (not stat.S_ISDIR(st.st_mode) and
'/%s' % relpath in self.blacklist):
if debug:
syslog.syslog('Not copying %s' % relpath)
continue
# Remove the target if necessary and if we can.
install_misc.remove_target(
self.source, self.target, relpath, st)
# Now actually copy source to target.
mode = stat.S_IMODE(st.st_mode)
if stat.S_ISLNK(st.st_mode):
linkto = os.readlink(sourcepath)
os.symlink(linkto, targetpath)
elif stat.S_ISDIR(st.st_mode):
if not os.path.isdir(targetpath):
try:
os.mkdir(targetpath, mode)
except OSError as e:
# there is a small window where update-apt-cache
# can race with us since it creates
# "/target/var/cache/apt/...". Hence, ignore
# failure if the directory does now exist where
# brief moments before it didn't.
if e.errno != errno.EEXIST:
raise
elif stat.S_ISCHR(st.st_mode):
os.mknod(targetpath, stat.S_IFCHR | mode, st.st_rdev)
elif stat.S_ISBLK(st.st_mode):
os.mknod(targetpath, stat.S_IFBLK | mode, st.st_rdev)
elif stat.S_ISFIFO(st.st_mode):
os.mknod(targetpath, stat.S_IFIFO | mode)
elif stat.S_ISSOCK(st.st_mode):
os.mknod(targetpath, stat.S_IFSOCK | mode)
elif stat.S_ISREG(st.st_mode):
install_misc.copy_file(
self.db, sourcepath, targetpath, md5_check)
# Copy metadata.
copied_size += st.st_size
os.lchown(targetpath, st.st_uid, st.st_gid)
if not stat.S_ISLNK(st.st_mode):
os.chmod(targetpath, mode)
if stat.S_ISDIR(st.st_mode):
directory_times.append(
(targetpath, st.st_atime, st.st_mtime))
# os.utime() sets timestamp of target, not link
elif not stat.S_ISLNK(st.st_mode):
try:
os.utime(targetpath, (st.st_atime, st.st_mtime))
except Exception:
# We can live with timestamps being wrong.
pass
if (hasattr(os, "listxattr") and
hasattr(os, "supports_follow_symlinks") and
os.supports_follow_symlinks):
try:
attrnames = os.listxattr(
sourcepath, follow_symlinks=False)
for attrname in attrnames:
attrvalue = os.getxattr(
sourcepath, attrname, follow_symlinks=False)
os.setxattr(
targetpath, attrname, attrvalue,
follow_symlinks=False)
except OSError as e:
if e.errno not in (
errno.EPERM, errno.ENOTSUP, errno.ENODATA):
raise
if int((copied_size * 90) / total_size) != copy_progress:
copy_progress = int((copied_size * 90) / total_size)
self.db.progress('SET', 10 + copy_progress)
time_now = time.time()
if (time_now - times[-1][0]) >= 0.5:
times.append((time_now, copied_size))
if not long_enough and time_now - times[0][0] >= 10:
long_enough = True
if long_enough and time_now - time_last_update >= 2:
time_last_update = time_now
while (time_now - times[0][0] > 60 and
time_now - times[1][0] >= 60):
times.pop(0)
speed = ((times[-1][1] - times[0][1]) /
(times[-1][0] - times[0][0]))
if speed != 0:
time_remaining = (
int((total_size - copied_size) / speed))
if time_remaining < 60:
self.db.progress(
'INFO', 'ubiquity/install/copying_minute')
# Apply timestamps to all directories now that the items within them
# have been copied.
for dirtime in directory_times:
(directory, atime, mtime) = dirtime
try:
os.utime(directory, (atime, mtime))
except Exception:
# I have no idea why I've been getting lots of bug reports
# about this failing, but I really don't care. Ignore it.
pass
# Revert to previous kernel flush times.
if dirty_writeback_centisecs is not None:
with open('/proc/sys/vm/dirty_writeback_centisecs', 'w') as dwc:
print(dirty_writeback_centisecs, file=dwc)
if dirty_expire_centisecs is not None:
with open('/proc/sys/vm/dirty_expire_centisecs', 'w') as dec:
print(dirty_expire_centisecs, file=dec)
# Try some possible locations for the kernel we used to boot. This
# lets us save a couple of megabytes of CD space.
bootdir = self.target_file('boot')
kernel = self.find_cd_kernel()
if kernel:
prefix = os.path.basename(kernel).split('-', 1)[0]
release = os.uname()[2]
target_kernel = os.path.join(bootdir, '%s-%s' % (prefix, release))
copies = []
# ISO9660 images may have to use .efi rather than .efi.signed in
# order to support being booted using isolinux, which must abide
# by archaic 8.3 restrictions.
for suffix in (".efi", ".efi.signed"):
if os.path.exists(kernel + suffix):
signed_kernel = kernel + suffix
break
else:
signed_kernel = None
if os.path.exists(kernel):
copies.append((kernel, target_kernel))
elif signed_kernel is not None:
# No unsigned kernel. We'll construct it using sbsigntool.
copies.append((signed_kernel, target_kernel))
if signed_kernel is not None:
copies.append((signed_kernel, "%s.efi.signed" % target_kernel))
for source, target in copies:
osextras.unlink_force(target)
install_misc.copy_file(self.db, source, target, md5_check)
os.lchown(target, 0, 0)
os.chmod(target, 0o644)
st = os.lstat(source)
try:
os.utime(target, (st.st_atime, st.st_mtime))
except Exception:
# We can live with timestamps being wrong.
pass
if not os.path.exists(kernel) and signed_kernel is not None:
# Construct the unsigned kernel.
subprocess.check_call(["sbattach", "--remove", target_kernel])
os.umask(old_umask)
self.db.progress('SET', 100)
self.db.progress('STOP')