def preserve_xattrs(path, nofollow=False, namespace=None):
"""Context manager to save/restore extended attributes on |path|
If you want to rewrite a file (possibly replacing it with a new one), but
want to preserve the extended attributes, this will do the trick.
# First read all the extended attributes.
with save_xattrs('/some/file'):
... rewrite the file ...
# Now the extended attributes are restored as needed.
"""
kwargs = {'nofollow': nofollow,}
if namespace:
# Compiled xattr python module does not like it when namespace=None.
kwargs['namespace'] = namespace
old_attrs = dict(xattr.get_all(path, **kwargs))
try:
yield
finally:
new_attrs = dict(xattr.get_all(path, **kwargs))
for name, value in new_attrs.items():
if name not in old_attrs:
# Clear out new ones.
xattr.remove(path, name, **kwargs)
elif new_attrs[name] != old_attrs[name]:
# Update changed ones.
xattr.set(path, name, value, **kwargs)
for name, value in old_attrs.items():
if name not in new_attrs:
# Re-add missing ones.
xattr.set(path, name, value, **kwargs)
def xattr_sync(file_name, meta):
""" Synchronize the xattrs on a file with a hash of our choosing """
to_delete, to_set = xattr_sync_impl(file_name, meta)
for k in to_delete:
xattr.remove(file_name, k)
for k, v in to_set.items():
xattr.set(file_name, k, v, namespace=xattr.NS_USER)
def xattr_sync(file_name, meta):
""" Synchronize the xattrs on a file with a hash of our choosing """
to_delete, to_set = xattr_sync_impl(file_name, meta)
for k in to_delete:
xattr.remove(file_name, k)
for k,v in to_set.items():
xattr.set(file_name, k, v, namespace=xattr.NS_USER)
def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
if not xattr:
if self.linux_xattr:
add_error("%s: can't restore xattr; xattr support missing.\n"
% path)
return
existing_xattrs = set(xattr.list(path, nofollow=True))
if self.linux_xattr:
for k, v in self.linux_xattr:
if k not in existing_xattrs \
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM \
or e.errno == errno.EOPNOTSUPP:
raise ApplyError('xattr.set: %s' % e)
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.remove: %s' % e)
else:
raise
def test_set_get_remove(subject, use_ns):
item = subject[0]
lists_equal(xattr.list(item), [])
if use_ns:
xattr.set(item, USER_NN, USER_VAL,
namespace=NAMESPACE)
else:
xattr.set(item, USER_ATTR, USER_VAL)
if use_ns:
assert xattr.list(item, namespace=NAMESPACE) == [USER_NN]
else:
lists_equal(xattr.list(item), [USER_ATTR])
lists_equal(xattr.list(item, namespace=EMPTY_NS),
[USER_ATTR])
if use_ns:
assert xattr.get(item, USER_NN, namespace=NAMESPACE) == USER_VAL
else:
assert xattr.get(item, USER_ATTR) == USER_VAL
if use_ns:
assert xattr.get_all(item, namespace=NAMESPACE) == \
[(USER_NN, USER_VAL)]
else:
tuples_equal(xattr.get_all(item),
[(USER_ATTR, USER_VAL)])
if use_ns:
xattr.remove(item, USER_NN, namespace=NAMESPACE)
else:
xattr.remove(item, USER_ATTR)
lists_equal(xattr.list(item), [])
tuples_equal(xattr.get_all(item), [])
def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
if not xattr:
if self.linux_xattr:
add_error("%s: can't restore xattr; xattr support missing.\n"
% path_msg(path))
return
if not self.linux_xattr:
return
try:
existing_xattrs = set(xattr.list(path, nofollow=True))
except IOError as e:
if e.errno == errno.EACCES:
raise ApplyError('xattr.set %r: %s' % (path_msg(path), e))
else:
raise
for k, v in self.linux_xattr:
if k not in existing_xattrs \
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError as e:
if e.errno == errno.EPERM \
or e.errno == errno.EOPNOTSUPP:
raise ApplyError('xattr.set %r: %s' % (path_msg(path), e))
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError as e:
if e.errno in (errno.EPERM, errno.EACCES):
raise ApplyError('xattr.remove %r: %s' % (path_msg(path), e))
else:
raise
def preserve_xattrs(path, nofollow=False, namespace=None):
"""Context manager to save/restore extended attributes on |path|
If you want to rewrite a file (possibly replacing it with a new one), but
want to preserve the extended attributes, this will do the trick.
# First read all the extended attributes.
with save_xattrs('/some/file'):
... rewrite the file ...
# Now the extended attributes are restored as needed.
"""
kwargs = {
'nofollow': nofollow,
}
if namespace:
# Compiled xattr python module does not like it when namespace=None.
kwargs['namespace'] = namespace
old_attrs = dict(xattr.get_all(path, **kwargs))
try:
yield
finally:
new_attrs = dict(xattr.get_all(path, **kwargs))
for name, value in new_attrs.items():
if name not in old_attrs:
# Clear out new ones.
xattr.remove(path, name, **kwargs)
elif new_attrs[name] != old_attrs[name]:
# Update changed ones.
xattr.set(path, name, value, **kwargs)
for name, value in old_attrs.items():
if name not in new_attrs:
# Re-add missing ones.
xattr.set(path, name, value, **kwargs)
def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
if not xattr:
if self.linux_xattr:
add_error("%s: can't restore xattr; xattr support missing.\n" %
path)
return
existing_xattrs = set(xattr.list(path, nofollow=True))
if self.linux_xattr:
for k, v in self.linux_xattr:
if k not in existing_xattrs \
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.set: %s' % e)
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.remove: %s' % e)
else:
raise
def test_loading_attr_ensures_all_xattrs_are_set(self):
dumbattr.set(self.file1, "file_1", "1")
xattr.remove(self.file1, "file_1", namespace=xattr.NS_USER)
self.assertEqual(self.all_xattrs(self.file1), {})
self.assertEqual(dumbattr.load(self.file1).copy(), {'file_1':'1'})
self.assertEqual(self.all_xattrs(self.file1), {b'file_1':b'1'})
def _checkListSetGet(self, item, symlink=False, use_ns=False):
"""check list, set, get operations against an item"""
self.assertEqual(self._ignore(xattr.list(item, symlink)),
[])
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_ATTR, self.USER_VAL, flags=XATTR_REPLACE)
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_NN, self.USER_VAL, flags=XATTR_REPLACE,
namespace=NS_USER)
try:
if use_ns:
xattr.set(item, self.USER_NN, self.USER_VAL,
namespace=NS_USER,
nofollow=symlink)
else:
xattr.set(item, self.USER_ATTR, self.USER_VAL,
nofollow=symlink)
except IOError:
err = sys.exc_info()[1]
if err.errno == errno.EPERM and symlink:
# symlinks may fail, in which case we abort the rest
# of the test for this case
return
raise
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_ATTR, self.USER_VAL, flags=XATTR_CREATE)
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_NN, self.USER_VAL,
flags=XATTR_CREATE, namespace=NS_USER)
self.assertEqual(self._ignore(xattr.list(item, nofollow=symlink)),
[self.USER_ATTR])
self.assertEqual(self._ignore(xattr.list(item, nofollow=symlink,
namespace=EMPTY_NS)),
[self.USER_ATTR])
self.assertEqual(xattr.list(item, namespace=NS_USER, nofollow=symlink),
[self.USER_NN])
self.assertEqual(xattr.get(item, self.USER_ATTR, nofollow=symlink),
self.USER_VAL)
self.assertEqual(xattr.get(item, self.USER_NN, nofollow=symlink,
namespace=NS_USER), self.USER_VAL)
self.assertEqual(self._ignore_tuples(xattr.get_all(item,
nofollow=symlink)),
[(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(item, nofollow=symlink,
namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
if use_ns:
xattr.remove(item, self.USER_NN, namespace=NS_USER)
else:
xattr.remove(item, self.USER_ATTR)
self.assertEqual(self._ignore(xattr.list(item, symlink)), [])
self.assertEqual(self._ignore_tuples(xattr.get_all(item,
nofollow=symlink)),
[])
self.assertRaises(EnvironmentError, xattr.remove,
item, self.USER_ATTR, nofollow=symlink)
self.assertRaises(EnvironmentError, xattr.remove, item,
self.USER_NN, namespace=NS_USER, nofollow=symlink)
def clean_metadata(path):
key = 0
while True:
try:
xattr.remove(path, '%s%s' % (METADATA_KEY, (key or '')))
except IOError as err:
if err.errno == errno.ENODATA:
break
key += 1
def test_remove_on_missing(any_subject, use_ns):
item, nofollow = any_subject
lists_equal(xattr.list(item, nofollow=nofollow), [])
with pytest.raises(EnvironmentError):
if use_ns:
xattr.remove(item, USER_NN, namespace=NAMESPACE,
nofollow=nofollow)
else:
xattr.remove(item, USER_ATTR, nofollow=nofollow)
def _checkListSetGet(self, item, symlink=False, use_ns=False):
"""check list, set, get operations against an item"""
self.checkList(xattr.list(item, symlink), [])
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_ATTR, self.USER_VAL, flags=XATTR_REPLACE)
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_NN, self.USER_VAL, flags=XATTR_REPLACE,
namespace=NS_USER)
try:
if use_ns:
xattr.set(item, self.USER_NN, self.USER_VAL,
namespace=NS_USER,
nofollow=symlink)
else:
xattr.set(item, self.USER_ATTR, self.USER_VAL,
nofollow=symlink)
except IOError:
err = sys.exc_info()[1]
if symlink and (err.errno == errno.EPERM or
err.errno == errno.ENOENT):
# symlinks may fail, in which case we abort the rest
# of the test for this case (Linux returns EPERM; OS X
# returns ENOENT)
return
raise
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_ATTR, self.USER_VAL, flags=XATTR_CREATE)
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_NN, self.USER_VAL,
flags=XATTR_CREATE, namespace=NS_USER)
self.checkList(xattr.list(item, nofollow=symlink), [self.USER_ATTR])
self.checkList(xattr.list(item, nofollow=symlink,
namespace=EMPTY_NS),
[self.USER_ATTR])
self.assertEqual(xattr.list(item, namespace=NS_USER, nofollow=symlink),
[self.USER_NN])
self.assertEqual(xattr.get(item, self.USER_ATTR, nofollow=symlink),
self.USER_VAL)
self.assertEqual(xattr.get(item, self.USER_NN, nofollow=symlink,
namespace=NS_USER), self.USER_VAL)
self.checkTuples(xattr.get_all(item, nofollow=symlink),
[(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(item, nofollow=symlink,
namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
if use_ns:
xattr.remove(item, self.USER_NN, namespace=NS_USER)
else:
xattr.remove(item, self.USER_ATTR)
self.checkList(xattr.list(item, symlink), [])
self.checkTuples(xattr.get_all(item, nofollow=symlink),
[])
self.assertRaises(EnvironmentError, xattr.remove,
item, self.USER_ATTR, nofollow=symlink)
self.assertRaises(EnvironmentError, xattr.remove, item,
self.USER_NN, namespace=NS_USER, nofollow=symlink)
def clean_metadata(path):
key = 0
while True:
try:
xattr.remove(path, '%s%s' % (METADATA_KEY, (key or '')))
except IOError as err:
if err.errno == errno.ENODATA:
break
raise
key += 1
def xattr_remove(fn, k):
if not is_file(fn):
return False
try:
xattr.remove(fn, k, namespace=xattr.NS_USER)
except Exception as e:
# print(e)
return None
return True
def testSymlinkOps(self):
"""test symlink operations"""
_, sname = self._getsymlink()
self.assertRaises(EnvironmentError, xattr.list, sname)
self._checkListSetGet(sname, symlink=True)
self._checkListSetGet(sname, symlink=True, use_ns=True)
target, sname = self._getsymlink(dangling=False)
xattr.set(target, self.USER_ATTR, self.USER_VAL)
self.assertEqual(xattr.list(target), [self.USER_ATTR])
self.assertEqual(xattr.list(sname, nofollow=True), [])
self.assertRaises(EnvironmentError, xattr.remove, sname, self.USER_ATTR, nofollow=True)
xattr.remove(sname, self.USER_ATTR, nofollow=False)
def test_binary_payload(subject):
"""test binary values"""
item = subject[0]
BINVAL = b"abc\0def"
xattr.set(item, USER_ATTR, BINVAL)
lists_equal(xattr.list(item), [USER_ATTR])
assert xattr.list(item, namespace=NAMESPACE) == [USER_NN]
assert xattr.get(item, USER_ATTR) == BINVAL
assert xattr.get(item, USER_NN, namespace=NAMESPACE) == BINVAL
tuples_equal(xattr.get_all(item), [(USER_ATTR, BINVAL)])
assert xattr.get_all(item, namespace=NAMESPACE) == [(USER_NN, BINVAL)]
xattr.remove(item, USER_ATTR)
def check_user_xattr(path):
if not os.path.exists(path):
return False
try:
xattr.set(path, 'user.test.key1', 'value1')
except IOError as err:
logging.exception("check_user_xattr: set failed on %s err: %s", path, str(err))
raise
try:
xattr.remove(path, 'user.test.key1')
except Exception, err:
logging.exception("xattr.remove failed on %s err: %s", path, str(err))
def testSymlinkOps(self):
"""test symlink operations"""
_, sname = self._getsymlink()
self.assertRaises(EnvironmentError, xattr.list, sname)
self._checkListSetGet(sname, symlink=True)
self._checkListSetGet(sname, symlink=True, use_ns=True)
target, sname = self._getsymlink(dangling=False)
xattr.set(target, self.USER_ATTR, self.USER_VAL)
self.checkList(xattr.list(target), [self.USER_ATTR])
self.checkList(xattr.list(sname, nofollow=True), [])
self.assertRaises(EnvironmentError, xattr.remove, sname,
self.USER_ATTR, nofollow=True)
xattr.remove(sname, self.USER_ATTR, nofollow=False)
def check_user_xattr(path):
if not os.path.exists(path):
return False
try:
xattr.set(path, 'user.test.key1', 'value1')
except IOError as err:
logging.exception("check_user_xattr: set failed on %s err: %s", path, str(err))
raise
try:
xattr.remove(path, 'user.test.key1')
except IOError as err:
logging.exception("check_user_xattr: remove failed on %s err: %s", path, str(err))
#Remove xattr may fail in case of concurrent remove.
return True
def tmp_path_xattrs_ok(tmp_path_factory):
try:
import xattr # type: ignore
except ImportError:
pytest.skip("xattr module not available")
tmpp = tmp_path_factory.mktemp("needs_xattrs")
try:
xattr.set(str(tmpp), "user.deleteme", "1")
xattr.remove(str(tmpp), "user.deleteme")
except OSError:
raise pytest.skip(
"temp dir does not support xattrs"
" (try changing basetmp to a file system that supports xattrs)")
return tmpp
def testBinaryPayload(self):
"""test binary values"""
fh, fname = self._getfile()
os.close(fh)
BINVAL = "abc" + "\0" + "def"
if PY3K:
BINVAL = BINVAL.encode()
xattr.set(fname, self.USER_ATTR, BINVAL)
self.assertEqual(xattr.list(fname), [self.USER_ATTR])
self.assertEqual(xattr.list(fname, namespace=NS_USER), [self.USER_NN])
self.assertEqual(xattr.get(fname, self.USER_ATTR), BINVAL)
self.assertEqual(xattr.get(fname, self.USER_NN, namespace=NS_USER), BINVAL)
self.assertEqual(xattr.get_all(fname), [(self.USER_ATTR, BINVAL)])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER), [(self.USER_NN, BINVAL)])
xattr.remove(fname, self.USER_ATTR)
def check_user_xattr(path):
if not os.path.exists(path):
return False
try:
xattr.set(path, 'user.test.key1', 'value1')
except IOError as err:
logging.exception("check_user_xattr: set failed on %s err: %s", path,
str(err))
raise
try:
xattr.remove(path, 'user.test.key1')
except IOError as err:
logging.exception("check_user_xattr: remove failed on %s err: %s",
path, str(err))
#Remove xattr may fail in case of concurrent remove.
return True
def testBinaryPayload(self):
"""test binary values"""
fh, fname = self._getfile()
os.close(fh)
BINVAL = "abc" + '\0' + "def"
if PY3K:
BINVAL = BINVAL.encode()
xattr.set(fname, self.USER_ATTR, BINVAL)
self.checkList(xattr.list(fname), [self.USER_ATTR])
self.assertEqual(xattr.list(fname, namespace=NS_USER), [self.USER_NN])
self.assertEqual(xattr.get(fname, self.USER_ATTR), BINVAL)
self.assertEqual(xattr.get(fname, self.USER_NN, namespace=NS_USER),
BINVAL)
self.checkTuples(xattr.get_all(fname), [(self.USER_ATTR, BINVAL)])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER),
[(self.USER_NN, BINVAL)])
xattr.remove(fname, self.USER_ATTR)
def process(path):
# get tags
try: attrvalue = xattr.get(path, ATTRNAME)
except IOError: tags = set();
else: tags = set(attrvalue.split("\x1f"))
oldtags = tags
# perform operations
if options.filter and not set(options.filter) <= tags: return
if options.clear: tags = []
if options.add: tags = tags | set(options.add)
if options.delete: tags = tags - set(options.delete)
if not tags == oldtags:
if tags:
xattr.set(path, ATTRNAME, "\x1f".join(tags))
else:
try: xattr.remove(path, ATTRNAME)
except IOError: pass
if options.list: print path, "::", ", ".join(tags)
def _set(self, filename, key, value):
path = os.path.join(self.dirpath, filename)
logger.info("Setting %s=%s (%s)", key, value, path)
if os.path.islink(path):
if value is None:
h = self._saved_attrs[filename]
del h[key]
if not h:
del self._saved_attrs[filename]
else:
if not filename in self._saved_attrs:
self._saved_attrs[filename] = {}
self._saved_attrs[filename][key] = value
else:
logger.debug("Setting xattr %s=%s (%s)", key, value, path)
if value is None:
xattr.remove(path, key, namespace=xattr.NS_USER)
else:
xattr.set(path, key, value, namespace=xattr.NS_USER)
self._update_saved_attrs(filename)
def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
existing_xattrs = set(xattr.list(path, nofollow=True))
if(self.linux_xattr):
for k, v in self.linux_xattr:
if k not in existing_xattrs \
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.set: %s' % e)
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.remove: %s' % e)
else:
raise
def test_handling_of_incorrect_existing_linux_xattrs():
if os.geteuid() != 0 or detect_fakeroot():
return
setup_testfs()
subprocess.check_call('rm -rf testfs/*', shell=True)
path = 'testfs/foo'
open(path, 'w').close()
xattr.set(path, 'foo', 'bar', namespace=xattr.NS_USER)
m = metadata.from_path(path, archive_path=path, save_symlinks=True)
xattr.set(path, 'baz', 'bax', namespace=xattr.NS_USER)
m.apply_to_path(path, restore_numeric_ids=False)
WVPASSEQ(xattr.list(path), ['user.foo'])
WVPASSEQ(xattr.get(path, 'user.foo'), 'bar')
xattr.set(path, 'foo', 'baz', namespace=xattr.NS_USER)
m.apply_to_path(path, restore_numeric_ids=False)
WVPASSEQ(xattr.list(path), ['user.foo'])
WVPASSEQ(xattr.get(path, 'user.foo'), 'bar')
xattr.remove(path, 'foo', namespace=xattr.NS_USER)
m.apply_to_path(path, restore_numeric_ids=False)
WVPASSEQ(xattr.list(path), ['user.foo'])
WVPASSEQ(xattr.get(path, 'user.foo'), 'bar')
os.chdir(top_dir)
cleanup_testfs()
def delxattr(self, key, fail=False, namespace=XATTR_DEFAULT_NS):
try:
xattr.remove(self.as_posix(), key, namespace=namespace)
except OSError as e:
if fail or e.errno != 61: # 61 -> No data available
raise e
def clear_cache(self):
for name in xattr.list(self.name):
if name.decode().startswith('user.pyanidb.'):
xattr.remove(self.name, name)
def __remove(self,path,key): xattr.remove(path,key,namespace=self.ns)
def __delitem__(self, key):
"""
Removes a filesystem attribute
"""
xattr.remove(self._path, key, namespace=xattr.NS_USER)
raise ApplyError("xattr.set: %s" % e)
else:
raise
for k, v in self.linux_xattr:
if k not in existing_xattrs or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM or e.errno == errno.EOPNOTSUPP:
raise ApplyError("xattr.set: %s" % e)
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError("xattr.remove: %s" % e)
else:
raise
def __init__(self):
self.mode = None
# optional members
self.path = None
self.size = None
self.symlink_target = None
self.hardlink_target = None
self.linux_attr = None
self.linux_xattr = None
raise
for k, v in self.linux_xattr:
if k not in existing_xattrs \
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM \
or e.errno == errno.EOPNOTSUPP:
raise ApplyError('xattr.set %r: %s' % (path, e))
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.remove %r: %s' % (path, e))
else:
raise
def __init__(self):
self.mode = self.uid = self.gid = self.user = self.group = None
self.atime = self.mtime = self.ctime = None
# optional members
self.path = None
self.size = None
self.symlink_target = None
self.hardlink_target = None
self.linux_attr = None
def clear_cache(self):
for name in xattr.list(self.name):
if name.decode().startswith('user.pyanidb.'):
xattr.remove(self.name, name)
def __delitem__(self, key):
"""
Removes a filesystem attribute
"""
xattr.remove(self._path, key, namespace=xattr.NS_USER)