def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
if not xattr:
if self.linux_xattr:
add_error("%s: can't restore xattr; xattr support missing.\n"
% path_msg(path))
return
if not self.linux_xattr:
return
try:
existing_xattrs = set(xattr.list(path, nofollow=True))
except IOError as e:
if e.errno == errno.EACCES:
raise ApplyError('xattr.set %r: %s' % (path_msg(path), e))
else:
raise
for k, v in self.linux_xattr:
if k not in existing_xattrs \
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError as e:
if e.errno == errno.EPERM \
or e.errno == errno.EOPNOTSUPP:
raise ApplyError('xattr.set %r: %s' % (path_msg(path), e))
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError as e:
if e.errno in (errno.EPERM, errno.EACCES):
raise ApplyError('xattr.remove %r: %s' % (path_msg(path), e))
else:
raise
def preserve_xattrs(path, nofollow=False, namespace=None):
"""Context manager to save/restore extended attributes on |path|
If you want to rewrite a file (possibly replacing it with a new one), but
want to preserve the extended attributes, this will do the trick.
# First read all the extended attributes.
with save_xattrs('/some/file'):
... rewrite the file ...
# Now the extended attributes are restored as needed.
"""
kwargs = {
'nofollow': nofollow,
}
if namespace:
# Compiled xattr python module does not like it when namespace=None.
kwargs['namespace'] = namespace
old_attrs = dict(xattr.get_all(path, **kwargs))
try:
yield
finally:
new_attrs = dict(xattr.get_all(path, **kwargs))
for name, value in new_attrs.items():
if name not in old_attrs:
# Clear out new ones.
xattr.remove(path, name, **kwargs)
elif new_attrs[name] != old_attrs[name]:
# Update changed ones.
xattr.set(path, name, value, **kwargs)
for name, value in old_attrs.items():
if name not in new_attrs:
# Re-add missing ones.
xattr.set(path, name, value, **kwargs)
def recurse_directory( directory ):
for filename in os.listdir(directory):
if os.path.isdir(directory + "/" + filename):
if debug:
print("%s is a folder" % filename)
recurse_directory(directory + "/" + filename)
for p in xattr.listxattr(directory + "/" + filename):
s=p.decode()
if s.find(stringtoskip) != 2:
print(s.find(stringtoskip))
print("= %s : %s" % ( directory + "/" + filename, p ) )
print("%s", xattr.get(directory + "/" + filename,p))
if re.search(matchchecksum, s):
try:
thisxattr=xattr.get(directory + "/" + filename, 'user.swift.metadata_checksum')
except:
thisxattr=''
try:
if not thisxattr:
new_checksum = hashlib.md5(xattr.get(directory + "/" + filename,p)).hexdigest()
print("SET CHKSUM: %s" % ( new_checksum ))
xattr.set(directory + "/" + filename, 'user.swift.metadata_checksum', new_checksum)
else:
if debug:
new_checksum = hashlib.md5(xattr.get(directory + "/" + filename,p)).hexdigest()
print("CHKSUM: %s %s" % (xattr.get(directory + "/" + filename, 'user.swift.metadata_checksum'), new_checksum))
except:
raise
else:
print("%s didn't match %s" % ( matchchecksum, s ))
def test_xattrs_with_no_serialized_value_are_kept(self):
dumbattr.set(self.file1, "test1", "1")
xattr.set(self.file1, "test2", "2", namespace=xattr.NS_USER)
self.assertEqual(self.serialized_metadata(), {'file1': {'test1':'1'}})
self.assertEqual(dumbattr.load(self.file1).copy(), {'test1': '1', 'test2':'2'})
self.assertEqual(self.serialized_metadata(), {'file1': {'test1': '1', 'test2':'2'}})
def _copyxattr(src, dest, exclude=None):
try:
attrs = xattr.list(src)
except IOError as e:
if e.errno != OperationNotSupported.errno:
raise
attrs = ()
if attrs:
if exclude is not None and isinstance(attrs[0], bytes):
exclude = exclude.encode(_encodings['fs'])
exclude = _get_xattr_excluder(exclude)
for attr in attrs:
if exclude(attr):
continue
try:
xattr.set(dest, attr, xattr.get(src, attr))
raise_exception = False
except IOError:
raise_exception = True
if raise_exception:
raise OperationNotSupported(
_("Filesystem containing file '%s' "
"does not support extended attribute '%s'") %
(_unicode_decode(dest), _unicode_decode(attr)))
def test_set_get_remove(subject, use_ns):
item = subject[0]
lists_equal(xattr.list(item), [])
if use_ns:
xattr.set(item, USER_NN, USER_VAL,
namespace=NAMESPACE)
else:
xattr.set(item, USER_ATTR, USER_VAL)
if use_ns:
assert xattr.list(item, namespace=NAMESPACE) == [USER_NN]
else:
lists_equal(xattr.list(item), [USER_ATTR])
lists_equal(xattr.list(item, namespace=EMPTY_NS),
[USER_ATTR])
if use_ns:
assert xattr.get(item, USER_NN, namespace=NAMESPACE) == USER_VAL
else:
assert xattr.get(item, USER_ATTR) == USER_VAL
if use_ns:
assert xattr.get_all(item, namespace=NAMESPACE) == \
[(USER_NN, USER_VAL)]
else:
tuples_equal(xattr.get_all(item),
[(USER_ATTR, USER_VAL)])
if use_ns:
xattr.remove(item, USER_NN, namespace=NAMESPACE)
else:
xattr.remove(item, USER_ATTR)
lists_equal(xattr.list(item), [])
tuples_equal(xattr.get_all(item), [])
def preserve_xattrs(path, nofollow=False, namespace=None):
"""Context manager to save/restore extended attributes on |path|
If you want to rewrite a file (possibly replacing it with a new one), but
want to preserve the extended attributes, this will do the trick.
# First read all the extended attributes.
with save_xattrs('/some/file'):
... rewrite the file ...
# Now the extended attributes are restored as needed.
"""
kwargs = {'nofollow': nofollow,}
if namespace:
# Compiled xattr python module does not like it when namespace=None.
kwargs['namespace'] = namespace
old_attrs = dict(xattr.get_all(path, **kwargs))
try:
yield
finally:
new_attrs = dict(xattr.get_all(path, **kwargs))
for name, value in new_attrs.items():
if name not in old_attrs:
# Clear out new ones.
xattr.remove(path, name, **kwargs)
elif new_attrs[name] != old_attrs[name]:
# Update changed ones.
xattr.set(path, name, value, **kwargs)
for name, value in old_attrs.items():
if name not in new_attrs:
# Re-add missing ones.
xattr.set(path, name, value, **kwargs)
def _copyxattr(src, dest, exclude=None):
try:
attrs = xattr.list(src)
except IOError as e:
if e.errno != OperationNotSupported.errno:
raise
attrs = ()
if attrs:
if exclude is not None and isinstance(attrs[0], bytes):
exclude = exclude.encode(_encodings['fs'])
exclude = _get_xattr_excluder(exclude)
for attr in attrs:
if exclude(attr):
continue
try:
xattr.set(dest, attr, xattr.get(src, attr))
raise_exception = False
except IOError:
raise_exception = True
if raise_exception:
raise OperationNotSupported(_("Filesystem containing file '%s' "
"does not support extended attribute '%s'") %
(_unicode_decode(dest), _unicode_decode(attr)))
def xattr_sync(file_name, meta):
""" Synchronize the xattrs on a file with a hash of our choosing """
to_delete, to_set = xattr_sync_impl(file_name, meta)
for k in to_delete:
xattr.remove(file_name, k)
for k, v in to_set.items():
xattr.set(file_name, k, v, namespace=xattr.NS_USER)
def xattr_sync(file_name, meta):
""" Synchronize the xattrs on a file with a hash of our choosing """
to_delete, to_set = xattr_sync_impl(file_name, meta)
for k in to_delete:
xattr.remove(file_name, k)
for k,v in to_set.items():
xattr.set(file_name, k, v, namespace=xattr.NS_USER)
def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
if not xattr:
if self.linux_xattr:
add_error("%s: can't restore xattr; xattr support missing.\n"
% path)
return
existing_xattrs = set(xattr.list(path, nofollow=True))
if self.linux_xattr:
for k, v in self.linux_xattr:
if k not in existing_xattrs \
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM \
or e.errno == errno.EOPNOTSUPP:
raise ApplyError('xattr.set: %s' % e)
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.remove: %s' % e)
else:
raise
def _fix(self):
logger.debug("Fixing records for %s", self)
files = Set(os.listdir(self.dirpath))
attrs = {}
for filename in files:
path = os.path.join(self.dirpath, filename)
if os.path.islink(path):
# just copy symlink attrs, as they can't be placed on a symlink
try:
attrs[filename] = self._saved_attrs[filename]
except KeyError: pass # no entry in self._saved_attrs
continue
try:
recovered_attrs = self._saved_attrs[filename]
except KeyError: pass
else:
file_attrs = self._get_xattrs(filename)
logger.debug("xattrs for %s: %r", filename, file_attrs)
for key, val in recovered_attrs.items():
xattr_val = file_attrs.get(key, None)
if xattr_val != val:
logger.info("File %s has xattr %r=%s, but serialized data has %s - using serialized data", filename, key, xattr_val, val)
xattr.set(path, key, val, namespace=xattr.NS_USER)
self._update_saved_attrs(filename, dest=attrs)
removed_files = Set(self._saved_attrs.keys()).difference(Set(attrs.keys()))
for filename in removed_files:
logger.debug("Dropping metadata for missing file %s: %r", os.path.join(self.dirpath, filename), self._saved_attrs[filename])
if attrs != self._saved_attrs:
self._update(attrs)
def test_mixed_access(testdir, gen):
"""test mixed access to file"""
with gen(testdir) as (a, b):
# Check empty
lists_equal(xattr.list(a), [])
lists_equal(xattr.listxattr(b), [])
# Check value
xattr.set(a, USER_ATTR, USER_VAL)
for i in [a, b]:
# Deprecated functions
lists_equal(xattr.listxattr(i), [USER_ATTR])
assert xattr.getxattr(i, USER_ATTR) == USER_VAL
tuples_equal(xattr.get_all(i), [(USER_ATTR, USER_VAL)])
# Current functions
lists_equal(xattr.list(i), [USER_ATTR])
assert xattr.list(i, namespace=NAMESPACE) == [USER_NN]
assert xattr.get(i, USER_ATTR) == USER_VAL
assert xattr.get(i, USER_NN, namespace=NAMESPACE) == USER_VAL
tuples_equal(xattr.get_all(i),
[(USER_ATTR, USER_VAL)])
assert xattr.get_all(i, namespace=NAMESPACE) == \
[(USER_NN, USER_VAL)]
# Overwrite
xattr.set(b, USER_ATTR, LARGE_VAL, flags=xattr.XATTR_REPLACE)
assert xattr.get(a, USER_ATTR) == LARGE_VAL
assert xattr.getxattr(a, USER_ATTR) == LARGE_VAL
xattr.removexattr(b, USER_ATTR)
assert xattr.get_all(a, namespace=NAMESPACE) == []
assert xattr.get_all(b, namespace=NAMESPACE) == []
def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
if not xattr:
if self.linux_xattr:
add_error("%s: can't restore xattr; xattr support missing.\n" %
path)
return
existing_xattrs = set(xattr.list(path, nofollow=True))
if self.linux_xattr:
for k, v in self.linux_xattr:
if k not in existing_xattrs \
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.set: %s' % e)
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.remove: %s' % e)
else:
raise
def test_xattrs_overriden_by_serialized_attrs(self):
dumbattr.set(self.file1, "test1", "1")
xattr.set(self.file1, "test1", "2", namespace=xattr.NS_USER)
self.assertEqual(self.serialized_metadata(), {'file1': {'test1':'1'}})
# load() has the side effect of fixing any discrepancies, just like `fix()`
self.assertEqual(dumbattr.load(self.file1).copy(), {'test1': '1'})
self.assertEqual(self.all_xattrs(self.file1), {b'test1': b'1'})
def put(self, key: str, stream) -> None:
f = NamedTemporaryFile(dir=self.tempdir, delete=False)
# TODO: read in chunks to save ram
# Save real name in xattr which will help rebuild db.
xattr.set(f.name, "user.key", key)
f.write(stream.read())
# TODO: check hash
os.rename(f.name, self.keytopath(key))
def setxattr(self, key, value, namespace=XATTR_DEFAULT_NS):
if not isinstance(value, bytes): # checksums
raise TypeError('setxattr only accepts values already '
f'encoded to bytes!\n{value!r}')
else:
bytes_value = value
xattr.set(self.as_posix(), key, bytes_value, namespace=namespace)
def _checkListSetGet(self, item, symlink=False, use_ns=False):
"""check list, set, get operations against an item"""
self.assertEqual(self._ignore(xattr.list(item, symlink)),
[])
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_ATTR, self.USER_VAL, flags=XATTR_REPLACE)
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_NN, self.USER_VAL, flags=XATTR_REPLACE,
namespace=NS_USER)
try:
if use_ns:
xattr.set(item, self.USER_NN, self.USER_VAL,
namespace=NS_USER,
nofollow=symlink)
else:
xattr.set(item, self.USER_ATTR, self.USER_VAL,
nofollow=symlink)
except IOError:
err = sys.exc_info()[1]
if err.errno == errno.EPERM and symlink:
# symlinks may fail, in which case we abort the rest
# of the test for this case
return
raise
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_ATTR, self.USER_VAL, flags=XATTR_CREATE)
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_NN, self.USER_VAL,
flags=XATTR_CREATE, namespace=NS_USER)
self.assertEqual(self._ignore(xattr.list(item, nofollow=symlink)),
[self.USER_ATTR])
self.assertEqual(self._ignore(xattr.list(item, nofollow=symlink,
namespace=EMPTY_NS)),
[self.USER_ATTR])
self.assertEqual(xattr.list(item, namespace=NS_USER, nofollow=symlink),
[self.USER_NN])
self.assertEqual(xattr.get(item, self.USER_ATTR, nofollow=symlink),
self.USER_VAL)
self.assertEqual(xattr.get(item, self.USER_NN, nofollow=symlink,
namespace=NS_USER), self.USER_VAL)
self.assertEqual(self._ignore_tuples(xattr.get_all(item,
nofollow=symlink)),
[(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(item, nofollow=symlink,
namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
if use_ns:
xattr.remove(item, self.USER_NN, namespace=NS_USER)
else:
xattr.remove(item, self.USER_ATTR)
self.assertEqual(self._ignore(xattr.list(item, symlink)), [])
self.assertEqual(self._ignore_tuples(xattr.get_all(item,
nofollow=symlink)),
[])
self.assertRaises(EnvironmentError, xattr.remove,
item, self.USER_ATTR, nofollow=symlink)
self.assertRaises(EnvironmentError, xattr.remove, item,
self.USER_NN, namespace=NS_USER, nofollow=symlink)
def test_symlinks_user_fail(testdir, use_dangling):
_, sname = get_symlink(testdir, dangling=use_dangling)
with pytest.raises(IOError):
xattr.set(sname, USER_ATTR, USER_VAL, nofollow=True)
with pytest.raises(IOError):
xattr.set(sname, USER_NN, USER_VAL, namespace=NAMESPACE,
nofollow=True)
with pytest.raises(IOError):
xattr.setxattr(sname, USER_ATTR, USER_VAL, XATTR_CREATE, True)
def test_replace_on_missing(subject, use_ns):
item = subject[0]
lists_equal(xattr.list(item), [])
with pytest.raises(EnvironmentError):
if use_ns:
xattr.set(item, USER_NN, USER_VAL, flags=XATTR_REPLACE,
namespace=NAMESPACE)
else:
xattr.set(item, USER_ATTR, USER_VAL, flags=XATTR_REPLACE)
def _copyxattr(src, dest):
for attr in xattr.list(src):
try:
xattr.set(dest, attr, xattr.get(src, attr))
raise_exception = False
except IOError:
raise_exception = True
if raise_exception:
raise OperationNotSupported("Filesystem containing file '%s' does not support extended attributes" % dest)
def _checkListSetGet(self, item, symlink=False, use_ns=False):
"""check list, set, get operations against an item"""
self.checkList(xattr.list(item, symlink), [])
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_ATTR, self.USER_VAL, flags=XATTR_REPLACE)
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_NN, self.USER_VAL, flags=XATTR_REPLACE,
namespace=NS_USER)
try:
if use_ns:
xattr.set(item, self.USER_NN, self.USER_VAL,
namespace=NS_USER,
nofollow=symlink)
else:
xattr.set(item, self.USER_ATTR, self.USER_VAL,
nofollow=symlink)
except IOError:
err = sys.exc_info()[1]
if symlink and (err.errno == errno.EPERM or
err.errno == errno.ENOENT):
# symlinks may fail, in which case we abort the rest
# of the test for this case (Linux returns EPERM; OS X
# returns ENOENT)
return
raise
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_ATTR, self.USER_VAL, flags=XATTR_CREATE)
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_NN, self.USER_VAL,
flags=XATTR_CREATE, namespace=NS_USER)
self.checkList(xattr.list(item, nofollow=symlink), [self.USER_ATTR])
self.checkList(xattr.list(item, nofollow=symlink,
namespace=EMPTY_NS),
[self.USER_ATTR])
self.assertEqual(xattr.list(item, namespace=NS_USER, nofollow=symlink),
[self.USER_NN])
self.assertEqual(xattr.get(item, self.USER_ATTR, nofollow=symlink),
self.USER_VAL)
self.assertEqual(xattr.get(item, self.USER_NN, nofollow=symlink,
namespace=NS_USER), self.USER_VAL)
self.checkTuples(xattr.get_all(item, nofollow=symlink),
[(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(item, nofollow=symlink,
namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
if use_ns:
xattr.remove(item, self.USER_NN, namespace=NS_USER)
else:
xattr.remove(item, self.USER_ATTR)
self.checkList(xattr.list(item, symlink), [])
self.checkTuples(xattr.get_all(item, nofollow=symlink),
[])
self.assertRaises(EnvironmentError, xattr.remove,
item, self.USER_ATTR, nofollow=symlink)
self.assertRaises(EnvironmentError, xattr.remove, item,
self.USER_NN, namespace=NS_USER, nofollow=symlink)
def write_cache(self):
if not xattr:
return
try:
self.clear_cache()
xattr.set(self.name, 'user.pyanidb.mtime', str(int(self.mtime)))
for n in ('ed2k', 'md5', 'sha1', 'crc32'):
if hasattr(self, n):
xattr.set(self.name, 'user.pyanidb.' + n, getattr(self, n))
except IOError:
pass
def set_volume_id_xattr():
"""Set Volume ID xattr"""
try:
xattr.set(BRICK_PATH, VOLUME_ID_XATTR_NAME, VOLUME_ID_BYTES,
xattr.XATTR_CREATE)
except FileExistsError:
pass
except OSError as err:
sys.stderr.write("Unable to set volume-id on "
"brick root: %s\n" % err)
sys.exit(1)
def write_cache(self):
if not xattr:
return
try:
self.clear_cache()
xattr.set(self.name, 'user.pyanidb.mtime', str(int(self.mtime)))
for n in ('ed2k', 'md5', 'sha1', 'crc32'):
if hasattr(self, n):
xattr.set(self.name, 'user.pyanidb.' + n, getattr(self, n))
except IOError:
pass
def _copyxattr(src, dest):
for attr in xattr.list(src):
try:
xattr.set(dest, attr, xattr.get(src, attr))
raise_exception = False
except IOError:
raise_exception = True
if raise_exception:
raise OperationNotSupported(
"Filesystem containing file '%s' does not support extended attributes"
% dest)
def xattrsave(ldoff, txtbuf, xattrname):
global loadfile
if loadphase[ldoff] != False:
#print('loadphase set')
return
if loadfile is None:
#print('loadfile none')
return
attrtxt = txtbuf.get_text().encode('utf8')
xattr.set(loadfile, xattrname, attrtxt)
print('%s:%s=%s' % (loadfile, xattrname, attrtxt))
def check_user_xattr(path):
if not os.path.exists(path):
return False
try:
xattr.set(path, 'user.test.key1', 'value1')
except IOError as err:
logging.exception("check_user_xattr: set failed on %s err: %s", path, str(err))
raise
try:
xattr.remove(path, 'user.test.key1')
except Exception, err:
logging.exception("xattr.remove failed on %s err: %s", path, str(err))
def save_license(license_text, license_signature):
license_file = os.path.join(get_oscar_dir(), "etc/license.txt")
with open(license_file, "w") as f:
f.write(license_text + '\n')
f.write(license_signature + '\n')
license_file = os.path.join(get_oscar_dir(), "bin/oscar")
try:
xattr.set(license_file, "user.oscar.license", license_text)
xattr.set(license_file, "user.oscar.license.signature", license_signature)
except IOError, e:
if e.errno != errno.ENODATA: raise e
def testSymlinkOps(self):
"""test symlink operations"""
_, sname = self._getsymlink()
self.assertRaises(EnvironmentError, xattr.list, sname)
self._checkListSetGet(sname, symlink=True)
self._checkListSetGet(sname, symlink=True, use_ns=True)
target, sname = self._getsymlink(dangling=False)
xattr.set(target, self.USER_ATTR, self.USER_VAL)
self.assertEqual(xattr.list(target), [self.USER_ATTR])
self.assertEqual(xattr.list(sname, nofollow=True), [])
self.assertRaises(EnvironmentError, xattr.remove, sname, self.USER_ATTR, nofollow=True)
xattr.remove(sname, self.USER_ATTR, nofollow=False)
def test_binary_payload(subject):
"""test binary values"""
item = subject[0]
BINVAL = b"abc\0def"
xattr.set(item, USER_ATTR, BINVAL)
lists_equal(xattr.list(item), [USER_ATTR])
assert xattr.list(item, namespace=NAMESPACE) == [USER_NN]
assert xattr.get(item, USER_ATTR) == BINVAL
assert xattr.get(item, USER_NN, namespace=NAMESPACE) == BINVAL
tuples_equal(xattr.get_all(item), [(USER_ATTR, BINVAL)])
assert xattr.get_all(item, namespace=NAMESPACE) == [(USER_NN, BINVAL)]
xattr.remove(item, USER_ATTR)
def testSymlinkOps(self):
"""test symlink operations"""
_, sname = self._getsymlink()
self.assertRaises(EnvironmentError, xattr.list, sname)
self._checkListSetGet(sname, symlink=True)
self._checkListSetGet(sname, symlink=True, use_ns=True)
target, sname = self._getsymlink(dangling=False)
xattr.set(target, self.USER_ATTR, self.USER_VAL)
self.checkList(xattr.list(target), [self.USER_ATTR])
self.checkList(xattr.list(sname, nofollow=True), [])
self.assertRaises(EnvironmentError, xattr.remove, sname,
self.USER_ATTR, nofollow=True)
xattr.remove(sname, self.USER_ATTR, nofollow=False)
def verify_brickdir_xattr_support():
"""Verify Brick dir supports xattrs"""
try:
xattr.set(BRICK_PATH, TEST_XATTR_NAME, TEST_XATTR_VALUE)
val = xattr.get(BRICK_PATH, TEST_XATTR_NAME)
if val != TEST_XATTR_VALUE:
sys.stderr.write("Xattr value mismatch. Actual=%s Expected=%s\n" %
(val, TEST_XATTR_VALUE))
sys.exit(1)
except OSError as err:
sys.stderr.write("Extended attributes are not "
"supported: %s\n" % err)
sys.exit(1)
def check_user_xattr(path):
if not os.path.exists(path):
return False
try:
xattr.set(path, 'user.test.key1', 'value1')
except IOError as err:
logging.exception("check_user_xattr: set failed on %s err: %s", path, str(err))
raise
try:
xattr.remove(path, 'user.test.key1')
except IOError as err:
logging.exception("check_user_xattr: remove failed on %s err: %s", path, str(err))
#Remove xattr may fail in case of concurrent remove.
return True
def set_volume_id_xattr(brick_path, volume_id):
"""Set Volume ID xattr"""
volume_id_bytes = uuid.UUID(volume_id).bytes
try:
xattr.set(brick_path, VOLUME_ID_XATTR_NAME,
volume_id_bytes, xattr.XATTR_CREATE)
except FileExistsError:
pass
except OSError as err:
logging.error(logf("Unable to set volume-id on "
"brick root",
error=err))
sys.exit(1)
def tmp_path_xattrs_ok(tmp_path_factory):
try:
import xattr # type: ignore
except ImportError:
pytest.skip("xattr module not available")
tmpp = tmp_path_factory.mktemp("needs_xattrs")
try:
xattr.set(str(tmpp), "user.deleteme", "1")
xattr.remove(str(tmpp), "user.deleteme")
except OSError:
raise pytest.skip(
"temp dir does not support xattrs"
" (try changing basetmp to a file system that supports xattrs)")
return tmpp
def testBinaryPayload(self):
"""test binary values"""
fh, fname = self._getfile()
os.close(fh)
BINVAL = "abc" + "\0" + "def"
if PY3K:
BINVAL = BINVAL.encode()
xattr.set(fname, self.USER_ATTR, BINVAL)
self.assertEqual(xattr.list(fname), [self.USER_ATTR])
self.assertEqual(xattr.list(fname, namespace=NS_USER), [self.USER_NN])
self.assertEqual(xattr.get(fname, self.USER_ATTR), BINVAL)
self.assertEqual(xattr.get(fname, self.USER_NN, namespace=NS_USER), BINVAL)
self.assertEqual(xattr.get_all(fname), [(self.USER_ATTR, BINVAL)])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER), [(self.USER_NN, BINVAL)])
xattr.remove(fname, self.USER_ATTR)
def testMixedAccess(self):
"""test mixed access to file"""
fh, fname = self._getfile()
fo = os.fdopen(fh)
self.assertEqual(xattr.list(fname), [])
xattr.set(fname, self.USER_ATTR, self.USER_VAL)
self.assertEqual(xattr.list(fh), [self.USER_ATTR])
self.assertEqual(xattr.list(fh, namespace=NS_USER), [self.USER_NN])
self.assertEqual(xattr.get(fo, self.USER_ATTR), self.USER_VAL)
self.assertEqual(xattr.get(fo, self.USER_NN, namespace=NS_USER), self.USER_VAL)
self.assertEqual(xattr.get_all(fo), [(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fo, namespace=NS_USER), [(self.USER_NN, self.USER_VAL)])
self.assertEqual(xattr.get_all(fname), [(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER), [(self.USER_NN, self.USER_VAL)])
fo.close()
def check_user_xattr(path):
if not os.path.exists(path):
return False
try:
xattr.set(path, 'user.test.key1', 'value1')
except IOError as err:
logging.exception("check_user_xattr: set failed on %s err: %s", path,
str(err))
raise
try:
xattr.remove(path, 'user.test.key1')
except IOError as err:
logging.exception("check_user_xattr: remove failed on %s err: %s",
path, str(err))
#Remove xattr may fail in case of concurrent remove.
return True
def testManyOps(self):
"""test many ops"""
fh, fname = self._getfile()
xattr.set(fh, self.USER_ATTR, self.USER_VAL)
VL = [self.USER_ATTR]
VN = [self.USER_NN]
for i in range(self.MANYOPS_COUNT):
self.assertEqual(xattr.list(fh), VL)
self.assertEqual(xattr.list(fh, namespace=EMPTY_NS), VL)
self.assertEqual(xattr.list(fh, namespace=NS_USER), VN)
for i in range(self.MANYOPS_COUNT):
self.assertEqual(xattr.get(fh, self.USER_ATTR), self.USER_VAL)
self.assertEqual(xattr.get(fh, self.USER_NN, namespace=NS_USER), self.USER_VAL)
for i in range(self.MANYOPS_COUNT):
self.assertEqual(xattr.get_all(fh), [(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fh, namespace=NS_USER), [(self.USER_NN, self.USER_VAL)])
def write_metadata(path, metadata):
"""
Helper function to write pickled metadata for a File/Directory.
:param path: File/Directory path to write the metadata
:param metadata: metadata to write
"""
metastr = pickle.dumps(metadata, PICKLE_PROTOCOL)
key = 0
while metastr:
try:
xattr.set(path, '%s%s' % (METADATA_KEY, key or ''), metastr[:254])
except IOError as err:
logging.exception("xattr.set failed on %s key %s err: %s", path, key, str(err))
raise
metastr = metastr[254:]
key += 1
def testBinaryPayload(self):
"""test binary values"""
fh, fname = self._getfile()
os.close(fh)
BINVAL = "abc" + '\0' + "def"
if PY3K:
BINVAL = BINVAL.encode()
xattr.set(fname, self.USER_ATTR, BINVAL)
self.checkList(xattr.list(fname), [self.USER_ATTR])
self.assertEqual(xattr.list(fname, namespace=NS_USER), [self.USER_NN])
self.assertEqual(xattr.get(fname, self.USER_ATTR), BINVAL)
self.assertEqual(xattr.get(fname, self.USER_NN, namespace=NS_USER),
BINVAL)
self.checkTuples(xattr.get_all(fname), [(self.USER_ATTR, BINVAL)])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER),
[(self.USER_NN, BINVAL)])
xattr.remove(fname, self.USER_ATTR)
def verify_brickdir_xattr_support(brick_path):
"""Verify Brick dir supports xattrs"""
test_xattr_name = "user.testattr"
test_xattr_value = b"testvalue"
try:
xattr.set(brick_path, test_xattr_name, test_xattr_value)
val = xattr.get(brick_path, test_xattr_name)
if val != test_xattr_value:
logging.error(logf("Xattr value mismatch.",
actual=val,
expected=test_xattr_value))
sys.exit(1)
except OSError as err:
logging.error(logf("Extended attributes are not "
"supported",
error=err))
sys.exit(1)
def test_many_ops(subject):
"""test many ops"""
item = subject[0]
xattr.set(item, USER_ATTR, USER_VAL)
VL = [USER_ATTR]
VN = [USER_NN]
for i in range(MANYOPS_COUNT):
lists_equal(xattr.list(item), VL)
lists_equal(xattr.list(item, namespace=EMPTY_NS), VL)
assert xattr.list(item, namespace=NAMESPACE) == VN
for i in range(MANYOPS_COUNT):
assert xattr.get(item, USER_ATTR) == USER_VAL
assert xattr.get(item, USER_NN, namespace=NAMESPACE) == USER_VAL
for i in range(MANYOPS_COUNT):
tuples_equal(xattr.get_all(item),
[(USER_ATTR, USER_VAL)])
assert xattr.get_all(item, namespace=NAMESPACE) == \
[(USER_NN, USER_VAL)]
def process(path):
# get tags
try: attrvalue = xattr.get(path, ATTRNAME)
except IOError: tags = set();
else: tags = set(attrvalue.split("\x1f"))
oldtags = tags
# perform operations
if options.filter and not set(options.filter) <= tags: return
if options.clear: tags = []
if options.add: tags = tags | set(options.add)
if options.delete: tags = tags - set(options.delete)
if not tags == oldtags:
if tags:
xattr.set(path, ATTRNAME, "\x1f".join(tags))
else:
try: xattr.remove(path, ATTRNAME)
except IOError: pass
if options.list: print path, "::", ", ".join(tags)
def write_metadata(path, metadata):
"""
Helper function to write pickled metadata for a File/Directory.
:param path: File/Directory path to write the metadata
:param metadata: dictionary to metadata write
"""
assert isinstance(metadata, dict)
metastr = pickle.dumps(metadata, PICKLE_PROTOCOL)
key = 0
while metastr:
try:
xattr.set(path, '%s%s' % (METADATA_KEY, key or ''), metastr[:MAX_XATTR_SIZE])
except IOError as err:
logging.exception("xattr.set failed on %s key %s err: %s", path, key, str(err))
raise
metastr = metastr[MAX_XATTR_SIZE:]
key += 1
def testMixedAccess(self):
"""test mixed access to file"""
fh, fname = self._getfile()
fo = os.fdopen(fh)
self.checkList(xattr.list(fname), [])
xattr.set(fname, self.USER_ATTR, self.USER_VAL)
self.checkList(xattr.list(fh), [self.USER_ATTR])
self.assertEqual(xattr.list(fh, namespace=NS_USER), [self.USER_NN])
self.assertEqual(xattr.get(fo, self.USER_ATTR), self.USER_VAL)
self.assertEqual(xattr.get(fo, self.USER_NN, namespace=NS_USER),
self.USER_VAL)
self.checkTuples(xattr.get_all(fo), [(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fo, namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
self.checkTuples(xattr.get_all(fname),
[(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
fo.close()
def testManyOps(self):
"""test many ops"""
fh, fname = self._getfile()
xattr.set(fh, self.USER_ATTR, self.USER_VAL)
VL = [self.USER_ATTR]
VN = [self.USER_NN]
for i in range(self.MANYOPS_COUNT):
self.checkList(xattr.list(fh), VL)
self.checkList(xattr.list(fh, namespace=EMPTY_NS), VL)
self.assertEqual(xattr.list(fh, namespace=NS_USER), VN)
for i in range(self.MANYOPS_COUNT):
self.assertEqual(xattr.get(fh, self.USER_ATTR), self.USER_VAL)
self.assertEqual(xattr.get(fh, self.USER_NN, namespace=NS_USER),
self.USER_VAL)
for i in range(self.MANYOPS_COUNT):
self.checkTuples(xattr.get_all(fh),
[(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fh, namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
def savemessageflags(self, uid, flags):
"""Sets the specified message's flags to the given set.
This function moves the message to the cur or new subdir,
depending on the 'S'een flag.
Note that this function does not check against dryrun settings,
so you need to ensure that it is never called in a
dryrun mode."""
oldfilename = self.messagelist[uid]['filename']
dir_prefix, filename = os.path.split(oldfilename)
# If a message has been seen, it goes into 'cur'
dir_prefix = 'cur' if '\\Seen' in flags else 'new'
if flags != self.messagelist[uid]['flags']:
# Flags have actually changed, construct new filename Strip
# off existing infostring (possibly discarding small letter
# flags that dovecot uses TODO)
infomatch = self.re_flagmatch.search(filename)
if infomatch:
filename = filename[:-len(infomatch.group())] #strip off
infostr = '%s2,%s' % (self.infosep, ''.join(
sorted(imaputil.flagsimap2maildir(flags))))
filename += infostr
newfilename = os.path.join(dir_prefix, filename)
if (newfilename != oldfilename):
try:
os.rename(os.path.join(self.getfullname(), oldfilename),
os.path.join(self.getfullname(), newfilename))
except OSError as e:
raise OfflineImapError(
"Can't rename file '%s' to '%s': %s" %
(oldfilename, newfilename, e[1]),
OfflineImapError.ERROR.FOLDER)
self.messagelist[uid]['filename'] = newfilename
self.messagelist[uid]['flags'] = flags
xattr.set(os.path.join(self.getfullname(), newfilename),
'org.offlineimap.flags',
' '.join(flags),
namespace=xattr.NS_USER)
def get_object_uuid(real_path):
fd = os.open(real_path, os.O_RDONLY)
fcntl.flock(fd, fcntl.LOCK_EX)
try:
try:
object_uuid = xattr.get(real_path, xattr_name)
except IOError, e:
if e.errno != errno.ENODATA:
logger.exception("xattr.get:%s" % real_path)
raise e
object_uuid = uuid.uuid4().hex
try:
mode = os.fstat(fd).st_mode
if not (mode & stat.S_IWUSR): # read only は強制解除
os.fchmod(fd, mode | stat.S_IWUSR)
xattr.set(real_path, xattr_name, object_uuid)
except:
logger.exception("xattr.set:%s" % real_path)
raise
return object_uuid
def _set(self, filename, key, value):
path = os.path.join(self.dirpath, filename)
logger.info("Setting %s=%s (%s)", key, value, path)
if os.path.islink(path):
if value is None:
h = self._saved_attrs[filename]
del h[key]
if not h:
del self._saved_attrs[filename]
else:
if not filename in self._saved_attrs:
self._saved_attrs[filename] = {}
self._saved_attrs[filename][key] = value
else:
logger.debug("Setting xattr %s=%s (%s)", key, value, path)
if value is None:
xattr.remove(path, key, namespace=xattr.NS_USER)
else:
xattr.set(path, key, value, namespace=xattr.NS_USER)
self._update_saved_attrs(filename)
def write_metadata(path, metadata):
"""
Helper function to write pickled metadata for a File/Directory.
:param path: File/Directory path to write the metadata
:param metadata: dictionary to metadata write
"""
assert isinstance(metadata, dict)
metastr = pickle.dumps(metadata, PICKLE_PROTOCOL)
key = 0
while metastr:
try:
xattr.set(path, '%s%s' % (METADATA_KEY, key or ''),
metastr[:MAX_XATTR_SIZE])
except IOError as err:
logging.exception("xattr.set failed on %s key %s err: %s", path,
key, str(err))
raise
metastr = metastr[MAX_XATTR_SIZE:]
key += 1
def restore_xattrs(file_in):
"""Read |file_in| and restore xattrs content from it
This expects textual data in the format written by dump_xattrs.
"""
pathname = None
for i, line in enumerate(file_in):
if line.startswith(b'# file: '):
pathname = unquote(line.rstrip(b'\n')[8:])
else:
parts = line.split(b'=', 1)
if len(parts) == 2:
if pathname is None:
raise ValueError('line %d: missing pathname' % (i + 1,))
attr = unquote(parts[0])
# strip trailing newline and quotes
value = unquote(parts[1].rstrip(b'\n')[1:-1])
xattr.set(pathname, attr, value)
elif line.strip():
raise ValueError('line %d: malformed entry' % (i + 1,))
def savemessageflags(self, uid, flags):
"""Sets the specified message's flags to the given set.
This function moves the message to the cur or new subdir,
depending on the 'S'een flag.
Note that this function does not check against dryrun settings,
so you need to ensure that it is never called in a
dryrun mode."""
oldfilename = self.messagelist[uid]['filename']
dir_prefix, filename = os.path.split(oldfilename)
# If a message has been seen, it goes into 'cur'
dir_prefix = 'cur' if '\\Seen' in flags else 'new'
if flags != self.messagelist[uid]['flags']:
# Flags have actually changed, construct new filename Strip
# off existing infostring (possibly discarding small letter
# flags that dovecot uses TODO)
infomatch = self.re_flagmatch.search(filename)
if infomatch:
filename = filename[:-len(infomatch.group())] #strip off
infostr = '%s2,%s' % (self.infosep,
''.join(sorted(imaputil.flagsimap2maildir(flags))))
filename += infostr
newfilename = os.path.join(dir_prefix, filename)
if (newfilename != oldfilename):
try:
os.rename(os.path.join(self.getfullname(), oldfilename),
os.path.join(self.getfullname(), newfilename))
except OSError as e:
raise OfflineImapError("Can't rename file '%s' to '%s': %s" % (
oldfilename, newfilename, e[1]),
OfflineImapError.ERROR.FOLDER)
self.messagelist[uid]['filename'] = newfilename
self.messagelist[uid]['flags'] = flags
xattr.set(os.path.join(self.getfullname(), newfilename),
'org.offlineimap.flags', ' '.join(flags),
namespace=xattr.NS_USER)
def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
existing_xattrs = set(xattr.list(path, nofollow=True))
if(self.linux_xattr):
for k, v in self.linux_xattr:
if k not in existing_xattrs \
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.set: %s' % e)
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.remove: %s' % e)
else:
raise
def test_loading_attr_ensures_all_attrs_in_dir_are_serialized(self):
xattr.set(self.file1, "file_1", "1", namespace=xattr.NS_USER)
xattr.set(self.file1, "file_2", "2", namespace=xattr.NS_USER)
xattr.set(self.dir, "dir_1", "1", namespace=xattr.NS_USER)
xattr.set(self.link_target, "target", "1", namespace=xattr.NS_USER)
# can't set any xattrs on link itself
self.assertEqual(self.serialized_metadata(), None)
dumbattr.fix(self.base)
self.assertEqual(self.all_xattrs(self.file1), {b"file_1":b"1", b"file_2": b"2"})
self.assertEqual(self.all_xattrs(self.dir), {b"dir_1":b"1"})
self.assertEqual(self.all_xattrs(self.link_target), {b"target":b"1"})
self.assertEqual(self.serialized_metadata(), {
"file1": {"file_1":"1", "file_2": "2"},
"directory": {"dir_1":"1"},
"link_target": {"target":"1"},
})
def test_handling_of_incorrect_existing_linux_xattrs():
if os.geteuid() != 0 or detect_fakeroot():
return
setup_testfs()
subprocess.check_call('rm -rf testfs/*', shell=True)
path = 'testfs/foo'
open(path, 'w').close()
xattr.set(path, 'foo', 'bar', namespace=xattr.NS_USER)
m = metadata.from_path(path, archive_path=path, save_symlinks=True)
xattr.set(path, 'baz', 'bax', namespace=xattr.NS_USER)
m.apply_to_path(path, restore_numeric_ids=False)
WVPASSEQ(xattr.list(path), ['user.foo'])
WVPASSEQ(xattr.get(path, 'user.foo'), 'bar')
xattr.set(path, 'foo', 'baz', namespace=xattr.NS_USER)
m.apply_to_path(path, restore_numeric_ids=False)
WVPASSEQ(xattr.list(path), ['user.foo'])
WVPASSEQ(xattr.get(path, 'user.foo'), 'bar')
xattr.remove(path, 'foo', namespace=xattr.NS_USER)
m.apply_to_path(path, restore_numeric_ids=False)
WVPASSEQ(xattr.list(path), ['user.foo'])
WVPASSEQ(xattr.get(path, 'user.foo'), 'bar')
os.chdir(top_dir)
cleanup_testfs()
def __set(self,path,key,value): xattr.set(path,key,value,namespace=self.ns)
if self.linux_xattr:
add_error("%s: can't restore xattr; xattr support missing.\n" % path)
return
if not self.linux_xattr:
return
try:
existing_xattrs = set(xattr.list(path, nofollow=True))
except IOError, e:
if e.errno == errno.EACCES:
raise ApplyError("xattr.set: %s" % e)
else:
raise
for k, v in self.linux_xattr:
if k not in existing_xattrs or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM or e.errno == errno.EOPNOTSUPP:
raise ApplyError("xattr.set: %s" % e)
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError("xattr.remove: %s" % e)
else:
raise
def write_xattr(path, key, value):
try:
xattr.set(path, key, value)
except EnvironmentError as e:
raise XAttrMetadataError(e.errno, e.strerror)
