def test_set_get_remove(subject, use_ns):
item = subject[0]
lists_equal(xattr.list(item), [])
if use_ns:
xattr.set(item, USER_NN, USER_VAL,
namespace=NAMESPACE)
else:
xattr.set(item, USER_ATTR, USER_VAL)
if use_ns:
assert xattr.list(item, namespace=NAMESPACE) == [USER_NN]
else:
lists_equal(xattr.list(item), [USER_ATTR])
lists_equal(xattr.list(item, namespace=EMPTY_NS),
[USER_ATTR])
if use_ns:
assert xattr.get(item, USER_NN, namespace=NAMESPACE) == USER_VAL
else:
assert xattr.get(item, USER_ATTR) == USER_VAL
if use_ns:
assert xattr.get_all(item, namespace=NAMESPACE) == \
[(USER_NN, USER_VAL)]
else:
tuples_equal(xattr.get_all(item),
[(USER_ATTR, USER_VAL)])
if use_ns:
xattr.remove(item, USER_NN, namespace=NAMESPACE)
else:
xattr.remove(item, USER_ATTR)
lists_equal(xattr.list(item), [])
tuples_equal(xattr.get_all(item), [])
def test_mixed_access(testdir, gen):
"""test mixed access to file"""
with gen(testdir) as (a, b):
# Check empty
lists_equal(xattr.list(a), [])
lists_equal(xattr.listxattr(b), [])
# Check value
xattr.set(a, USER_ATTR, USER_VAL)
for i in [a, b]:
# Deprecated functions
lists_equal(xattr.listxattr(i), [USER_ATTR])
assert xattr.getxattr(i, USER_ATTR) == USER_VAL
tuples_equal(xattr.get_all(i), [(USER_ATTR, USER_VAL)])
# Current functions
lists_equal(xattr.list(i), [USER_ATTR])
assert xattr.list(i, namespace=NAMESPACE) == [USER_NN]
assert xattr.get(i, USER_ATTR) == USER_VAL
assert xattr.get(i, USER_NN, namespace=NAMESPACE) == USER_VAL
tuples_equal(xattr.get_all(i),
[(USER_ATTR, USER_VAL)])
assert xattr.get_all(i, namespace=NAMESPACE) == \
[(USER_NN, USER_VAL)]
# Overwrite
xattr.set(b, USER_ATTR, LARGE_VAL, flags=xattr.XATTR_REPLACE)
assert xattr.get(a, USER_ATTR) == LARGE_VAL
assert xattr.getxattr(a, USER_ATTR) == LARGE_VAL
xattr.removexattr(b, USER_ATTR)
assert xattr.get_all(a, namespace=NAMESPACE) == []
assert xattr.get_all(b, namespace=NAMESPACE) == []
def _checkListSetGet(self, item, symlink=False, use_ns=False):
"""check list, set, get operations against an item"""
self.assertEqual(self._ignore(xattr.list(item, symlink)),
[])
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_ATTR, self.USER_VAL, flags=XATTR_REPLACE)
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_NN, self.USER_VAL, flags=XATTR_REPLACE,
namespace=NS_USER)
try:
if use_ns:
xattr.set(item, self.USER_NN, self.USER_VAL,
namespace=NS_USER,
nofollow=symlink)
else:
xattr.set(item, self.USER_ATTR, self.USER_VAL,
nofollow=symlink)
except IOError:
err = sys.exc_info()[1]
if err.errno == errno.EPERM and symlink:
# symlinks may fail, in which case we abort the rest
# of the test for this case
return
raise
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_ATTR, self.USER_VAL, flags=XATTR_CREATE)
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_NN, self.USER_VAL,
flags=XATTR_CREATE, namespace=NS_USER)
self.assertEqual(self._ignore(xattr.list(item, nofollow=symlink)),
[self.USER_ATTR])
self.assertEqual(self._ignore(xattr.list(item, nofollow=symlink,
namespace=EMPTY_NS)),
[self.USER_ATTR])
self.assertEqual(xattr.list(item, namespace=NS_USER, nofollow=symlink),
[self.USER_NN])
self.assertEqual(xattr.get(item, self.USER_ATTR, nofollow=symlink),
self.USER_VAL)
self.assertEqual(xattr.get(item, self.USER_NN, nofollow=symlink,
namespace=NS_USER), self.USER_VAL)
self.assertEqual(self._ignore_tuples(xattr.get_all(item,
nofollow=symlink)),
[(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(item, nofollow=symlink,
namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
if use_ns:
xattr.remove(item, self.USER_NN, namespace=NS_USER)
else:
xattr.remove(item, self.USER_ATTR)
self.assertEqual(self._ignore(xattr.list(item, symlink)), [])
self.assertEqual(self._ignore_tuples(xattr.get_all(item,
nofollow=symlink)),
[])
self.assertRaises(EnvironmentError, xattr.remove,
item, self.USER_ATTR, nofollow=symlink)
self.assertRaises(EnvironmentError, xattr.remove, item,
self.USER_NN, namespace=NS_USER, nofollow=symlink)
def _checkListSetGet(self, item, symlink=False, use_ns=False):
"""check list, set, get operations against an item"""
self.checkList(xattr.list(item, symlink), [])
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_ATTR, self.USER_VAL, flags=XATTR_REPLACE)
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_NN, self.USER_VAL, flags=XATTR_REPLACE,
namespace=NS_USER)
try:
if use_ns:
xattr.set(item, self.USER_NN, self.USER_VAL,
namespace=NS_USER,
nofollow=symlink)
else:
xattr.set(item, self.USER_ATTR, self.USER_VAL,
nofollow=symlink)
except IOError:
err = sys.exc_info()[1]
if symlink and (err.errno == errno.EPERM or
err.errno == errno.ENOENT):
# symlinks may fail, in which case we abort the rest
# of the test for this case (Linux returns EPERM; OS X
# returns ENOENT)
return
raise
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_ATTR, self.USER_VAL, flags=XATTR_CREATE)
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_NN, self.USER_VAL,
flags=XATTR_CREATE, namespace=NS_USER)
self.checkList(xattr.list(item, nofollow=symlink), [self.USER_ATTR])
self.checkList(xattr.list(item, nofollow=symlink,
namespace=EMPTY_NS),
[self.USER_ATTR])
self.assertEqual(xattr.list(item, namespace=NS_USER, nofollow=symlink),
[self.USER_NN])
self.assertEqual(xattr.get(item, self.USER_ATTR, nofollow=symlink),
self.USER_VAL)
self.assertEqual(xattr.get(item, self.USER_NN, nofollow=symlink,
namespace=NS_USER), self.USER_VAL)
self.checkTuples(xattr.get_all(item, nofollow=symlink),
[(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(item, nofollow=symlink,
namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
if use_ns:
xattr.remove(item, self.USER_NN, namespace=NS_USER)
else:
xattr.remove(item, self.USER_ATTR)
self.checkList(xattr.list(item, symlink), [])
self.checkTuples(xattr.get_all(item, nofollow=symlink),
[])
self.assertRaises(EnvironmentError, xattr.remove,
item, self.USER_ATTR, nofollow=symlink)
self.assertRaises(EnvironmentError, xattr.remove, item,
self.USER_NN, namespace=NS_USER, nofollow=symlink)
def test_binary_payload(subject):
"""test binary values"""
item = subject[0]
BINVAL = b"abc\0def"
xattr.set(item, USER_ATTR, BINVAL)
lists_equal(xattr.list(item), [USER_ATTR])
assert xattr.list(item, namespace=NAMESPACE) == [USER_NN]
assert xattr.get(item, USER_ATTR) == BINVAL
assert xattr.get(item, USER_NN, namespace=NAMESPACE) == BINVAL
tuples_equal(xattr.get_all(item), [(USER_ATTR, BINVAL)])
assert xattr.get_all(item, namespace=NAMESPACE) == [(USER_NN, BINVAL)]
xattr.remove(item, USER_ATTR)
def testSymlinkOps(self):
"""test symlink operations"""
_, sname = self._getsymlink()
self.assertRaises(EnvironmentError, xattr.list, sname)
self._checkListSetGet(sname, symlink=True)
self._checkListSetGet(sname, symlink=True, use_ns=True)
target, sname = self._getsymlink(dangling=False)
xattr.set(target, self.USER_ATTR, self.USER_VAL)
self.assertEqual(xattr.list(target), [self.USER_ATTR])
self.assertEqual(xattr.list(sname, nofollow=True), [])
self.assertRaises(EnvironmentError, xattr.remove, sname, self.USER_ATTR, nofollow=True)
xattr.remove(sname, self.USER_ATTR, nofollow=False)
def test_no_attributes(any_subject):
"""test no attributes"""
item, nofollow = any_subject
lists_equal(xattr.list(item, nofollow=nofollow), [])
assert xattr.list(item, nofollow=nofollow,
namespace=NAMESPACE) == []
tuples_equal(xattr.get_all(item, nofollow=nofollow), [])
assert xattr.get_all(item, nofollow=nofollow,
namespace=NAMESPACE) == []
with pytest.raises(EnvironmentError):
xattr.get(item, USER_NN, nofollow=nofollow,
namespace=NAMESPACE)
def testSymlinkOps(self):
"""test symlink operations"""
_, sname = self._getsymlink()
self.assertRaises(EnvironmentError, xattr.list, sname)
self._checkListSetGet(sname, symlink=True)
self._checkListSetGet(sname, symlink=True, use_ns=True)
target, sname = self._getsymlink(dangling=False)
xattr.set(target, self.USER_ATTR, self.USER_VAL)
self.checkList(xattr.list(target), [self.USER_ATTR])
self.checkList(xattr.list(sname, nofollow=True), [])
self.assertRaises(EnvironmentError, xattr.remove, sname,
self.USER_ATTR, nofollow=True)
xattr.remove(sname, self.USER_ATTR, nofollow=False)
def testBinaryPayload(self):
"""test binary values"""
fh, fname = self._getfile()
os.close(fh)
BINVAL = "abc" + "\0" + "def"
if PY3K:
BINVAL = BINVAL.encode()
xattr.set(fname, self.USER_ATTR, BINVAL)
self.assertEqual(xattr.list(fname), [self.USER_ATTR])
self.assertEqual(xattr.list(fname, namespace=NS_USER), [self.USER_NN])
self.assertEqual(xattr.get(fname, self.USER_ATTR), BINVAL)
self.assertEqual(xattr.get(fname, self.USER_NN, namespace=NS_USER), BINVAL)
self.assertEqual(xattr.get_all(fname), [(self.USER_ATTR, BINVAL)])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER), [(self.USER_NN, BINVAL)])
xattr.remove(fname, self.USER_ATTR)
def testMixedAccess(self):
"""test mixed access to file"""
fh, fname = self._getfile()
fo = os.fdopen(fh)
self.assertEqual(xattr.list(fname), [])
xattr.set(fname, self.USER_ATTR, self.USER_VAL)
self.assertEqual(xattr.list(fh), [self.USER_ATTR])
self.assertEqual(xattr.list(fh, namespace=NS_USER), [self.USER_NN])
self.assertEqual(xattr.get(fo, self.USER_ATTR), self.USER_VAL)
self.assertEqual(xattr.get(fo, self.USER_NN, namespace=NS_USER), self.USER_VAL)
self.assertEqual(xattr.get_all(fo), [(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fo, namespace=NS_USER), [(self.USER_NN, self.USER_VAL)])
self.assertEqual(xattr.get_all(fname), [(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER), [(self.USER_NN, self.USER_VAL)])
fo.close()
def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
if not xattr:
if self.linux_xattr:
add_error("%s: can't restore xattr; xattr support missing.\n" %
path)
return
existing_xattrs = set(xattr.list(path, nofollow=True))
if self.linux_xattr:
for k, v in self.linux_xattr:
if k not in existing_xattrs \
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.set: %s' % e)
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.remove: %s' % e)
else:
raise
def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
if not xattr:
if self.linux_xattr:
add_error("%s: can't restore xattr; xattr support missing.\n"
% path_msg(path))
return
if not self.linux_xattr:
return
try:
existing_xattrs = set(xattr.list(path, nofollow=True))
except IOError as e:
if e.errno == errno.EACCES:
raise ApplyError('xattr.set %r: %s' % (path_msg(path), e))
else:
raise
for k, v in self.linux_xattr:
if k not in existing_xattrs \
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError as e:
if e.errno == errno.EPERM \
or e.errno == errno.EOPNOTSUPP:
raise ApplyError('xattr.set %r: %s' % (path_msg(path), e))
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError as e:
if e.errno in (errno.EPERM, errno.EACCES):
raise ApplyError('xattr.remove %r: %s' % (path_msg(path), e))
else:
raise
def populate_pgfid_and_inodegfid(brick, changelog_data):
"""
For all the DATA/METADATA modifications GFID,
If symlink, directly convert to Path using Readlink.
If not symlink, try to get PGFIDs via xattr query and populate it
to pgfid table, collect inodes in inodegfid table
"""
for row in changelog_data.gfidpath_get({"path1": "", "type": "MODIFY"}):
gfid = row[3].strip()
p = os.path.join(brick, ".glusterfs", gfid[0:2], gfid[2:4], gfid)
if os.path.islink(p):
# It is a Directory if GFID backend path is symlink
try:
path = symlink_gfid_to_path(brick, gfid)
path = output_path_prepare(path, args)
changelog_data.gfidpath_update({"path1": path},
{"gfid": gfid})
except (IOError, OSError) as e:
logger.warn("Error converting to path: %s" % e)
continue
else:
try:
# INODE and GFID to inodegfid table
changelog_data.inodegfid_add(os.stat(p).st_ino, gfid)
file_xattrs = xattr.list(p)
for x in file_xattrs:
if x.startswith("trusted.pgfid."):
# PGFID in pgfid table
changelog_data.pgfid_add(x.split(".")[-1])
except (IOError, OSError):
# All OS Errors ignored, since failures will be logged
# in End. All GFIDs present in gfidpath table
continue
def testManyOps(self):
"""test many ops"""
fh, fname = self._getfile()
xattr.set(fh, self.USER_ATTR, self.USER_VAL)
VL = [self.USER_ATTR]
VN = [self.USER_NN]
for i in range(self.MANYOPS_COUNT):
self.assertEqual(xattr.list(fh), VL)
self.assertEqual(xattr.list(fh, namespace=EMPTY_NS), VL)
self.assertEqual(xattr.list(fh, namespace=NS_USER), VN)
for i in range(self.MANYOPS_COUNT):
self.assertEqual(xattr.get(fh, self.USER_ATTR), self.USER_VAL)
self.assertEqual(xattr.get(fh, self.USER_NN, namespace=NS_USER), self.USER_VAL)
for i in range(self.MANYOPS_COUNT):
self.assertEqual(xattr.get_all(fh), [(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fh, namespace=NS_USER), [(self.USER_NN, self.USER_VAL)])
def enum_hard_links_using_gfid2path(brick, gfid, args):
hardlinks = []
p = os.path.join(brick, ".glusterfs", gfid[0:2], gfid[2:4], gfid)
if not os.path.isdir(p):
# we have a symlink or a normal file
try:
file_xattrs = xattr.list(p)
for x in file_xattrs:
x_str = bytearray_to_str(x)
if x_str.startswith("trusted.gfid2path."):
# get the value for the xattr i.e. <PGFID>/<BN>
v = xattr.getxattr(p, x_str)
v_str = bytearray_to_str(v)
pgfid, bn = v_str.split(os.sep)
try:
path = symlink_gfid_to_path(brick, pgfid)
fullpath = os.path.join(path, bn)
fullpath = output_path_prepare(fullpath, args)
hardlinks.append(fullpath)
except (IOError, OSError) as e:
logger.warn("Error converting to path: %s" % e)
continue
except (IOError, OSError):
pass
return hardlinks
def populate_pgfid_and_inodegfid(brick, changelog_data):
"""
For all the DATA/METADATA modifications GFID,
If symlink, directly convert to Path using Readlink.
If not symlink, try to get PGFIDs via xattr query and populate it
to pgfid table, collect inodes in inodegfid table
"""
for row in changelog_data.gfidpath_get({"path1": "", "type": "MODIFY"}):
gfid = row[3].strip()
p = os.path.join(brick, ".glusterfs", gfid[0:2], gfid[2:4], gfid)
if os.path.islink(p):
# It is a Directory if GFID backend path is symlink
try:
path = symlink_gfid_to_path(brick, gfid)
path = output_path_prepare(path, args)
changelog_data.gfidpath_update({"path1": path}, {"gfid": gfid})
except (IOError, OSError) as e:
logger.warn("Error converting to path: %s" % e)
continue
else:
try:
# INODE and GFID to inodegfid table
changelog_data.inodegfid_add(os.stat(p).st_ino, gfid)
file_xattrs = xattr.list(p)
for x in file_xattrs:
if x.startswith("trusted.pgfid."):
# PGFID in pgfid table
changelog_data.pgfid_add(x.split(".")[-1])
except (IOError, OSError):
# All OS Errors ignored, since failures will be logged
# in End. All GFIDs present in gfidpath table
continue
def _copyxattr(src, dest, exclude=None):
try:
attrs = xattr.list(src)
except IOError as e:
if e.errno != OperationNotSupported.errno:
raise
attrs = ()
if attrs:
if exclude is not None and isinstance(attrs[0], bytes):
exclude = exclude.encode(_encodings['fs'])
exclude = _get_xattr_excluder(exclude)
for attr in attrs:
if exclude(attr):
continue
try:
xattr.set(dest, attr, xattr.get(src, attr))
raise_exception = False
except IOError:
raise_exception = True
if raise_exception:
raise OperationNotSupported(_("Filesystem containing file '%s' "
"does not support extended attribute '%s'") %
(_unicode_decode(dest), _unicode_decode(attr)))
def _copyxattr(src, dest, exclude=None):
try:
attrs = xattr.list(src)
except IOError as e:
if e.errno != OperationNotSupported.errno:
raise
attrs = ()
if attrs:
if exclude is not None and isinstance(attrs[0], bytes):
exclude = exclude.encode(_encodings['fs'])
exclude = _get_xattr_excluder(exclude)
for attr in attrs:
if exclude(attr):
continue
try:
xattr.set(dest, attr, xattr.get(src, attr))
raise_exception = False
except IOError:
raise_exception = True
if raise_exception:
raise OperationNotSupported(
_("Filesystem containing file '%s' "
"does not support extended attribute '%s'") %
(_unicode_decode(dest), _unicode_decode(attr)))
def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
if not xattr:
if self.linux_xattr:
add_error("%s: can't restore xattr; xattr support missing.\n"
% path)
return
existing_xattrs = set(xattr.list(path, nofollow=True))
if self.linux_xattr:
for k, v in self.linux_xattr:
if k not in existing_xattrs \
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM \
or e.errno == errno.EOPNOTSUPP:
raise ApplyError('xattr.set: %s' % e)
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.remove: %s' % e)
else:
raise
def testBinaryPayload(self):
"""test binary values"""
fh, fname = self._getfile()
os.close(fh)
BINVAL = "abc" + '\0' + "def"
if PY3K:
BINVAL = BINVAL.encode()
xattr.set(fname, self.USER_ATTR, BINVAL)
self.checkList(xattr.list(fname), [self.USER_ATTR])
self.assertEqual(xattr.list(fname, namespace=NS_USER), [self.USER_NN])
self.assertEqual(xattr.get(fname, self.USER_ATTR), BINVAL)
self.assertEqual(xattr.get(fname, self.USER_NN, namespace=NS_USER),
BINVAL)
self.checkTuples(xattr.get_all(fname), [(self.USER_ATTR, BINVAL)])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER),
[(self.USER_NN, BINVAL)])
xattr.remove(fname, self.USER_ATTR)
def keys(self):
"""
Returns a list of all filesystem attributes
"""
return [
key.decode()
for key in xattr.list(self._path, namespace=xattr.NS_USER)
]
def _copyxattr(src, dest):
for attr in xattr.list(src):
try:
xattr.set(dest, attr, xattr.get(src, attr))
raise_exception = False
except IOError:
raise_exception = True
if raise_exception:
raise OperationNotSupported("Filesystem containing file '%s' does not support extended attributes" % dest)
def test_replace_on_missing(subject, use_ns):
item = subject[0]
lists_equal(xattr.list(item), [])
with pytest.raises(EnvironmentError):
if use_ns:
xattr.set(item, USER_NN, USER_VAL, flags=XATTR_REPLACE,
namespace=NAMESPACE)
else:
xattr.set(item, USER_ATTR, USER_VAL, flags=XATTR_REPLACE)
def test_remove_on_missing(any_subject, use_ns):
item, nofollow = any_subject
lists_equal(xattr.list(item, nofollow=nofollow), [])
with pytest.raises(EnvironmentError):
if use_ns:
xattr.remove(item, USER_NN, namespace=NAMESPACE,
nofollow=nofollow)
else:
xattr.remove(item, USER_ATTR, nofollow=nofollow)
def get_sha1(self): name = self.get_real_name() if "user.sha1" in xattr.list(name): val = xattr.get(name, "user.sha1")[:-1] else: with open(name) as f: val = sha1(f.read()).hexdigest() xattr.setxattr(name, "user.sha1", "%s\x00" % val) return val
def read_cache(self):
if not xattr:
return
cache = dict([(n[13:], xattr.get(self.name, n)) for n in xattr.list(self.name) if n.decode().startswith('user.pyanidb.')])
if 'mtime' not in cache or str(int(self.mtime)) != cache.pop('mtime'):
return
for n, v in cache.items():
setattr(self, n, v)
self.cached = True
def test_many_ops(subject):
"""test many ops"""
item = subject[0]
xattr.set(item, USER_ATTR, USER_VAL)
VL = [USER_ATTR]
VN = [USER_NN]
for i in range(MANYOPS_COUNT):
lists_equal(xattr.list(item), VL)
lists_equal(xattr.list(item, namespace=EMPTY_NS), VL)
assert xattr.list(item, namespace=NAMESPACE) == VN
for i in range(MANYOPS_COUNT):
assert xattr.get(item, USER_ATTR) == USER_VAL
assert xattr.get(item, USER_NN, namespace=NAMESPACE) == USER_VAL
for i in range(MANYOPS_COUNT):
tuples_equal(xattr.get_all(item),
[(USER_ATTR, USER_VAL)])
assert xattr.get_all(item, namespace=NAMESPACE) == \
[(USER_NN, USER_VAL)]
def testMixedAccess(self):
"""test mixed access to file"""
fh, fname = self._getfile()
fo = os.fdopen(fh)
self.checkList(xattr.list(fname), [])
xattr.set(fname, self.USER_ATTR, self.USER_VAL)
self.checkList(xattr.list(fh), [self.USER_ATTR])
self.assertEqual(xattr.list(fh, namespace=NS_USER), [self.USER_NN])
self.assertEqual(xattr.get(fo, self.USER_ATTR), self.USER_VAL)
self.assertEqual(xattr.get(fo, self.USER_NN, namespace=NS_USER),
self.USER_VAL)
self.checkTuples(xattr.get_all(fo), [(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fo, namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
self.checkTuples(xattr.get_all(fname),
[(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
fo.close()
def testManyOps(self):
"""test many ops"""
fh, fname = self._getfile()
xattr.set(fh, self.USER_ATTR, self.USER_VAL)
VL = [self.USER_ATTR]
VN = [self.USER_NN]
for i in range(self.MANYOPS_COUNT):
self.checkList(xattr.list(fh), VL)
self.checkList(xattr.list(fh, namespace=EMPTY_NS), VL)
self.assertEqual(xattr.list(fh, namespace=NS_USER), VN)
for i in range(self.MANYOPS_COUNT):
self.assertEqual(xattr.get(fh, self.USER_ATTR), self.USER_VAL)
self.assertEqual(xattr.get(fh, self.USER_NN, namespace=NS_USER),
self.USER_VAL)
for i in range(self.MANYOPS_COUNT):
self.checkTuples(xattr.get_all(fh),
[(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fh, namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
def _copyxattr(src, dest):
for attr in xattr.list(src):
try:
xattr.set(dest, attr, xattr.get(src, attr))
raise_exception = False
except IOError:
raise_exception = True
if raise_exception:
raise OperationNotSupported(
"Filesystem containing file '%s' does not support extended attributes"
% dest)
def read_cache(self):
if not xattr:
return
cache = dict([(n[13:], xattr.get(self.name, n))
for n in xattr.list(self.name)
if n.decode().startswith('user.pyanidb.')])
if 'mtime' not in cache or str(int(self.mtime)) != cache.pop('mtime'):
return
for n, v in cache.items():
setattr(self, n, v)
self.cached = True
def testNoXattr(self):
"""test no attributes"""
fh, fname = self._getfile()
self.checkList(xattr.list(fname), [])
self.assertEqual(xattr.list(fname, namespace=NS_USER), [])
self.checkTuples(xattr.get_all(fname), [])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER), [])
self.assertRaises(EnvironmentError, xattr.get, fname,
self.USER_NN, namespace=NS_USER)
dname = self._getdir()
self.checkList(xattr.list(dname), [])
self.assertEqual(xattr.list(dname, namespace=NS_USER), [])
self.checkTuples(xattr.get_all(dname), [])
self.assertEqual(xattr.get_all(dname, namespace=NS_USER), [])
self.assertRaises(EnvironmentError, xattr.get, dname,
self.USER_NN, namespace=NS_USER)
_, sname = self._getsymlink()
self.checkList(xattr.list(sname, nofollow=True), [])
self.assertEqual(xattr.list(sname, nofollow=True,
namespace=NS_USER), [])
self.checkTuples(xattr.get_all(sname, nofollow=True), [])
self.assertEqual(xattr.get_all(sname, nofollow=True,
namespace=NS_USER), [])
self.assertRaises(EnvironmentError, xattr.get, sname,
self.USER_NN, namespace=NS_USER, nofollow=True)
def testNoXattr(self):
"""test no attributes"""
fh, fname = self._getfile()
self.checkList(xattr.list(fname), [])
self.assertEqual(xattr.list(fname, namespace=NS_USER), [])
self.checkTuples(xattr.get_all(fname), [])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER), [])
self.assertRaises(EnvironmentError, xattr.get, fname,
self.USER_NN, namespace=NS_USER)
dname = self._getdir()
self.checkList(xattr.list(dname), [])
self.assertEqual(xattr.list(dname, namespace=NS_USER), [])
self.checkTuples(xattr.get_all(dname), [])
self.assertEqual(xattr.get_all(dname, namespace=NS_USER), [])
self.assertRaises(EnvironmentError, xattr.get, dname,
self.USER_NN, namespace=NS_USER)
_, sname = self._getsymlink()
self.checkList(xattr.list(sname, nofollow=True), [])
self.assertEqual(xattr.list(sname, nofollow=True,
namespace=NS_USER), [])
self.checkTuples(xattr.get_all(sname, nofollow=True), [])
self.assertEqual(xattr.get_all(sname, nofollow=True,
namespace=NS_USER), [])
self.assertRaises(EnvironmentError, xattr.get, sname,
self.USER_NN, namespace=NS_USER, nofollow=True)
def xattr_key_list(fn):
if not is_file(fn):
return False
try:
r = [
i.decode("utf-8") for i in xattr.list(fn, namespace=xattr.NS_USER)
]
except Exception as e:
# print(e)
return list()
return r
def test_handling_of_incorrect_existing_linux_xattrs():
if os.geteuid() != 0 or detect_fakeroot():
return
setup_testfs()
subprocess.check_call('rm -rf testfs/*', shell=True)
path = 'testfs/foo'
open(path, 'w').close()
xattr.set(path, 'foo', 'bar', namespace=xattr.NS_USER)
m = metadata.from_path(path, archive_path=path, save_symlinks=True)
xattr.set(path, 'baz', 'bax', namespace=xattr.NS_USER)
m.apply_to_path(path, restore_numeric_ids=False)
WVPASSEQ(xattr.list(path), ['user.foo'])
WVPASSEQ(xattr.get(path, 'user.foo'), 'bar')
xattr.set(path, 'foo', 'baz', namespace=xattr.NS_USER)
m.apply_to_path(path, restore_numeric_ids=False)
WVPASSEQ(xattr.list(path), ['user.foo'])
WVPASSEQ(xattr.get(path, 'user.foo'), 'bar')
xattr.remove(path, 'foo', namespace=xattr.NS_USER)
m.apply_to_path(path, restore_numeric_ids=False)
WVPASSEQ(xattr.list(path), ['user.foo'])
WVPASSEQ(xattr.get(path, 'user.foo'), 'bar')
os.chdir(top_dir)
cleanup_testfs()
def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
if not xattr:
if self.linux_xattr:
add_error("%s: can't restore xattr; xattr support missing.\n" % path)
return
if not self.linux_xattr:
return
try:
existing_xattrs = set(xattr.list(path, nofollow=True))
except IOError, e:
if e.errno == errno.EACCES:
raise ApplyError("xattr.set: %s" % e)
else:
raise
def test_create_on_existing(subject, use_ns):
item = subject[0]
lists_equal(xattr.list(item), [])
if use_ns:
xattr.set(item, USER_NN, USER_VAL,
namespace=NAMESPACE)
else:
xattr.set(item, USER_ATTR, USER_VAL)
with pytest.raises(EnvironmentError):
if use_ns:
xattr.set(item, USER_NN, USER_VAL,
flags=XATTR_CREATE, namespace=NAMESPACE)
else:
xattr.set(item, USER_ATTR, USER_VAL, flags=XATTR_CREATE)
def run(args):
# Volmark from Master side
fmt_string = "!" + "B" * 19 + "II"
try:
vm = struct.unpack(
fmt_string,
xattr.get(args.path,
"trusted.glusterfs.volume-mark",
nofollow=True))
print "UUID : %s" % uuid.UUID("".join(
['%02x' % x for x in vm[2:18]]))
print "VERSION : %s.%s" % vm[0:2]
print "RETVAL : %s" % vm[18]
print "VOLUME MARK : %s.%s (%s)" % (vm[19], vm[20],
human_time("%s.%s" %
(vm[19], vm[20])))
except (OSError, IOError) as e:
if e.errno == ENODATA:
pass
else:
print "[Error %s] %s" % (e.errno, os.strerror(e.errno))
sys.exit(-1)
# Volmark from slave side
all_xattrs = xattr.list(args.path)
fmt_string = "!" + "B" * 19 + "II" + "I"
volmark_xattrs = []
for x in all_xattrs:
if x.startswith("trusted.glusterfs.volume-mark."):
volmark_xattrs.append(x)
for vx in volmark_xattrs:
try:
vm = struct.unpack(fmt_string, xattr.get(args.path, vx))
print "UUID : %s" % uuid.UUID("".join(
['%02x' % x for x in vm[2:18]]))
print "VERSION : %s.%s" % vm[0:2]
print "RETVAL : %s" % vm[18]
print "VOLUME MARK : %s.%s (%s)" % (vm[19], vm[20],
human_time("%s.%s" %
(vm[19], vm[20])))
print "TIMEOUT : %s (%s)" % (vm[-1], human_time(vm[-1]))
except (OSError, IOError) as e:
if e.errno == ENODATA:
pass
else:
print "[Error %s] %s" % (e.errno, os.strerror(e.errno))
sys.exit(-1)
def get_archive_stime(brick_path):
"""
Get all stime xattrs and find minimum stime
"""
stime_xattr_values = []
xattrs_list = xattr.list(brick_path)
for x in xattrs_list:
if x.endswith(".stime"):
stime = struct.unpack("!II", xattr.get(brick_path, x))
stime_xattr_values.append(stime[0])
if stime_xattr_values:
return min(stime_xattr_values)
else:
return 0
def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
if not xattr:
if self.linux_xattr:
add_error("%s: can't restore xattr; xattr support missing.\n" %
path)
return
if not self.linux_xattr:
return
try:
existing_xattrs = set(xattr.list(path, nofollow=True))
except IOError, e:
if e.errno == errno.EACCES:
raise ApplyError('xattr.set %r: %s' % (path, e))
else:
raise
def run(args):
# Volmark from Master side
fmt_string = "!" + "B" * 19 + "II"
try:
vm = struct.unpack(fmt_string, xattr.get(
args.path,
"trusted.glusterfs.volume-mark",
nofollow=True))
print "UUID : %s" % uuid.UUID(
"".join(['%02x' % x for x in vm[2:18]]))
print "VERSION : %s.%s" % vm[0:2]
print "RETVAL : %s" % vm[18]
print "VOLUME MARK : %s.%s (%s)" % (
vm[19], vm[20], human_time("%s.%s" % (vm[19], vm[20])))
except (OSError, IOError) as e:
if e.errno == ENODATA:
pass
else:
print "[Error %s] %s" % (e.errno, os.strerror(e.errno))
sys.exit(-1)
# Volmark from slave side
all_xattrs = xattr.list(args.path)
fmt_string = "!" + "B" * 19 + "II" + "I"
volmark_xattrs = []
for x in all_xattrs:
if x.startswith("trusted.glusterfs.volume-mark."):
volmark_xattrs.append(x)
for vx in volmark_xattrs:
try:
vm = struct.unpack(fmt_string, xattr.get(
args.path,
vx))
print "UUID : %s" % uuid.UUID(
"".join(['%02x' % x for x in vm[2:18]]))
print "VERSION : %s.%s" % vm[0:2]
print "RETVAL : %s" % vm[18]
print "VOLUME MARK : %s.%s (%s)" % (
vm[19], vm[20], human_time("%s.%s" % (vm[19], vm[20])))
print "TIMEOUT : %s (%s)" % (vm[-1], human_time(vm[-1]))
except (OSError, IOError) as e:
if e.errno == ENODATA:
pass
else:
print "[Error %s] %s" % (e.errno, os.strerror(e.errno))
sys.exit(-1)
def dump_xattrs(pathnames, file_out):
"""Dump the xattr data for |pathnames| to |file_out|"""
# NOTE: Always quote backslashes, in order to ensure that they are
# not interpreted as quotes when they are processed by unquote.
quote_chars = b'\n\r\\\\'
for pathname in pathnames:
attrs = xattr.list(pathname)
if not attrs:
continue
file_out.write(b'# file: %s\n' % quote(pathname, quote_chars))
for attr in attrs:
attr = unicode_encode(attr)
value = xattr.get(pathname, attr)
file_out.write(b'%s="%s"\n' % (quote(
attr, b'=' + quote_chars), quote(value, b'\0"' + quote_chars)))
def dump_xattrs(pathnames, file_out): """Dump the xattr data for |pathnames| to |file_out|""" # NOTE: Always quote backslashes, in order to ensure that they are # not interpreted as quotes when they are processed by unquote. quote_chars = b'\n\r\\\\' for pathname in pathnames: attrs = xattr.list(pathname) if not attrs: continue file_out.write(b'# file: %s\n' % quote(pathname, quote_chars)) for attr in attrs: attr = unicode_encode(attr) value = xattr.get(pathname, attr) file_out.write(b'%s="%s"\n' % ( quote(attr, b'=' + quote_chars), quote(value, b'\0"' + quote_chars)))
def gfid_to_paths(brick, gfid):
import xattr
gfidfile = os.path.join(brick, ".glusterfs", gfid[0:2], gfid[2:4], gfid)
paths = []
gfids = []
try:
for xtr in xattr.list(gfidfile):
if xtr.startswith(b"trusted.gfid2path."):
data = xattr.get(gfidfile, xtr).decode().split("/")
pgfid = data[-2]
basename = data[-1]
ppath = symlink_gfid_to_path(brick, pgfid)
paths.append(os.path.join(ppath, basename))
except (OSError, IOError):
gfids.append(gfid)
return (paths, gfids)
def main():
print("main")
sys.stdout.flush()
hosts = ("localhost", )
attrs = xattr.list("/opt/gms_sample/227064_000202_BLA_SR.bsq")
#print(attrs)
if len(attrs) > 0:
jsonString = xattr.get("/opt/gms_sample/227064_000202_BLA_SR.bsq",
"xtreemfs.locations")
#jsonString = xattr.get("/home/mathiasp/mount/localScenes/227064_020717_BLA_SR.bsq", "xtreemfs.locations")
parsed = json.loads(jsonString)
print(parsed["replicas"][0]['osds'][0]['address'].split(":")[0])
hosts = ()
replicas = parsed["replicas"]
for replica in replicas:
#assume that scenes are not striped across osds
osd = replica["osds"][0]
address = osd['address'].split(":")[0]
hosts += (address, )
print(hosts)
def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
existing_xattrs = set(xattr.list(path, nofollow=True))
if(self.linux_xattr):
for k, v in self.linux_xattr:
if k not in existing_xattrs \
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.set: %s' % e)
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.remove: %s' % e)
else:
raise
def testNoXattr(self):
"""test no attributes"""
fh, fname = self._getfile()
self.assertEqual(xattr.list(fname), [])
self.assertEqual(xattr.list(fname, namespace=NS_USER), [])
self.assertEqual(xattr.get_all(fname), [])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER), [])
dname = self._getdir()
self.assertEqual(xattr.list(dname), [])
self.assertEqual(xattr.list(dname, namespace=NS_USER), [])
self.assertEqual(xattr.get_all(dname), [])
self.assertEqual(xattr.get_all(dname, namespace=NS_USER), [])
_, sname = self._getsymlink()
self.assertEqual(xattr.list(sname, nofollow=True), [])
self.assertEqual(xattr.list(sname, nofollow=True, namespace=NS_USER), [])
self.assertEqual(xattr.get_all(sname, nofollow=True), [])
self.assertEqual(xattr.get_all(sname, nofollow=True, namespace=NS_USER), [])
def testNoXattr(self):
"""test no attributes"""
fh, fname = self._getfile()
self.checkList(xattr.list(fname), [])
self.assertEqual(xattr.list(fname, namespace=NS_USER), [])
self.checkTuples(xattr.get_all(fname), [])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER), [])
dname = self._getdir()
self.checkList(xattr.list(dname), [])
self.assertEqual(xattr.list(dname, namespace=NS_USER), [])
self.checkTuples(xattr.get_all(dname), [])
self.assertEqual(xattr.get_all(dname, namespace=NS_USER), [])
_, sname = self._getsymlink()
self.checkList(xattr.list(sname, nofollow=True), [])
self.assertEqual(xattr.list(sname, nofollow=True, namespace=NS_USER),
[])
self.checkTuples(xattr.get_all(sname, nofollow=True), [])
self.assertEqual(
xattr.get_all(sname, nofollow=True, namespace=NS_USER), [])
def keys(self):
"""
Returns a list of all filesystem attributes
"""
return [ key.decode() for key in xattr.list(self._path, namespace=xattr.NS_USER) ]
def clear_cache(self):
for name in xattr.list(self.name):
if name.decode().startswith('user.pyanidb.'):
xattr.remove(self.name, name)
def gfid_to_path_using_pgfid(brick, gfids_file, output_file, outfile_failures):
"""
Parent GFID is saved as xattr, collect Parent GFIDs from all
the files from gfids_file. Convert parent GFID to path and Crawl
each directories to get the list of files/dirs having same inode number.
Do find with maxdepth as 1 and print the output in <INODE_NUM> <PATH>
format, use this output to look into in memory dictionary of inode
numbers got from the list of GFIDs
"""
with open(output_file, "a+") as fout:
pgfids = set()
inode_dict = {}
with open(gfids_file) as f:
for gfid in f:
gfid = gfid.strip()
p = os.path.join(brick,
".glusterfs",
gfid[0:2],
gfid[2:4],
gfid)
if os.path.islink(p):
path = symlink_gfid_to_path(brick, gfid)
output_write(fout, path, args.output_prefix)
else:
try:
inode_dict[str(os.stat(p).st_ino)] = 1
file_xattrs = xattr.list(p)
num_parent_gfid = 0
for x in file_xattrs:
if x.startswith("trusted.pgfid."):
num_parent_gfid += 1
pgfids.add(x.split(".")[-1])
if num_parent_gfid == 0:
with open(outfile_failures, "a") as f:
f.write("%s\n" % gfid)
f.flush()
os.fsync(f.fileno())
except (IOError, OSError) as e:
if e.errno == ENOENT:
continue
else:
fail("%s Failed to convert to path from "
"GFID %s: %s" % (brick, gfid, e),
logger=logger)
if not inode_dict:
return
def inode_filter(path):
try:
st = os.lstat(path)
except (OSError, IOError) as e:
if e.errno == ENOENT:
st = None
else:
raise
if st and inode_dict.get(str(st.st_ino), None):
return True
return False
# Length of brick path, to remove from output path
brick_path_len = len(brick)
def output_callback(path):
path = path.strip()
path = path[brick_path_len+1:]
output_write(fout, path, args.output_prefix)
ignore_dirs = [os.path.join(brick, dirname)
for dirname in
conf.get_opt("brick_ignore_dirs").split(",")]
for pgfid in pgfids:
path = symlink_gfid_to_path(brick, pgfid)
find(os.path.join(brick, path),
callback_func=output_callback,
filter_func=inode_filter,
ignore_dirs=ignore_dirs,
subdirs_crawl=False)
fout.flush()
os.fsync(fout.fileno())