def test_purge_lock(self):
resp = self.test_file.write(data='test',
hdrs={'X-Hpss-Purgelock-Status': 'true',
'X-Hpss-Class-Of-Service-Id': '1'},
return_resp=True)
print resp.status
print resp.getheaders()
print resp.read()
test_file_name = os.path.join(self.hpss_dir,
self.account.name,
self.container.name,
'testfile')
print test_file_name
print os.stat(test_file_name)
print xattr.listxattr(test_file_name)
self.assertEqual(xattr.get(test_file_name,
'system.hpss.purgelock'),
'1')
self.test_file.post(hdrs={'X-Hpss-Purgelock-Status': 'false'})
self.assertEqual(xattr.get(test_file_name,
'system.hpss.purgelock'), '0')
def test_mixed_access(testdir, gen):
"""test mixed access to file"""
with gen(testdir) as (a, b):
# Check empty
lists_equal(xattr.list(a), [])
lists_equal(xattr.listxattr(b), [])
# Check value
xattr.set(a, USER_ATTR, USER_VAL)
for i in [a, b]:
# Deprecated functions
lists_equal(xattr.listxattr(i), [USER_ATTR])
assert xattr.getxattr(i, USER_ATTR) == USER_VAL
tuples_equal(xattr.get_all(i), [(USER_ATTR, USER_VAL)])
# Current functions
lists_equal(xattr.list(i), [USER_ATTR])
assert xattr.list(i, namespace=NAMESPACE) == [USER_NN]
assert xattr.get(i, USER_ATTR) == USER_VAL
assert xattr.get(i, USER_NN, namespace=NAMESPACE) == USER_VAL
tuples_equal(xattr.get_all(i),
[(USER_ATTR, USER_VAL)])
assert xattr.get_all(i, namespace=NAMESPACE) == \
[(USER_NN, USER_VAL)]
# Overwrite
xattr.set(b, USER_ATTR, LARGE_VAL, flags=xattr.XATTR_REPLACE)
assert xattr.get(a, USER_ATTR) == LARGE_VAL
assert xattr.getxattr(a, USER_ATTR) == LARGE_VAL
xattr.removexattr(b, USER_ATTR)
assert xattr.get_all(a, namespace=NAMESPACE) == []
assert xattr.get_all(b, namespace=NAMESPACE) == []
def recurse_directory( directory ):
for filename in os.listdir(directory):
if os.path.isdir(directory + "/" + filename):
if debug:
print("%s is a folder" % filename)
recurse_directory(directory + "/" + filename)
for p in xattr.listxattr(directory + "/" + filename):
s=p.decode()
if s.find(stringtoskip) != 2:
print(s.find(stringtoskip))
print("= %s : %s" % ( directory + "/" + filename, p ) )
print("%s", xattr.get(directory + "/" + filename,p))
if re.search(matchchecksum, s):
try:
thisxattr=xattr.get(directory + "/" + filename, 'user.swift.metadata_checksum')
except:
thisxattr=''
try:
if not thisxattr:
new_checksum = hashlib.md5(xattr.get(directory + "/" + filename,p)).hexdigest()
print("SET CHKSUM: %s" % ( new_checksum ))
xattr.set(directory + "/" + filename, 'user.swift.metadata_checksum', new_checksum)
else:
if debug:
new_checksum = hashlib.md5(xattr.get(directory + "/" + filename,p)).hexdigest()
print("CHKSUM: %s %s" % (xattr.get(directory + "/" + filename, 'user.swift.metadata_checksum'), new_checksum))
except:
raise
else:
print("%s didn't match %s" % ( matchchecksum, s ))
def test_set_get_remove(subject, use_ns):
item = subject[0]
lists_equal(xattr.list(item), [])
if use_ns:
xattr.set(item, USER_NN, USER_VAL,
namespace=NAMESPACE)
else:
xattr.set(item, USER_ATTR, USER_VAL)
if use_ns:
assert xattr.list(item, namespace=NAMESPACE) == [USER_NN]
else:
lists_equal(xattr.list(item), [USER_ATTR])
lists_equal(xattr.list(item, namespace=EMPTY_NS),
[USER_ATTR])
if use_ns:
assert xattr.get(item, USER_NN, namespace=NAMESPACE) == USER_VAL
else:
assert xattr.get(item, USER_ATTR) == USER_VAL
if use_ns:
assert xattr.get_all(item, namespace=NAMESPACE) == \
[(USER_NN, USER_VAL)]
else:
tuples_equal(xattr.get_all(item),
[(USER_ATTR, USER_VAL)])
if use_ns:
xattr.remove(item, USER_NN, namespace=NAMESPACE)
else:
xattr.remove(item, USER_ATTR)
lists_equal(xattr.list(item), [])
tuples_equal(xattr.get_all(item), [])
def _checkListSetGet(self, item, symlink=False, use_ns=False):
"""check list, set, get operations against an item"""
self.assertEqual(self._ignore(xattr.list(item, symlink)),
[])
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_ATTR, self.USER_VAL, flags=XATTR_REPLACE)
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_NN, self.USER_VAL, flags=XATTR_REPLACE,
namespace=NS_USER)
try:
if use_ns:
xattr.set(item, self.USER_NN, self.USER_VAL,
namespace=NS_USER,
nofollow=symlink)
else:
xattr.set(item, self.USER_ATTR, self.USER_VAL,
nofollow=symlink)
except IOError:
err = sys.exc_info()[1]
if err.errno == errno.EPERM and symlink:
# symlinks may fail, in which case we abort the rest
# of the test for this case
return
raise
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_ATTR, self.USER_VAL, flags=XATTR_CREATE)
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_NN, self.USER_VAL,
flags=XATTR_CREATE, namespace=NS_USER)
self.assertEqual(self._ignore(xattr.list(item, nofollow=symlink)),
[self.USER_ATTR])
self.assertEqual(self._ignore(xattr.list(item, nofollow=symlink,
namespace=EMPTY_NS)),
[self.USER_ATTR])
self.assertEqual(xattr.list(item, namespace=NS_USER, nofollow=symlink),
[self.USER_NN])
self.assertEqual(xattr.get(item, self.USER_ATTR, nofollow=symlink),
self.USER_VAL)
self.assertEqual(xattr.get(item, self.USER_NN, nofollow=symlink,
namespace=NS_USER), self.USER_VAL)
self.assertEqual(self._ignore_tuples(xattr.get_all(item,
nofollow=symlink)),
[(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(item, nofollow=symlink,
namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
if use_ns:
xattr.remove(item, self.USER_NN, namespace=NS_USER)
else:
xattr.remove(item, self.USER_ATTR)
self.assertEqual(self._ignore(xattr.list(item, symlink)), [])
self.assertEqual(self._ignore_tuples(xattr.get_all(item,
nofollow=symlink)),
[])
self.assertRaises(EnvironmentError, xattr.remove,
item, self.USER_ATTR, nofollow=symlink)
self.assertRaises(EnvironmentError, xattr.remove, item,
self.USER_NN, namespace=NS_USER, nofollow=symlink)
def _checkListSetGet(self, item, symlink=False, use_ns=False):
"""check list, set, get operations against an item"""
self.checkList(xattr.list(item, symlink), [])
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_ATTR, self.USER_VAL, flags=XATTR_REPLACE)
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_NN, self.USER_VAL, flags=XATTR_REPLACE,
namespace=NS_USER)
try:
if use_ns:
xattr.set(item, self.USER_NN, self.USER_VAL,
namespace=NS_USER,
nofollow=symlink)
else:
xattr.set(item, self.USER_ATTR, self.USER_VAL,
nofollow=symlink)
except IOError:
err = sys.exc_info()[1]
if symlink and (err.errno == errno.EPERM or
err.errno == errno.ENOENT):
# symlinks may fail, in which case we abort the rest
# of the test for this case (Linux returns EPERM; OS X
# returns ENOENT)
return
raise
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_ATTR, self.USER_VAL, flags=XATTR_CREATE)
self.assertRaises(EnvironmentError, xattr.set, item,
self.USER_NN, self.USER_VAL,
flags=XATTR_CREATE, namespace=NS_USER)
self.checkList(xattr.list(item, nofollow=symlink), [self.USER_ATTR])
self.checkList(xattr.list(item, nofollow=symlink,
namespace=EMPTY_NS),
[self.USER_ATTR])
self.assertEqual(xattr.list(item, namespace=NS_USER, nofollow=symlink),
[self.USER_NN])
self.assertEqual(xattr.get(item, self.USER_ATTR, nofollow=symlink),
self.USER_VAL)
self.assertEqual(xattr.get(item, self.USER_NN, nofollow=symlink,
namespace=NS_USER), self.USER_VAL)
self.checkTuples(xattr.get_all(item, nofollow=symlink),
[(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(item, nofollow=symlink,
namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
if use_ns:
xattr.remove(item, self.USER_NN, namespace=NS_USER)
else:
xattr.remove(item, self.USER_ATTR)
self.checkList(xattr.list(item, symlink), [])
self.checkTuples(xattr.get_all(item, nofollow=symlink),
[])
self.assertRaises(EnvironmentError, xattr.remove,
item, self.USER_ATTR, nofollow=symlink)
self.assertRaises(EnvironmentError, xattr.remove, item,
self.USER_NN, namespace=NS_USER, nofollow=symlink)
def get_gfid(path):
try:
return uuid.UUID(bytes=xattr.get(path, "trusted.gfid",
nofollow=True))
except (IOError, OSError) as e:
if e.errno == ENODATA:
return uuid.UUID(bytes=xattr.get(path, "glusterfs.gfid",
nofollow=True))
else:
raise
def test_binary_payload(subject):
"""test binary values"""
item = subject[0]
BINVAL = b"abc\0def"
xattr.set(item, USER_ATTR, BINVAL)
lists_equal(xattr.list(item), [USER_ATTR])
assert xattr.list(item, namespace=NAMESPACE) == [USER_NN]
assert xattr.get(item, USER_ATTR) == BINVAL
assert xattr.get(item, USER_NN, namespace=NAMESPACE) == BINVAL
tuples_equal(xattr.get_all(item), [(USER_ATTR, BINVAL)])
assert xattr.get_all(item, namespace=NAMESPACE) == [(USER_NN, BINVAL)]
xattr.remove(item, USER_ATTR)
def mode_gfid(args):
try:
try:
print uuid.UUID(bytes=xattr.get(args.path, "trusted.gfid", nofollow=True))
except (IOError, OSError) as e:
if e.errno == ENODATA:
print uuid.UUID(bytes=xattr.get(args.path, "glusterfs.gfid", nofollow=True))
else:
raise
except (OSError, IOError) as e:
print "[Error %s] %s" % (e.errno, os.strerror(e.errno))
sys.exit(-1)
def test_no_attributes(any_subject):
"""test no attributes"""
item, nofollow = any_subject
lists_equal(xattr.list(item, nofollow=nofollow), [])
assert xattr.list(item, nofollow=nofollow,
namespace=NAMESPACE) == []
tuples_equal(xattr.get_all(item, nofollow=nofollow), [])
assert xattr.get_all(item, nofollow=nofollow,
namespace=NAMESPACE) == []
with pytest.raises(EnvironmentError):
xattr.get(item, USER_NN, nofollow=nofollow,
namespace=NAMESPACE)
def run(args):
# Volmark from Master side
fmt_string = "!" + "B" * 19 + "II"
try:
vm = struct.unpack(
fmt_string,
xattr.get(args.path,
"trusted.glusterfs.volume-mark",
nofollow=True))
print "UUID : %s" % uuid.UUID("".join(
['%02x' % x for x in vm[2:18]]))
print "VERSION : %s.%s" % vm[0:2]
print "RETVAL : %s" % vm[18]
print "VOLUME MARK : %s.%s (%s)" % (vm[19], vm[20],
human_time("%s.%s" %
(vm[19], vm[20])))
except (OSError, IOError) as e:
if e.errno == ENODATA:
pass
else:
print "[Error %s] %s" % (e.errno, os.strerror(e.errno))
sys.exit(-1)
# Volmark from slave side
all_xattrs = xattr.list(args.path)
fmt_string = "!" + "B" * 19 + "II" + "I"
volmark_xattrs = []
for x in all_xattrs:
if x.startswith("trusted.glusterfs.volume-mark."):
volmark_xattrs.append(x)
for vx in volmark_xattrs:
try:
vm = struct.unpack(fmt_string, xattr.get(args.path, vx))
print "UUID : %s" % uuid.UUID("".join(
['%02x' % x for x in vm[2:18]]))
print "VERSION : %s.%s" % vm[0:2]
print "RETVAL : %s" % vm[18]
print "VOLUME MARK : %s.%s (%s)" % (vm[19], vm[20],
human_time("%s.%s" %
(vm[19], vm[20])))
print "TIMEOUT : %s (%s)" % (vm[-1], human_time(vm[-1]))
except (OSError, IOError) as e:
if e.errno == ENODATA:
pass
else:
print "[Error %s] %s" % (e.errno, os.strerror(e.errno))
sys.exit(-1)
def testMixedAccess(self):
"""test mixed access to file"""
fh, fname = self._getfile()
fo = os.fdopen(fh)
self.assertEqual(xattr.list(fname), [])
xattr.set(fname, self.USER_ATTR, self.USER_VAL)
self.assertEqual(xattr.list(fh), [self.USER_ATTR])
self.assertEqual(xattr.list(fh, namespace=NS_USER), [self.USER_NN])
self.assertEqual(xattr.get(fo, self.USER_ATTR), self.USER_VAL)
self.assertEqual(xattr.get(fo, self.USER_NN, namespace=NS_USER), self.USER_VAL)
self.assertEqual(xattr.get_all(fo), [(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fo, namespace=NS_USER), [(self.USER_NN, self.USER_VAL)])
self.assertEqual(xattr.get_all(fname), [(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER), [(self.USER_NN, self.USER_VAL)])
fo.close()
def testBinaryPayload(self):
"""test binary values"""
fh, fname = self._getfile()
os.close(fh)
BINVAL = "abc" + "\0" + "def"
if PY3K:
BINVAL = BINVAL.encode()
xattr.set(fname, self.USER_ATTR, BINVAL)
self.assertEqual(xattr.list(fname), [self.USER_ATTR])
self.assertEqual(xattr.list(fname, namespace=NS_USER), [self.USER_NN])
self.assertEqual(xattr.get(fname, self.USER_ATTR), BINVAL)
self.assertEqual(xattr.get(fname, self.USER_NN, namespace=NS_USER), BINVAL)
self.assertEqual(xattr.get_all(fname), [(self.USER_ATTR, BINVAL)])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER), [(self.USER_NN, BINVAL)])
xattr.remove(fname, self.USER_ATTR)
def run(args):
# Volmark from Master side
fmt_string = "!" + "B" * 19 + "II"
try:
vm = struct.unpack(fmt_string, xattr.get(
args.path,
"trusted.glusterfs.volume-mark",
nofollow=True))
print "UUID : %s" % uuid.UUID(
"".join(['%02x' % x for x in vm[2:18]]))
print "VERSION : %s.%s" % vm[0:2]
print "RETVAL : %s" % vm[18]
print "VOLUME MARK : %s.%s (%s)" % (
vm[19], vm[20], human_time("%s.%s" % (vm[19], vm[20])))
except (OSError, IOError) as e:
if e.errno == ENODATA:
pass
else:
print "[Error %s] %s" % (e.errno, os.strerror(e.errno))
sys.exit(-1)
# Volmark from slave side
all_xattrs = xattr.list(args.path)
fmt_string = "!" + "B" * 19 + "II" + "I"
volmark_xattrs = []
for x in all_xattrs:
if x.startswith("trusted.glusterfs.volume-mark."):
volmark_xattrs.append(x)
for vx in volmark_xattrs:
try:
vm = struct.unpack(fmt_string, xattr.get(
args.path,
vx))
print "UUID : %s" % uuid.UUID(
"".join(['%02x' % x for x in vm[2:18]]))
print "VERSION : %s.%s" % vm[0:2]
print "RETVAL : %s" % vm[18]
print "VOLUME MARK : %s.%s (%s)" % (
vm[19], vm[20], human_time("%s.%s" % (vm[19], vm[20])))
print "TIMEOUT : %s (%s)" % (vm[-1], human_time(vm[-1]))
except (OSError, IOError) as e:
if e.errno == ENODATA:
pass
else:
print "[Error %s] %s" % (e.errno, os.strerror(e.errno))
sys.exit(-1)
def mode_stime(args):
try:
stime_key = "trusted.glusterfs.%s.%s.stime" % (args.master_uuid, args.slave_uuid)
print struct.unpack("!II", xattr.get(args.path, stime_key, nofollow=True))
except (OSError, IOError) as e:
print "[Error %s] %s" % (e.errno, os.strerror(e.errno))
sys.exit(-1)
def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
if not xattr:
if self.linux_xattr:
add_error("%s: can't restore xattr; xattr support missing.\n"
% path)
return
existing_xattrs = set(xattr.list(path, nofollow=True))
if self.linux_xattr:
for k, v in self.linux_xattr:
if k not in existing_xattrs \
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM \
or e.errno == errno.EOPNOTSUPP:
raise ApplyError('xattr.set: %s' % e)
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.remove: %s' % e)
else:
raise
def gfid_to_path(gfid):
"""
Try readlink, if it is directory it succeeds.
Get ctime of the GFID file, Decrement by 5 sec
Search for Changelog filename, Since Changelog file generated
every 15 sec, Search and get immediate next Changelog after the file
Creation. Get the Path by searching in Changelog file.
Get the resultant file's GFID and Compare with the input, If these
GFIDs are different then Some thing is changed(May be Rename)
"""
gfid = gfid.strip()
gpath = os.path.join(".glusterfs", gfid[0:2], gfid[2:4], gfid)
try:
output_success(full_dir_path(gfid))
return
except OSError:
# Not an SymLink
pass
try:
ctime = int(os.stat(gpath).st_ctime)
ctime -= DEC_CTIME_START
except (OSError, IOError):
output_not_found(gfid)
return
path = None
found_changelog = False
changelog_parse_try = 0
for i in range(CHANGELOG_SEARCH_MAX_TRY):
cl = os.path.join(".glusterfs/changelogs", "CHANGELOG.%s" % ctime)
try:
with open(cl, "rb") as f:
changelog_parse_try += 1
found_changelog = True
path = find_path_from_changelog(f, gfid)
if not path and changelog_parse_try < MAX_NUM_CHANGELOGS_TRY:
ctime += 1
continue
break
except (IOError, OSError) as e:
if e.errno == errno.ENOENT:
ctime += 1
else:
break
if not found_changelog:
output_not_found(gfid)
return
if not path:
output_not_found(gfid)
return
gfid1 = str(uuid.UUID(bytes=xattr.get(path, "trusted.gfid")))
if gfid != gfid1:
output_not_found(gfid)
return
output_success(path)
def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
if not xattr:
if self.linux_xattr:
add_error("%s: can't restore xattr; xattr support missing.\n"
% path_msg(path))
return
if not self.linux_xattr:
return
try:
existing_xattrs = set(xattr.list(path, nofollow=True))
except IOError as e:
if e.errno == errno.EACCES:
raise ApplyError('xattr.set %r: %s' % (path_msg(path), e))
else:
raise
for k, v in self.linux_xattr:
if k not in existing_xattrs \
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError as e:
if e.errno == errno.EPERM \
or e.errno == errno.EOPNOTSUPP:
raise ApplyError('xattr.set %r: %s' % (path_msg(path), e))
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError as e:
if e.errno in (errno.EPERM, errno.EACCES):
raise ApplyError('xattr.remove %r: %s' % (path_msg(path), e))
else:
raise
def testManyOps(self):
"""test many ops"""
fh, fname = self._getfile()
xattr.set(fh, self.USER_ATTR, self.USER_VAL)
VL = [self.USER_ATTR]
VN = [self.USER_NN]
for i in range(self.MANYOPS_COUNT):
self.assertEqual(xattr.list(fh), VL)
self.assertEqual(xattr.list(fh, namespace=EMPTY_NS), VL)
self.assertEqual(xattr.list(fh, namespace=NS_USER), VN)
for i in range(self.MANYOPS_COUNT):
self.assertEqual(xattr.get(fh, self.USER_ATTR), self.USER_VAL)
self.assertEqual(xattr.get(fh, self.USER_NN, namespace=NS_USER), self.USER_VAL)
for i in range(self.MANYOPS_COUNT):
self.assertEqual(xattr.get_all(fh), [(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fh, namespace=NS_USER), [(self.USER_NN, self.USER_VAL)])
def generateList():
del localfilelist[:]
del localreplicalist[:]
for root, dirs, files in os.walk(RootDir):
#localfilelist.append(os.path.relpath(root,RootDir))
prefx = os.path.relpath(root, RootDir)
if (prefx != '.'):
prefx = '/' + prefx
else:
prefx = ''
for f in files:
try:
att = (xattr.get(RootDir + prefx + '/' + f,
'user.comment')).decode()
except OSError:
att = 0
if (f.count('%') > 0):
fname = prefx + f.replace('%', '/')
localfilelist.append([fname, str(att)])
localreplicalist.append(fname)
else:
localfilelist.append([prefx + '/' + f, str(att)])
for d in dirs:
localfilelist.append([prefx + '/' + d + '/', str(-1)])
serverlist[SERVER_ID].filelist = localfilelist
for files in localfilelist:
print(files)
def _apply_linux_xattr_rec(self, path, restore_numeric_ids=False):
if not xattr:
if self.linux_xattr:
add_error("%s: can't restore xattr; xattr support missing.\n" %
path)
return
existing_xattrs = set(xattr.list(path, nofollow=True))
if self.linux_xattr:
for k, v in self.linux_xattr:
if k not in existing_xattrs \
or v != xattr.get(path, k, nofollow=True):
try:
xattr.set(path, k, v, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.set: %s' % e)
else:
raise
existing_xattrs -= frozenset([k])
for k in existing_xattrs:
try:
xattr.remove(path, k, nofollow=True)
except IOError, e:
if e.errno == errno.EPERM:
raise ApplyError('xattr.remove: %s' % e)
else:
raise
def _copyxattr(src, dest, exclude=None):
try:
attrs = xattr.list(src)
except IOError as e:
if e.errno != OperationNotSupported.errno:
raise
attrs = ()
if attrs:
if exclude is not None and isinstance(attrs[0], bytes):
exclude = exclude.encode(_encodings['fs'])
exclude = _get_xattr_excluder(exclude)
for attr in attrs:
if exclude(attr):
continue
try:
xattr.set(dest, attr, xattr.get(src, attr))
raise_exception = False
except IOError:
raise_exception = True
if raise_exception:
raise OperationNotSupported(
_("Filesystem containing file '%s' "
"does not support extended attribute '%s'") %
(_unicode_decode(dest), _unicode_decode(attr)))
def _copyxattr(src, dest, exclude=None):
try:
attrs = xattr.list(src)
except IOError as e:
if e.errno != OperationNotSupported.errno:
raise
attrs = ()
if attrs:
if exclude is not None and isinstance(attrs[0], bytes):
exclude = exclude.encode(_encodings['fs'])
exclude = _get_xattr_excluder(exclude)
for attr in attrs:
if exclude(attr):
continue
try:
xattr.set(dest, attr, xattr.get(src, attr))
raise_exception = False
except IOError:
raise_exception = True
if raise_exception:
raise OperationNotSupported(_("Filesystem containing file '%s' "
"does not support extended attribute '%s'") %
(_unicode_decode(dest), _unicode_decode(attr)))
def __get(self,path,key,default=None): try: return xattr.get(path,key,namespace=self.ns) except IOError as e: if not default is None and ENODATA == e.errno: return default else: raise NoValueException()
def test_change_cos(self):
self.test_file.write(data='asdfasdf',
hdrs={'X-Hpss-Class-Of-Service-Id': '2'})
test_file_name = os.path.join(self.hpss_dir,
self.account.name,
self.container.name,
'testfile')
print test_file_name
time.sleep(10) # It takes a long time for HPSS to get around to it.
self.assertEqual(xattr.get(test_file_name, 'system.hpss.cos'), '2')
self.test_file.post(hdrs={'X-Hpss-Class-Of-Service-Id': '1'})
time.sleep(10)
self.assertEqual(xattr.get(test_file_name, 'system.hpss.cos'), '1')
def run(args):
xtime_key = "trusted.glusterfs.%s.xtime" % args.vol_uuid
try:
print struct.unpack("!II",
xattr.get(args.path, xtime_key, nofollow=True))
except (OSError, IOError) as e:
print "[Error %s] %s" % (e.errno, os.strerror(e.errno))
sys.exit(-1)
def testBinaryPayload(self):
"""test binary values"""
fh, fname = self._getfile()
os.close(fh)
BINVAL = "abc" + '\0' + "def"
if PY3K:
BINVAL = BINVAL.encode()
xattr.set(fname, self.USER_ATTR, BINVAL)
self.checkList(xattr.list(fname), [self.USER_ATTR])
self.assertEqual(xattr.list(fname, namespace=NS_USER), [self.USER_NN])
self.assertEqual(xattr.get(fname, self.USER_ATTR), BINVAL)
self.assertEqual(xattr.get(fname, self.USER_NN, namespace=NS_USER),
BINVAL)
self.checkTuples(xattr.get_all(fname), [(self.USER_ATTR, BINVAL)])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER),
[(self.USER_NN, BINVAL)])
xattr.remove(fname, self.USER_ATTR)
def run(args):
xtime_key = "trusted.glusterfs.%s.xtime" % args.vol_uuid
try:
print struct.unpack("!II", xattr.get(args.path, xtime_key,
nofollow=True))
except (OSError, IOError) as e:
print "[Error %s] %s" % (e.errno, os.strerror(e.errno))
sys.exit(-1)
def read_cache(self):
if not xattr:
return
cache = dict([(n[13:], xattr.get(self.name, n)) for n in xattr.list(self.name) if n.decode().startswith('user.pyanidb.')])
if 'mtime' not in cache or str(int(self.mtime)) != cache.pop('mtime'):
return
for n, v in cache.items():
setattr(self, n, v)
self.cached = True
def getxattr(self, key, namespace=XATTR_DEFAULT_NS):
# we don't deal with types here, we just act as a dumb store
try:
return xattr.get(self.as_posix(), key, namespace=namespace)
except OSError as e:
if e.errno == errno.ENODATA or e.errno == errno.ENOATTR:
raise exc.NoStreamError((self, key)) from e
else:
raise e
def _copyxattr(src, dest):
for attr in xattr.list(src):
try:
xattr.set(dest, attr, xattr.get(src, attr))
raise_exception = False
except IOError:
raise_exception = True
if raise_exception:
raise OperationNotSupported("Filesystem containing file '%s' does not support extended attributes" % dest)
def get_sha1(self): name = self.get_real_name() if "user.sha1" in xattr.list(name): val = xattr.get(name, "user.sha1")[:-1] else: with open(name) as f: val = sha1(f.read()).hexdigest() xattr.setxattr(name, "user.sha1", "%s\x00" % val) return val
def get_gfid(path):
try:
return str(uuid.UUID(bytes=xattr.get(path, "trusted.gfid",
nofollow=True)))
except (IOError, OSError) as e:
logger.error("Unable to get GFID of {0}: {1}, skipping".format(
path, e))
return None
def get_gfid(path):
try:
return str(uuid.UUID(bytes=xattr.get(path, "trusted.gfid",
nofollow=True)))
except (IOError, OSError) as e:
logger.error("Unable to get GFID of {0}: {1}, skipping".format(
path, e))
return None
def test_many_ops(subject):
"""test many ops"""
item = subject[0]
xattr.set(item, USER_ATTR, USER_VAL)
VL = [USER_ATTR]
VN = [USER_NN]
for i in range(MANYOPS_COUNT):
lists_equal(xattr.list(item), VL)
lists_equal(xattr.list(item, namespace=EMPTY_NS), VL)
assert xattr.list(item, namespace=NAMESPACE) == VN
for i in range(MANYOPS_COUNT):
assert xattr.get(item, USER_ATTR) == USER_VAL
assert xattr.get(item, USER_NN, namespace=NAMESPACE) == USER_VAL
for i in range(MANYOPS_COUNT):
tuples_equal(xattr.get_all(item),
[(USER_ATTR, USER_VAL)])
assert xattr.get_all(item, namespace=NAMESPACE) == \
[(USER_NN, USER_VAL)]
def run(args):
try:
stime_key = "trusted.glusterfs.%s.%s.stime" % (args.master_uuid,
args.slave_uuid)
print struct.unpack("!II",
xattr.get(args.path, stime_key, nofollow=True))
except (OSError, IOError) as e:
sys.stderr.write("[Error %s] %s\n" % (e.errno, os.strerror(e.errno)))
sys.exit(-1)
def walk_data_tree(current_path,
toplevel=False,
pgfid_path='00/00/00000000-0000-0000-0000-000000000001'):
l = os.listdir(current_path)
for direntry in l:
if direntry in ('.', '..') or toplevel and direntry == '.glusterfs':
continue
ln = None
ln_full = None
link_stat = None
depath = os.path.join(current_path, direntry)
data_stat = os.lstat(depath)
if not S_ISLNK(data_stat.st_mode):
# Links don't have a gfid of their own.
try:
x = xattr.get(depath, 'trusted.gfid')
ln = format_gfid(x)
ln_full = os.path.join(os.getcwd(), ln)
try:
link_stat = os.lstat(ln)
except OSError, e:
print "Missing expected link tree object at %s" % ln_full
except IOError, e:
print "Missing trusted.gfid extended attribute for %s" % depath
if S_ISDIR(data_stat.st_mode):
# This is a directory, link tree object should be a symbolinc link to it's directory in the parent structure
#
if not link_stat:
print "Missing link for directory for %s, you should probably run " % depath
print "ln -s '../../%s/%s' '%s'" % (pgfid_path, direntry,
ln_full)
else:
if not S_ISLNK(link_stat.st_mode) or not os.readlink(
ln) == '../../%s/%s' % (pgfid_path, direntry):
print "Unexpected: %s does not link to what we expected it to (../../%s/%s)." % (
ln_full, pgfid_path, direntry)
walk_data_tree(depath, pgfid_path=ln)
if S_ISREG(data_stat.st_mode):
if not link_stat:
print "Missing hard link for file %s, you should probably run " % depath
print "ln '%s' '%s'" % (depath, ln_full)
else:
if link_stat.st_dev != data_stat.st_dev or link_stat.st_ino != data_stat.st_ino:
print "Unexpected: %s is not what we expected (hard link to %s)" % (
ln_full, depath)
def get_selinux_context(self, file_path):
"""
return a permissions for 'file_path'
:param file_path: str, path to the file
:return: str
"""
# what if SELinux is not enabled?
p = self.p(file_path)
return xattr.get(p, "security.selinux")
def lstags(path):
tags = xattr.get(path,'tags')
if os.path.isdir(path):
subtags = tags.split('/')
for t in subtags:
print t+'\t',
print ""
else:
print 'tags: '+tags
def generateRepList():
global localreplicalist
localreplicalist = os.listdir(replicDir)
for ind, files in enumerate(localreplicalist):
try:
att = (xattr.get(replicDir + '/' + files, 'user.comment')).decode()
except OSError:
att = 0
localreplicalist[ind] = [files.replace('%', '/'), str(att)]
serverlist[SERVER_ID].replicalist = localreplicalist
def __getitem__(self, key):
"""
Retrieves an extended filesystem attribute. Raises ``KeyError`` if the file does not
have an attribute with that name.
"""
try:
return self._decodeVal(xattr.get(self._path, key, namespace=xattr.NS_USER))
except OSError:
pass
raise KeyError(key)
def list_mergerfs():
mergerfs=re.compile(r'^[^ ]* (.*) fuse\.mergerfs.*')
sanitize=re.compile(r'=[^:]*')
mountpoint={}
with open('/proc/mounts','r') as mounts:
for line in mounts.read().splitlines():
(point, match)=re.subn(mergerfs,'\g<1>',line)
if match==1:
mountpoint[point]=re.sub(sanitize,'',xattr.get(os.path.join(point,'.mergerfs'),'user.mergerfs.branches').decode('utf-8')).split(':')
return mountpoint
def lstags(path):
tags = xattr.get(path, 'tags')
if os.path.isdir(path):
subtags = tags.split('/')
for t in subtags:
print t + '\t',
print ""
else:
print 'tags: ' + tags
def testMixedAccess(self):
"""test mixed access to file"""
fh, fname = self._getfile()
fo = os.fdopen(fh)
self.checkList(xattr.list(fname), [])
xattr.set(fname, self.USER_ATTR, self.USER_VAL)
self.checkList(xattr.list(fh), [self.USER_ATTR])
self.assertEqual(xattr.list(fh, namespace=NS_USER), [self.USER_NN])
self.assertEqual(xattr.get(fo, self.USER_ATTR), self.USER_VAL)
self.assertEqual(xattr.get(fo, self.USER_NN, namespace=NS_USER),
self.USER_VAL)
self.checkTuples(xattr.get_all(fo), [(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fo, namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
self.checkTuples(xattr.get_all(fname),
[(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fname, namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
fo.close()
def testManyOps(self):
"""test many ops"""
fh, fname = self._getfile()
xattr.set(fh, self.USER_ATTR, self.USER_VAL)
VL = [self.USER_ATTR]
VN = [self.USER_NN]
for i in range(self.MANYOPS_COUNT):
self.checkList(xattr.list(fh), VL)
self.checkList(xattr.list(fh, namespace=EMPTY_NS), VL)
self.assertEqual(xattr.list(fh, namespace=NS_USER), VN)
for i in range(self.MANYOPS_COUNT):
self.assertEqual(xattr.get(fh, self.USER_ATTR), self.USER_VAL)
self.assertEqual(xattr.get(fh, self.USER_NN, namespace=NS_USER),
self.USER_VAL)
for i in range(self.MANYOPS_COUNT):
self.checkTuples(xattr.get_all(fh),
[(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fh, namespace=NS_USER),
[(self.USER_NN, self.USER_VAL)])
def __getitem__(self, key):
"""
Retrieves an extended filesystem attribute. Raises ``KeyError`` if the file does not
have an attribute with that name.
"""
try:
return self._decodeVal(
xattr.get(self._path, key, namespace=xattr.NS_USER))
except OSError:
pass
raise KeyError(key)
def _copyxattr(src, dest):
for attr in xattr.list(src):
try:
xattr.set(dest, attr, xattr.get(src, attr))
raise_exception = False
except IOError:
raise_exception = True
if raise_exception:
raise OperationNotSupported(
"Filesystem containing file '%s' does not support extended attributes"
% dest)
def read_cache(self):
if not xattr:
return
cache = dict([(n[13:], xattr.get(self.name, n))
for n in xattr.list(self.name)
if n.decode().startswith('user.pyanidb.')])
if 'mtime' not in cache or str(int(self.mtime)) != cache.pop('mtime'):
return
for n, v in cache.items():
setattr(self, n, v)
self.cached = True
def test_handling_of_incorrect_existing_linux_xattrs():
if os.geteuid() != 0 or detect_fakeroot():
return
setup_testfs()
subprocess.check_call('rm -rf testfs/*', shell=True)
path = 'testfs/foo'
open(path, 'w').close()
xattr.set(path, 'foo', 'bar', namespace=xattr.NS_USER)
m = metadata.from_path(path, archive_path=path, save_symlinks=True)
xattr.set(path, 'baz', 'bax', namespace=xattr.NS_USER)
m.apply_to_path(path, restore_numeric_ids=False)
WVPASSEQ(xattr.list(path), ['user.foo'])
WVPASSEQ(xattr.get(path, 'user.foo'), 'bar')
xattr.set(path, 'foo', 'baz', namespace=xattr.NS_USER)
m.apply_to_path(path, restore_numeric_ids=False)
WVPASSEQ(xattr.list(path), ['user.foo'])
WVPASSEQ(xattr.get(path, 'user.foo'), 'bar')
xattr.remove(path, 'foo', namespace=xattr.NS_USER)
m.apply_to_path(path, restore_numeric_ids=False)
WVPASSEQ(xattr.list(path), ['user.foo'])
WVPASSEQ(xattr.get(path, 'user.foo'), 'bar')
os.chdir(top_dir)
cleanup_testfs()
def mode_volmarkmaster(args):
fmt_string = "!" + "B" * 19 + "II"
try:
vm = struct.unpack(fmt_string, xattr.get(args.path, "trusted.glusterfs.volume-mark", nofollow=True))
print "UUID : %s" % uuid.UUID("".join(["%02x" % x for x in vm[2:18]]))
print "VERSION : %s.%s" % vm[0:2]
print "RETVAL : %s" % vm[18]
print "VOLUME MARK : %s.%s (%s)" % (vm[19], vm[20], human_time("%s.%s" % (vm[19], vm[20])))
except (OSError, IOError) as e:
if e.errno == ENODATA:
pass
else:
print "[Error %s] %s" % (e.errno, os.strerror(e.errno))
sys.exit(-1)
def read_metadata(path):
"""
Helper function to read the pickled metadata from a File/Directory.
:param path: File/Directory to read metadata from.
:returns: dictionary of metadata
"""
metadata = None
metadata_s = ''
key = 0
while metadata is None:
try:
metadata_s += xattr.get(path, '%s%s' % (METADATA_KEY, (key or '')))
except IOError as err:
if err.errno == errno.ENODATA:
if key > 0:
# No errors reading the xattr keys, but since we have not
# been able to find enough chunks to get a successful
# unpickle operation, we consider the metadata lost, and
# drop the existing data so that the internal state can be
# recreated.
clean_metadata(path)
# We either could not find any metadata key, or we could find
# some keys, but were not successful in performing the
# unpickling (missing keys perhaps)? Either way, just report
# to the caller we have no metadata.
metadata = {}
else:
logging.exception("xattr.get failed on %s key %s err: %s",
path, key, str(err))
# Note that we don't touch the keys on errors fetching the
# data since it could be a transient state.
raise
else:
try:
# If this key provides all or the remaining part of the pickle
# data, we don't need to keep searching for more keys. This
# means if we only need to store data in N xattr key/value
# pair, we only need to invoke xattr get N times. With large
# keys sizes we are shooting for N = 1.
metadata = pickle.loads(metadata_s)
assert isinstance(metadata, dict)
except EOFError, pickle.UnpicklingError:
# We still are not able recognize this existing data collected
# as a pickled object. Make sure we loop around to try to get
# more from another xattr key.
metadata = None
key += 1
def _ui_origin_url(filename):
# See: http://www.freedesktop.org/wiki/CommonExtendedAttributes
try:
return xattr.get(filename, 'user.xdg.origin.url')
except IOError, e:
ok = False
if e.errno in (errno.ENODATA,
errno.EOPNOTSUPP, errno.E2BIG, errno.ERANGE):
ok = True
for me in ("ENOATTR", "ENOTSUPP"):
if hasattr(errno, me) and e.errno == getattr(errno, me):
ok = True
break
if not ok:
raise
def __iter__(self):
# construct path to storage file
storage_file = self._data_file
# first check if file exists
if os.path.isfile(storage_file):
status = None
corrupted = 0;
# file exists, now check extended attributes
try:
status = xattr.get(storage_file, 'user.vs.online')
file_stat = os.stat(storage_file)
try:
corrupted_ea = xattr.get(storage_file, 'user.vs.corrupted')
corrupted = ord(corrupted_ea[0])
except Exception, ex:
pass
except IOError:
pass
# status meta-data must exist
if status is None:
self._logger.info("status for %s is none." % self._data_file)
# user.vsvfs.online, 1 for online, 2 for offline.
if (status is not None and (ord(status[0]) == 1 or corrupted == 1)):
# do normal auditing
self._logger.info("Performing normal object audit of file %s" % self._data_file)
for chunk in super(VSDiskFileReader, self).__iter__():
if chunk:
yield chunk
return
# file not completely in cache, skip auditing
self._logger.info("Skipping object audit of file %s because storage file %s doesn't exist with full content "
% (self._data_file, storage_file))
# close file when done
if not self._suppress_file_closing:
self.close()
def _scanfolder(self):
"""Cache the message list from a Maildir.
Maildir flags are: R (replied) S (seen) T (trashed) D (draft) F
(flagged).
:returns: dict that can be used as self.messagelist"""
maxage = self.config.getdefaultint("Account " + self.accountname,
"maxage", None)
maxsize = self.config.getdefaultint("Account " + self.accountname,
"maxsize", None)
retval = {}
files = []
nouidcounter = -1 # Messages without UIDs get negative UIDs.
for dirannex in ['new', 'cur']:
fulldirname = os.path.join(self.getfullname(), dirannex)
files.extend((dirannex, filename) for
filename in os.listdir(fulldirname))
for dirannex, filename in files:
# We store just dirannex and filename, ie 'cur/123...'
filepath = os.path.join(dirannex, filename)
# check maxage/maxsize if this message should be considered
if maxage and not self._iswithinmaxage(filename, maxage):
continue
if maxsize and (os.path.getsize(os.path.join(
self.getfullname(), filepath)) > maxsize):
continue
(prefix, uid, fmd5, maildirflags) = self._parse_filename(filename)
if uid is None: # assign negative uid to upload it.
uid = nouidcounter
nouidcounter -= 1
else: # It comes from our folder.
uidmatch = re_uidmatch.search(filename)
uid = None
if not uidmatch:
uid = nouidcounter
nouidcounter -= 1
else:
uid = long(uidmatch.group(1))
# 'filename' is 'dirannex/filename', e.g. cur/123,U=1,FMD5=1:2,S
retval[uid] = {'flags': set((xattr.get(os.path.join(self.getfullname(), filepath),
'org.offlineimap.flags',
namespace=xattr.NS_USER)).split()),
'filename': filepath}
return retval
def process(path):
# get tags
try: attrvalue = xattr.get(path, ATTRNAME)
except IOError: tags = set();
else: tags = set(attrvalue.split("\x1f"))
oldtags = tags
# perform operations
if options.filter and not set(options.filter) <= tags: return
if options.clear: tags = []
if options.add: tags = tags | set(options.add)
if options.delete: tags = tags - set(options.delete)
if not tags == oldtags:
if tags:
xattr.set(path, ATTRNAME, "\x1f".join(tags))
else:
try: xattr.remove(path, ATTRNAME)
except IOError: pass
if options.list: print path, "::", ", ".join(tags)
def dump_xattrs(pathnames, file_out): """Dump the xattr data for |pathnames| to |file_out|""" # NOTE: Always quote backslashes, in order to ensure that they are # not interpreted as quotes when they are processed by unquote. quote_chars = b'\n\r\\\\' for pathname in pathnames: attrs = xattr.list(pathname) if not attrs: continue file_out.write(b'# file: %s\n' % quote(pathname, quote_chars)) for attr in attrs: attr = unicode_encode(attr) value = xattr.get(pathname, attr) file_out.write(b'%s="%s"\n' % ( quote(attr, b'=' + quote_chars), quote(value, b'\0"' + quote_chars)))
def get_object_uuid(real_path):
fd = os.open(real_path, os.O_RDONLY)
fcntl.flock(fd, fcntl.LOCK_EX)
try:
try:
object_uuid = xattr.get(real_path, xattr_name)
except IOError, e:
if e.errno != errno.ENODATA:
logger.exception("xattr.get:%s" % real_path)
raise e
object_uuid = uuid.uuid4().hex
try:
mode = os.fstat(fd).st_mode
if not (mode & stat.S_IWUSR): # read only は強制解除
os.fchmod(fd, mode | stat.S_IWUSR)
xattr.set(real_path, xattr_name, object_uuid)
except:
logger.exception("xattr.set:%s" % real_path)
raise
return object_uuid
