def verify(self, target_dir):
""""""
with pushd(target_dir):
for f in self.source_files.keys():
fp = os.path.join(target_dir, f)
attrs = os.listxattr(path=fp, follow_symlinks=False)
assert len(attrs) == self.xattr_pairs
for k in self.source_xattrs[f].keys():
v = os.getxattr(fp, k.encode(self.encoding)).decode(
self.encoding)
assert v == self.source_xattrs[f][k]
attrs = os.listxattr(fp, follow_symlinks=False)
if self.encoding != "gb2312":
for attr in attrs:
v = xattr.getxattr(f, attr)
assert attr in self.source_xattrs[f].keys()
assert v.decode(
self.encoding) == self.source_xattrs[f][attr]
with pushd(target_dir):
for d in self.source_dirs.keys():
dp = os.path.join(target_dir, d)
attrs = xattr.listxattr(dp)
assert len(attrs) == self.xattr_pairs
for attr in attrs:
v = xattr.getxattr(d, attr)
assert attr in self.source_dirs_xattrs[d].keys()
assert v.decode(
self.encoding) == self.source_dirs_xattrs[d][attr]
def test_mixed_access(testdir, gen):
"""test mixed access to file"""
with gen(testdir) as (a, b):
# Check empty
lists_equal(xattr.list(a), [])
lists_equal(xattr.listxattr(b), [])
# Check value
xattr.set(a, USER_ATTR, USER_VAL)
for i in [a, b]:
# Deprecated functions
lists_equal(xattr.listxattr(i), [USER_ATTR])
assert xattr.getxattr(i, USER_ATTR) == USER_VAL
tuples_equal(xattr.get_all(i), [(USER_ATTR, USER_VAL)])
# Current functions
lists_equal(xattr.list(i), [USER_ATTR])
assert xattr.list(i, namespace=NAMESPACE) == [USER_NN]
assert xattr.get(i, USER_ATTR) == USER_VAL
assert xattr.get(i, USER_NN, namespace=NAMESPACE) == USER_VAL
tuples_equal(xattr.get_all(i),
[(USER_ATTR, USER_VAL)])
assert xattr.get_all(i, namespace=NAMESPACE) == \
[(USER_NN, USER_VAL)]
# Overwrite
xattr.set(b, USER_ATTR, LARGE_VAL, flags=xattr.XATTR_REPLACE)
assert xattr.get(a, USER_ATTR) == LARGE_VAL
assert xattr.getxattr(a, USER_ATTR) == LARGE_VAL
xattr.removexattr(b, USER_ATTR)
assert xattr.get_all(a, namespace=NAMESPACE) == []
assert xattr.get_all(b, namespace=NAMESPACE) == []
def test_no_attributes_deprecated(any_subject):
"""test no attributes (deprecated functions)"""
item, nofollow = any_subject
lists_equal(xattr.listxattr(item, True), [])
tuples_equal(xattr.get_all(item, True), [])
with pytest.raises(EnvironmentError):
xattr.getxattr(item, USER_ATTR, True)
def post_crash_processing(corefile):
try:
# Name of the Crashed process
# ToFix :: In case of python based daemon crash, this value just
# has the name "python", it doesn't provide information
# on which daemon crashed. This needs to be improved.
process = xattr.getxattr(corefile, 'user.coredump.comm')
# Signal Number leading to crash
signal = int(xattr.getxattr(corefile, 'user.coredump.signal'))
if signal >= len(strsignal):
signal = 0
# Timestamp of the crash event
timestamp = xattr.getxattr(corefile, 'user.coredump.timestamp')
timestamp_human = datetime.datetime.fromtimestamp(
int(timestamp)
).strftime('%Y-%m-%d %H:%M:%S')
ops_eventlog.log_event('SUPPORTABILITY_DAEMON_CRASH',
['process', process],
['signal', strsignal[signal]],
['timestamp', timestamp_human])
except:
vlog.dbg("Invalid core dump file found")
def read(self):
try:
self.ts = xattr.getxattr(fsencode(self.filename), 'user.shatag.ts').decode('ascii')
self.shatag = xattr.getxattr(fsencode(self.filename), 'user.shatag.sha256').decode('ascii')
except IOError as e:
if e.errno != errno.ENODATA: # no tag present
raise e
def _message_at_path(self, path, load_content=True):
try:
content = None
if load_content:
f = open(path, "rb")
content = f.read()
f.close()
mtime = os.path.getmtime(path)
directory, filename = os.path.split(path)
directory, subdir = os.path.split(directory)
msgid = None
info = None
parts = filename.split(":")
if len(parts) > 0:
msgid = parts[0]
if len(parts) > 1:
info = parts[1]
msg = Message(content=content, msgid=msgid, info=info, subdir=subdir, mtime=mtime)
if not msg.msg_md5 and self._use_xattrs and load_content:
try:
xattrs = xattr.listxattr(path)
# logging.debug(xattrs)
if XATTR_MD5SUM in xattrs:
msg.msg_md5 = xattr.getxattr(path, XATTR_MD5SUM)
# logging.debug("Read md5: %s", msg.msg_md5)
else:
c = msg.content_hash
if c:
# logging.debug("Setting shasum xattr: %r", c)
xattr.setxattr(path, XATTR_MD5SUM, c)
else:
logging.warning("Could not generate content hash of %s", msgid)
if XATTR_DATE in xattrs:
msg._date = xattr.getxattr(path, XATTR_DATE).decode("utf8")
msg._date = datetime.datetime.fromtimestamp(float(msg._date))
# logging.debug("Read date: %s", msg._date)
else:
d = str(msg.date.timestamp()).encode("utf8")
if d:
# logging.debug("Setting date xattr: %r", d)
xattr.setxattr(path, XATTR_DATE, d)
else:
logging.warning("Could not determine message date of %s", msgid)
except IOError:
# read-only FS, unsupported on FS, etc.
self._use_xattrs = False
log.debug("host filesystem for %s does not support xattrs; disabling" % self.name)
return msg
except OSError:
raise KeyError
def _mount_quobyte(self, quobyte_volume, mount_path, ensure=False):
"""Mount Quobyte volume to mount path."""
mounted = False
for l in QuobyteDriver.read_proc_mount():
if l.split()[1] == mount_path:
mounted = True
break
if mounted:
try:
os.stat(mount_path)
except OSError as exc:
if exc.errno == errno.ENOTCONN:
mounted = False
try:
LOG.info(_LI('Fixing previous mount %s which was not'
' unmounted correctly.') % mount_path)
self._execute('umount.quobyte', mount_path,
run_as_root=False)
except processutils.ProcessExecutionError as exc:
LOG.warn(_LW("Failed to unmount previous mount: %s"),
exc)
else:
# TODO(quobyte): Extend exc analysis in here?
LOG.warn(_LW("Unknown error occurred while checking mount"
" point: %s Trying to continue."), exc)
if not mounted:
if not os.path.isdir(mount_path):
self._execute('mkdir', '-p', mount_path)
command = ['mount.quobyte', quobyte_volume, mount_path]
if self.configuration.quobyte_client_cfg:
command.extend(['-c', self.configuration.quobyte_client_cfg])
try:
LOG.info(_LI('Mounting volume: %s ...') % quobyte_volume)
self._execute(*command, run_as_root=False)
LOG.info(_LI('Mounting volume: %s succeeded') % quobyte_volume)
mounted = True
except processutils.ProcessExecutionError as exc:
if ensure and 'already mounted' in exc.stderr:
LOG.warn(_LW("%s is already mounted"), quobyte_volume)
else:
raise
if mounted:
try:
xattr.getxattr(mount_path, 'quobyte.info')
except Exception as exc:
msg = _LE("The mount %(mount_path)s is not a valid"
" Quobyte USP volume. Error: %(exc)s") \
% {'mount_path': mount_path, 'exc': exc}
raise exception.VolumeDriverException(msg)
if not os.access(mount_path, os.W_OK | os.X_OK):
LOG.warn(_LW("Volume is not writable. Please broaden the file"
" permissions. Mount: %s"), mount_path)
def get_attrs(self, uuid):
attrs = {}
tags = xattr.getxattr(os.path.join(self.basepath, uuid), 'user.tags')
flags = xattr.getxattr(os.path.join(self.basepath, uuid), 'user.flags')
stored = xattr.getxattr(os.path.join(self.basepath, uuid), 'user.stored')
attrs['tags'] = json.loads(tags)
attrs['flags'] = json.loads(flags)
attrs['stored'] = int(stored)
return attrs
def read_mounttime(self, repo_mountpoint, timeout):
start = time.time()
self.safe_scandir(repo_mountpoint, timeout)
end = time.time()
# Did we really mount it ?
try:
xattr.getxattr(repo_mountpoint, 'user.fqrn') == repo_mountpoint
return end - start
except:
raise Exception("Repository was not mounted correctly")
def on_get(self, req, resp, cn):
try:
path, buf, cert = authority.get_signed(cn)
return dict(last_seen=xattr.getxattr(path, "user.lease.last_seen"),
inner_address=xattr.getxattr(
path, "user.lease.inner_address").decode("ascii"),
outer_address=xattr.getxattr(
path, "user.lease.outer_address").decode("ascii"))
except EnvironmentError: # Certificate or attribute not found
raise falcon.HTTPNotFound()
def read(self):
try:
self.ts = xattr.getxattr(fsencode(self.filename),
'user.shatag.ts').decode('ascii')
self.shatag = xattr.getxattr(fsencode(self.filename),
'user.shatag.sha256').decode('ascii')
except IOError as e:
if e.errno != errno.ENODATA: # no tag present
raise e
def run_expunge_dirname (dirname):
orig_cwd = os.getcwd()
execute_print ("orig dir: " + orig_cwd)
os.mkdir (dirname)
execute("kill -9 `cat /etc/glusterd/vols/vol/run/"+HOSTNAME+"-home-gluster1.pid`");
execute_print (binascii.b2a_hex (xattr.getxattr ("/home/gluster2/"+dirname, "trusted.gfid")))
os.rmdir(dirname)
execute_print (xattr.getxattr ("/home/gluster2/", "trusted.afr.vol-client-0").encode('base64'))
execute ("gluster volume start vol force")
execute ("sleep 20")
os.listdir ('.')
def read_xattr_metadata(file_list):
"""For the given list of files read us the kMDItemDescription and kMDItemKeywords."""
meta_data = []
for file in file_list:
md = {}
attrs = xattr.listxattr(file)
if 'com.apple.metadata:kMDItemDescription' in attrs:
md['Caption-Abstract'] = biplist.readPlistFromString(xattr.getxattr(file, 'com.apple.metadata:kMDItemDescription'))
if 'com.apple.metadata:kMDItemKeywords' in attrs:
md['Keywords'] = biplist.readPlistFromString(xattr.getxattr(file, 'com.apple.metadata:kMDItemKeywords'))
meta_data.append(md)
return meta_data
def check_file_status(self,index,file,status='done'):
'''check the file attrs
should return True if the status=status, False if not'''
attr='user.%s_%s'%(self.name,index)
try:
v=xattr.getxattr(file.path,attr+'.'+status)
if not self.cfg.get('updated'): return True #is done
try:
mtime=float(xattr.getxattr(file.path,attr+'.mtime'))
if mtime != file.stat().st_mtime: return False
except: pass #no mtime attr, consider done
return True
except: return False #attr not set
def on_get(self, req, resp, cn):
preferred_type = req.client_prefers(("application/json", "application/x-pem-file"))
try:
path, buf, cert, signed, expires = self.authority.get_signed(cn)
except EnvironmentError:
logger.warning("Failed to serve non-existant certificate %s to %s",
cn, req.context.get("remote_addr"))
raise falcon.HTTPNotFound()
if preferred_type == "application/x-pem-file":
resp.set_header("Content-Type", "application/x-pem-file")
resp.set_header("Content-Disposition", ("attachment; filename=%s.pem" % cn))
resp.body = buf
logger.debug("Served certificate %s to %s as application/x-pem-file",
cn, req.context.get("remote_addr"))
elif preferred_type == "application/json":
resp.set_header("Content-Type", "application/json")
resp.set_header("Content-Disposition", ("attachment; filename=%s.json" % cn))
try:
signer_username = getxattr(path, "user.signature.username").decode("ascii")
except IOError:
signer_username = None
attributes = {}
for key in listxattr(path):
if key.startswith(b"user.machine."):
attributes[key[13:].decode("ascii")] = getxattr(path, key).decode("ascii")
# TODO: dedup
resp.body = json.dumps(dict(
common_name = cn,
signer = signer_username,
serial = "%x" % cert.serial_number,
organizational_unit = cert.subject.native.get("organizational_unit_name"),
signed = cert["tbs_certificate"]["validity"]["not_before"].native.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z",
expires = cert["tbs_certificate"]["validity"]["not_after"].native.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z",
sha256sum = hashlib.sha256(buf).hexdigest(),
attributes = attributes or None,
lease = None,
extensions = dict([
(e["extn_id"].native, e["extn_value"].native)
for e in cert["tbs_certificate"]["extensions"]
if e["extn_value"] in ("extended_key_usage",)])
))
logger.debug("Served certificate %s to %s as application/json",
cn, req.context.get("remote_addr"))
else:
logger.debug("Client did not accept application/json or application/x-pem-file")
raise falcon.HTTPUnsupportedMediaType(
"Client did not accept application/json or application/x-pem-file")
def target_supports_extended_attributes(self):
"""
Check if the target directory supports extended filesystem
attributes
:rtype: bool
"""
try:
xattr.getxattr(self.target_dir, 'user.mime_type')
except Exception as e:
if format(e).startswith('[Errno 95]'):
# libc interface [Errno 95] Operation not supported:
return False
return True
def main():
masterurl = sys.argv[1]
slaveurl = sys.argv[2]
slave_node, slavevol = parse_url(slaveurl)
master_node, mastervol = parse_url(masterurl)
master_mnt = tempfile.mkdtemp()
slave_mnt = tempfile.mkdtemp()
try:
print "Mounting master volume on a temp mnt_pnt"
os.system("glusterfs -s %s --volfile-id %s %s" % (master_node,
mastervol,
master_mnt))
except:
print("Failed to mount the master volume")
cleanup(master_mnt, slave_mnt)
sys.exit(1)
try:
print "Mounting slave voluem on a temp mnt_pnt"
os.system("glusterfs -s %s --volfile-id %s %s" % (slave_node, slavevol,
slave_mnt))
except:
print("Failed to mount the master volume")
cleanup(master_mnt, slave_mnt)
sys.exit(1)
slave_file_list = [slave_mnt]
for top, dirs, files in os.walk(slave_mnt, topdown=False):
for subdir in dirs:
slave_file_list.append(os.path.join(top, subdir))
for file in files:
slave_file_list.append(os.path.join(top, file))
# chdir and then get the gfid, so that you don't need to replace
gfid_attr = 'glusterfs.gfid'
ret = 0
for sfile in slave_file_list:
mfile = sfile.replace(slave_mnt, master_mnt)
if xattr.getxattr(sfile, gfid_attr, True) != xattr.getxattr(
mfile, gfid_attr, True):
print ("gfid of file %s in slave is different from %s" +
" in master" % (sfile, mfile))
ret = 1
cleanup(master_mnt, slave_mnt)
sys.exit(ret)
def target_supports_extended_attributes(self):
"""
Check if the target directory supports extended filesystem
attributes
:rtype: bool
"""
try:
xattr.getxattr(self.target_dir, 'user.mime_type')
except Exception as e:
if format(e).startswith('[Errno 95]'):
# libc interface [Errno 95] Operation not supported:
return False
return True
def get_file(id):
source_file = file_from_id(id)
with open(source_file, 'rb') as f:
source = f.read().decode('utf-8')
try:
language = xattr.getxattr(source_file, 'user.upaste.language').decode('utf-8')
except:
language = config.DEFAULT_LANGUAGE
try:
original_id = xattr.getxattr(source_file, 'user.upaste.original_id').decode('utf-8')
except:
original_id = None
return source, language, original_id
def serialize_certificates(g):
for common_name, path, buf, obj, server in g():
# Extract certificate tags from filesystem
try:
tags = []
for tag in getxattr(path, "user.xdg.tags").split(","):
if "=" in tag:
k, v = tag.split("=", 1)
else:
k, v = "other", tag
tags.append(dict(id=tag, key=k, value=v))
except IOError: # No such attribute(s)
tags = None
attributes = {}
for key in listxattr(path):
if key.startswith("user.machine."):
attributes[key[13:]] = getxattr(path, key)
# Extract lease information from filesystem
try:
last_seen = datetime.strptime(
getxattr(path, "user.lease.last_seen"),
"%Y-%m-%dT%H:%M:%S.%fZ")
lease = dict(
inner_address=getxattr(path,
"user.lease.inner_address"),
outer_address=getxattr(path,
"user.lease.outer_address"),
last_seen=last_seen,
age=datetime.utcnow() - last_seen)
except IOError: # No such attribute(s)
lease = None
yield dict(
serial_number="%x" % obj.serial_number,
common_name=common_name,
server=server,
# TODO: key type, key length, key exponent, key modulo
signed=obj["tbs_certificate"]["validity"]
["not_before"].native,
expires=obj["tbs_certificate"]["validity"]
["not_after"].native,
sha256sum=hashlib.sha256(buf).hexdigest(),
lease=lease,
tags=tags,
attributes=attributes or None,
)
def main(self):
source_file = self.env['source_file']
destination_file = self.env["destination_file"]
# Check if we're trying to copy something inside a dmg.
(dmg_path, dmg, dmg_source_path) = self.parsePathForDMG(source_file)
try:
if dmg:
# Mount dmg and copy path inside.
mount_point = self.mount(dmg_path)
source_file = os.path.join(mount_point, dmg_source_path)
source_file = source_file.decode("string_escape")
destination_file = destination_file.decode("string_escape")
xattr.listxattr(source_file)
temp = xattr.getxattr(source_file, "com.apple.ResourceFork")
xattr.setxattr(destination_file, "com.apple.ResourceFork",
temp)
self.output("Resource Fork copied")
except:
var = traceback.format_exc()
print(("ERROR:", var))
self.output("Resoruce Fork error: " + var)
finally:
if dmg:
self.unmount(dmg_path)
def _checkDeprecated(self, item, symlink=False):
"""check deprecated list, set, get operations against an item"""
self.checkList(xattr.listxattr(item, symlink), [])
self.assertRaises(EnvironmentError, xattr.setxattr, item,
self.USER_ATTR, self.USER_VAL, XATTR_REPLACE)
try:
xattr.setxattr(item, self.USER_ATTR, self.USER_VAL, 0, symlink)
except IOError:
err = sys.exc_info()[1]
if symlink and (err.errno == errno.EPERM
or err.errno == errno.ENOENT):
# symlinks may fail, in which case we abort the rest
# of the test for this case (Linux returns EPERM; OS X returns ENOENT)
return
raise
self.assertRaises(EnvironmentError, xattr.setxattr, item,
self.USER_ATTR, self.USER_VAL, XATTR_CREATE)
self.checkList(xattr.listxattr(item, symlink), [self.USER_ATTR])
self.assertEqual(xattr.getxattr(item, self.USER_ATTR, symlink),
self.USER_VAL)
self.checkTuples(xattr.get_all(item, nofollow=symlink),
[(self.USER_ATTR, self.USER_VAL)])
xattr.removexattr(item, self.USER_ATTR)
self.checkList(xattr.listxattr(item, symlink), [])
self.checkTuples(xattr.get_all(item, nofollow=symlink), [])
self.assertRaises(EnvironmentError, xattr.removexattr, item,
self.USER_ATTR)
def get_local_files (self, channels):
downloads_dir = self.config.config['downloads']
files = []
for channel in channels:
if '..' in channel:
continue
dir_fp = os.path.join (downloads_dir, channel)
if not os.path.exists (dir_fp):
continue
if not os.path.isdir (dir_fp):
continue
for f in os.listdir (dir_fp):
if f[0] in '.#~':
continue
fname = os.path.join (channel, f)
fname_fp = os.path.join (dir_fp, f)
fname_size = os.path.getsize(fname_fp)
utils.set_md5_attr (fname_fp)
fname_md5 = xattr.getxattr (fname_fp, 'md5')
files.append ({'path': fname, 'size': fname_size, 'md5': fname_md5})
return files
def get_raw_tags(path):
if 'com.apple.metadata:_kMDItemUserTags' in xattr.listxattr(path):
d = xattr.getxattr(path, 'com.apple.metadata:_kMDItemUserTags')
d = biplist.readPlistFromString(d)
return d
else:
return []
def read_metadata(fd, md_key=None):
"""
Helper function to read the pickled metadata from an object file.
:param fd: file descriptor or filename to load the metadata from
:param md_key: metadata key to be read from object file
:returns: dictionary of metadata
"""
meta_key = SWIFT_METADATA_KEY
metadata = ''
key = 0
try:
while True:
metadata += xattr.getxattr(fd, '%s%s' % (meta_key,
(key or '')))
key += 1
except (IOError, OSError) as e:
if metadata == '':
return False
for err in 'ENOTSUP', 'EOPNOTSUPP':
if hasattr(errno, err) and e.errno == getattr(errno, err):
msg = "Filesystem at %s does not support xattr" % \
_get_filename(fd)
logging.exception(msg)
raise DiskFileXattrNotSupported(e)
if e.errno == errno.ENOENT:
raise DiskFileNotExist()
return pickle.loads(metadata)
def read_metadata(self, fd, obj_path):
"""
Helper function to read the pickled metadata from an object file.
:param fd: file descriptor or filename to load the metadata from
:param filename: full path of the file
:returns: dictionary of metadata
"""
metadata = ''
key = 0
try:
while True:
metadata += xattr.getxattr(
fd, '%s%s' % (SWIFT_METADATA_KEY, (key or '')))
key += 1
except (IOError, OSError) as e:
if metadata == '':
return False
for err in 'ENOTSUP', 'EOPNOTSUPP':
if hasattr(errno, err) and e.errno == getattr(errno, err):
msg = "Filesystem at %s does not support xattr" % \
obj_path
logging.exception(msg)
raise DiskFileXattrNotSupported(e)
if e.errno == errno.ENOENT:
raise DiskFileNotExist()
return pickle.loads(metadata)
def getMacCreatorAndType(path): """Returns file creator and file type codes for a path. Args: path (str): A file path. Returns: A tuple of two :py:class:`fontTools.py23.Tag` objects, the first representing the file creator and the second representing the file type. """ if xattr is not None: try: finderInfo = xattr.getxattr(path, 'com.apple.FinderInfo') except (KeyError, IOError): pass else: fileType = Tag(finderInfo[:4]) fileCreator = Tag(finderInfo[4:8]) return fileCreator, fileType if MacOS is not None: fileCreator, fileType = MacOS.GetCreatorAndType(path) if sys.version_info[:2] < (2, 7) and sys.byteorder == "little": # work around bug in MacOS.GetCreatorAndType() on intel: # http://bugs.python.org/issue1594 # (fixed with Python 2.7) fileCreator = _reverseString(fileCreator) fileType = _reverseString(fileType) return fileCreator, fileType else: return None, None
def getDownloadedFiles():
idList = [
xattr.getxattr(os.path.join('Downloaded', file), 'user.docid')
for file in os.listdir('./Downloaded')
]
# generator because it should only be compared once.
yield idList
def get_path_remote_id(path: Path, name: str = "ndrive") -> str:
"""Get a given extended attribute from a file/folder."""
try:
return (xattr.getxattr(str(path), f"user.{name}").decode(
"utf-8", errors="ignore") or "")
except OSError:
return ""
def _get_extended_attr(file_path, attr):
"""Get extended attributes from a file, returns an array of strings or None if no value is set.
Inspired by https://gist.github.com/dunhamsteve/2889617
Args:
file_path: str path of file to examine
attr: key of the attribute to retrieve
Returns:
a list of strings or None
"""
try:
xattr_val = getxattr(file_path, attr)
if xattr_val.startswith('bplist'):
try:
plist_array, _, plist_error = Foundation.NSPropertyListSerialization.propertyListWithData_options_format_error_(
buffer(xattr_val), 0, None, None)
if plist_error:
Logger.log_error(message='plist de-serialization error: {0}'.format(plist_error))
return None
return list(plist_array)
except Exception as deserialize_plist_e:
Logger.log_exception(deserialize_plist_e, message='_get_extended_attr failed on {0} for {1}'.format(file_path, attr))
else:
return [xattr_val]
except KeyError:
pass # ignore missing key in xattr
return None
def _getxattr(file, key):
try:
val = xattr.getxattr(file, key)
except Exception as err:
return False, err
return True, val
class LocalClient(LocalClientMixin):
"""macOS Client API implementation."""
def change_created_time(self, filepath: Path, d_ctime: datetime, /) -> None:
"""Change the created time of a given file."""
cmd = ["touch", "-mt", d_ctime.strftime("%Y%m%d%H%M.%S"), str(filepath)]
try:
subprocess.check_call(cmd)
except subprocess.CalledProcessError:
# Note: This is mostly due to the new Apple security layer asking for permissions.
# Note: Passing "exc_info=True" is useless as there will be no useful details.
log.warning(f"Cannot change the created time of {filepath!r}")
def has_folder_icon(self, ref: Path, /) -> bool:
"""Check if the folder icon is set."""
return (self.abspath(ref) / "Icon\r").is_file()
@staticmethod
def get_path_remote_id(path: Path, /, *, name: str = "ndrive") -> str:
"""Get a given extended attribute from a file/folder."""
try:
return (
xattr.getxattr(str(path), name).decode("utf-8", errors="ignore") or ""
)
except OSError:
return ""
def is_partition_supported(self, folder):
if folder is None:
return False
result = False
to_delete = not os.path.exists(folder)
try:
if to_delete:
os.mkdir(folder)
if not os.access(folder, os.W_OK):
import stat
os.chmod(folder, stat.S_IXUSR | stat.S_IRGRP | stat.S_IXGRP |
stat.S_IRUSR | stat.S_IWGRP | stat.S_IWUSR)
import xattr
attr = "drive-test"
xattr.setxattr(folder, attr, attr)
if xattr.getxattr(folder, attr) == attr:
result = True
xattr.removexattr(folder, attr)
finally:
try:
if to_delete:
os.rmdir(folder)
except:
pass
return result
def get_dir_layout(root_dir_path):
dir_map = dict()
for dir_path, dir_names, filenames in os.walk(root_dir_path):
if len(dir_map) == 0:
dir_map[dir_path] = DirLayout() # Root folder
else:
par_dir_path = os.path.dirname(dir_path)
par_dir_layout = dir_map[par_dir_path]
# Create the subfolder
dir_name = os.path.basename(dir_path)
par_dir_layout.add_subfolder(dir_name)
dir_map[dir_path] = par_dir_layout.get_subfolder(dir_name)
dir_layout = dir_map[dir_path]
# TODO: Ignore this part first. Figure out what to do with this folders
if '.glusterfs' in dir_names:
dir_names.remove('.glusterfs')
if '.trashcan' in dir_names:
dir_names.remove('.trashcan')
ext_attr = xattr.getxattr(dir_path, 'trusted.glusterfs.dht')
(dummy, start, end) = struct.unpack_from(">qII", ext_attr)
dir_layout.set_layout(Layout(start, end))
return dir_map[root_dir_path]
def _test_xattr_on_file_with_contents(filename, file_contents, xattrs=[], xar_create_flags=[], xar_extract_flags=[]):
try:
# Write file out
with open(filename, "w") as f:
f.write(file_contents)
for (key, value) in xattrs:
xattr.setxattr(f, key, value)
# Store it into a xarchive
archive_name = "{f}.xar".format(f=filename)
with util.archive_created(archive_name, filename, *xar_create_flags) as path:
# Extract xarchive
with util.directory_created("extracted") as directory:
# Validate resulting xattrs
subprocess.check_call(["xar", "-x", "-C", directory, "-f", path] + xar_extract_flags)
for (key, value) in xattrs:
try:
assert xattr.getxattr(os.path.join(directory, filename), key) == value, "extended attribute \"{n}\" has incorrect contents after extraction".format(n=key)
except KeyError:
raise MissingExtendedAttributeError("extended attribute \"{n}\" missing after extraction".format(n=key))
# Validate file contents
with open(os.path.join(directory, filename), "r") as f:
if f.read() != file_contents:
raise MissingExtendedAttributeError("archived file \"{f}\" has has incorrect contents after extraction".format(f=filename))
finally:
os.unlink(filename)
def get_local_files(self, channels):
downloads_dir = self.config.config['downloads']
files = []
for channel in channels:
if '..' in channel:
continue
dir_fp = os.path.join(downloads_dir, channel)
if not os.path.exists(dir_fp):
continue
if not os.path.isdir(dir_fp):
continue
for f in os.listdir(dir_fp):
if f[0] in '.#~':
continue
fname = os.path.join(channel, f)
fname_fp = os.path.join(dir_fp, f)
fname_size = os.path.getsize(fname_fp)
utils.set_md5_attr(fname_fp)
fname_md5 = xattr.getxattr(fname_fp, 'md5')
files.append({
'path': fname,
'size': fname_size,
'md5': fname_md5
})
return files
def export_crl(pem=True):
# To migrate older installations run following:
# for j in /var/lib/certidude/*/revoked/*.pem; do echo $(attr -s 'revocation.reason' -V key_compromise $j); done
builder = CertificateListBuilder(config.AUTHORITY_CRL_URL, certificate,
generate_serial())
for filename in os.listdir(config.REVOKED_DIR):
if not filename.endswith(".pem"):
continue
serial_number = filename[:-4]
# TODO: Assert serial against regex
revoked_path = os.path.join(config.REVOKED_DIR, filename)
try:
reason = getxattr(revoked_path, "user.revocation.reason").decode(
"ascii") # TODO: dedup
except IOError: # TODO: make sure it's not required
reason = "key_compromise"
# TODO: Skip expired certificates
s = os.stat(revoked_path)
builder.add_certificate(int(filename[:-4], 16),
datetime.utcfromtimestamp(s.st_ctime), reason)
certificate_list = builder.build(private_key)
if pem:
return pem_armor_crl(certificate_list)
return certificate_list.dump()
def _checkDeprecated(self, item, symlink=False):
"""check deprecated list, set, get operations against an item"""
self.assertEqual(self._ignore(xattr.listxattr(item, symlink)),
[])
self.assertRaises(EnvironmentError, xattr.setxattr, item,
self.USER_ATTR, self.USER_VAL,
XATTR_REPLACE)
try:
xattr.setxattr(item, self.USER_ATTR, self.USER_VAL, 0, symlink)
except IOError:
err = sys.exc_info()[1]
if err.errno == errno.EPERM and symlink:
# symlinks may fail, in which case we abort the rest
# of the test for this case
return
raise
self.assertRaises(EnvironmentError, xattr.setxattr, item,
self.USER_ATTR, self.USER_VAL, XATTR_CREATE)
self.assertEqual(self._ignore(xattr.listxattr(item, symlink)),
[self.USER_ATTR])
self.assertEqual(xattr.getxattr(item, self.USER_ATTR, symlink),
self.USER_VAL)
self.assertEqual(self._ignore_tuples(xattr.get_all(item,
nofollow=symlink)),
[(self.USER_ATTR, self.USER_VAL)])
xattr.removexattr(item, self.USER_ATTR)
self.assertEqual(self._ignore(xattr.listxattr(item, symlink)),
[])
self.assertEqual(self._ignore_tuples(xattr.get_all(item,
nofollow=symlink)),
[])
self.assertRaises(EnvironmentError, xattr.removexattr,
item, self.USER_ATTR)
def is_partition_supported(folder: Path) -> bool:
if folder is None:
return False
result = False
to_delete = not folder.exists()
try:
if to_delete:
folder.mkdir()
if not os.access(folder, os.W_OK):
folder.chmod(stat.S_IXUSR
| stat.S_IRGRP
| stat.S_IXGRP
| stat.S_IRUSR
| stat.S_IWGRP
| stat.S_IWUSR)
key, value = "drive-test", b"drive-test"
xattr.setxattr(folder, key, value)
if xattr.getxattr(folder, key) == value:
result = True
xattr.removexattr(folder, key)
finally:
if to_delete:
with suppress(OSError):
folder.rmdir()
return result
def get_xattr(self, filepath, key):
logger.info("get_xattr - %s, %s" % (filepath, key))
with self._get_lock():
ascii_path = filepath.encode('ascii', 'ignore')
localfs_path = self._make_localfs_path(ascii_path)
return xattr.getxattr(localfs_path, key)
def GetExtAttrs(filepath):
"""Fetches extended file attributes.
Args:
filepath: A path to the file.
Yields:
`ExtAttr` pairs.
"""
path = CanonicalPathToLocalPath(filepath)
try:
attr_names = xattr.listxattr(path)
except (IOError, OSError) as error:
msg = "Failed to retrieve extended attributes for '%s': %s"
logging.error(msg, path, error)
return
for attr_name in attr_names:
try:
attr_value = xattr.getxattr(path, attr_name)
except (IOError, OSError) as error:
msg = "Failed to retrieve attribute '%s' for '%s': %s"
logging.error(msg, attr_name, path, error)
continue
yield rdf_client.ExtAttr(name=attr_name, value=attr_value)
def get_info(self):
name = os.path.basename(self.datadir)
st = os.stat(self.datadir)
if self._type == 2:
cont_cnt_str = xattr.getxattr(self.datadir, CONTCNT_KEY)
try:
container_count = int(cont_cnt_str)
except ValueError:
cont_cnt_str = "0"
# XXX object_count, bytes_used
return {'account': name,
'created_at': normalize_timestamp(st.st_ctime),
'put_timestamp': normalize_timestamp(st.st_mtime),
'delete_timestamp': '0',
'container_count': cont_cnt_str,
'object_count': '0',
'bytes_used': '0',
'hash': '-',
'id': ''}
else:
# XXX container info has something different in it, what is it?
return {'container': name,
'created_at': normalize_timestamp(st.st_ctime),
'put_timestamp': normalize_timestamp(st.st_mtime),
'delete_timestamp': '0',
'object_count': '0',
'bytes_used': '0',
'hash': '-',
'id': ''}
def enum_hard_links_using_gfid2path(brick, gfid, args):
hardlinks = []
p = os.path.join(brick, ".glusterfs", gfid[0:2], gfid[2:4], gfid)
if not os.path.isdir(p):
# we have a symlink or a normal file
try:
file_xattrs = xattr.list(p)
for x in file_xattrs:
x_str = bytearray_to_str(x)
if x_str.startswith("trusted.gfid2path."):
# get the value for the xattr i.e. <PGFID>/<BN>
v = xattr.getxattr(p, x_str)
v_str = bytearray_to_str(v)
pgfid, bn = v_str.split(os.sep)
try:
path = symlink_gfid_to_path(brick, pgfid)
fullpath = os.path.join(path, bn)
fullpath = output_path_prepare(fullpath, args)
hardlinks.append(fullpath)
except (IOError, OSError) as e:
logger.warn("Error converting to path: %s" % e)
continue
except (IOError, OSError):
pass
return hardlinks
def dolistacl(args):
path = os.path.abspath(args.path[0])
try:
attrobj = xattr.getxattr(path, "afs.acl")
except FileNotFoundError:
print_err("No such file or directory: %s" % path)
except OSError as e:
if e.errno == 95:
print_err(
"Invalid argument; it is possible that %s is not in AFS" %
path)
else:
print_err("cannot open directory %s: %s" % (path, e.strerror))
attrs = attrobj.decode(encoding='UTF-8').split('\n')
print("Access list for %s" % path)
positiveacls = int(attrs[0])
negativeacls = int(attrs[1])
if (positiveacls > 0):
print("Normal rights:")
for attr in attrs[2:positiveacls + 2]:
if not '\t' in attr:
continue
(name, acl) = attr.split('\t')
print(" %s %s" % (name, acllookup(acl)))
if (negativeacls > 0):
print("Negative rights:")
for attr in attrs[2 + positiveacls:2 + positiveacls + negativeacls]:
if not '\t' in attr:
continue
(name, acl) = attr.split('\t')
print(" %s %s" % (name, acllookup(acl)))
def diff(filename, mf_path=''):
'''Use hashlib library to compare sha256 hash of
current file to the sha256 hash stored on MediaFire'''
full_expansion = get_path_expansion(filename)
if (mf_path == '' or not mf_path):
if ('/' in filename):
mf_path = os.path.basename(filename)
else:
mf_path = filename
if (os.path.isfile(full_expansion)):
in_file = open(filename, 'r')
mf_path = sanitize_path(mf_path)
file_contents = in_file.read().encode('utf-8')
new_hash = hashlib.sha256(file_contents).hexdigest()
media_hash = get_hash(mf_path)
try:
old_hash = xattr.getxattr(full_expansion, 'hash').decode('ascii')
except OSError:
old_hash = ''
if (media_hash == ''):
logger.die('No file path "' + mf_path + '/' +
os.path.basename(filename) + '" in MediaFire')
else:
figure_time_scale(media_hash, old_hash, new_hash,
os.path.basename(full_expansion))
else:
logger.die('No local file "' + os.path.basename(full_expansion) +
'" found')
def copy_missing(target_dir, target_relpath, logfile, dumped):
with io.open(logfile, 'wb') as log:
for hash, file in scan_files(USERDIR_LINUX):
filepath = USERDIR_WINDOWS + '\\' + file.replace('/', '\\')
absfile = os.path.join(USERDIR_LINUX, file)
if hash not in dumped:
# 55 4-byte UTF-8 characters + hash is guaranteed to always fit
# the Linux 255 character limit, and prevents impractically
# long filenames. some samples do generate massive filenames,
# and so does using Wikipedia articles as victim files...
filename = hash + '_' + file.decode('utf-8').split('/')[-1][-55:]
copy(os.path.join(USERDIR_LINUX, file),
os.path.join(target_dir, filename))
dumped[hash] = os.path.join(target_relpath, filename)
times = getxattr(absfile, 'system.ntfs_times_be')
# for the format of the raw NTFS times, see
# https://www.tuxera.com/community/ntfs-3g-advanced/extended-attributes/
info = { 'path': dumped[hash], 'md5': hash, 'filepath': filepath,
'time_create': ntfstime(times, 0),
'time_write': ntfstime(times, 8),
'time_access': ntfstime(times, 16),
'time_attrib_change': ntfstime(times, 24) }
json.dump(info, log)
log.write('\n')
def wrapped(self, req, resp, cn, *args, **kwargs):
from ipaddress import ip_address
from certidude import authority
from xattr import getxattr
try:
path, buf, cert, signed, expires = authority.get_signed(cn)
except IOError:
raise falcon.HTTPNotFound()
else:
# First attempt to authenticate client with certificate
buf = req.get_header("X-SSL-CERT")
if buf:
header, _, der_bytes = pem.unarmor(
buf.replace("\t", "").encode("ascii"))
origin_cert = x509.Certificate.load(der_bytes)
if origin_cert.native == cert.native:
logger.debug("Subject authenticated using certificates")
return func(self, req, resp, cn, *args, **kwargs)
# For backwards compatibility check source IP address
# TODO: make it disableable
try:
inner_address = getxattr(
path, "user.lease.inner_address").decode("ascii")
except IOError:
raise falcon.HTTPForbidden(
"Forbidden", "Remote address %s not whitelisted" %
req.context.get("remote_addr"))
else:
if req.context.get("remote_addr") != ip_address(inner_address):
raise falcon.HTTPForbidden(
"Forbidden", "Remote address %s mismatch" %
req.context.get("remote_addr"))
else:
return func(self, req, resp, cn, *args, **kwargs)
def diff(filename, mf_path=''):
'''Use hashlib library to compare sha256 hash of
current file to the sha256 hash stored on MediaFire'''
full_expansion = get_path_expansion(filename)
if(mf_path == '' or not mf_path):
if ('/' in filename):
mf_path = os.path.basename(filename)
else:
mf_path = filename
if (os.path.isfile(full_expansion)):
in_file = open(filename, 'r')
mf_path = sanitize_path(mf_path)
file_contents = in_file.read().encode('utf-8')
new_hash = hashlib.sha256(file_contents).hexdigest()
media_hash = get_hash(mf_path)
try:
old_hash = xattr.getxattr(full_expansion, 'hash').decode('ascii')
except OSError:
old_hash = ''
if (media_hash == ''):
logger.die('No file path "' + mf_path + '/' + os.path.basename(filename) + '" in MediaFire')
else:
figure_time_scale(media_hash, old_hash, new_hash, os.path.basename(full_expansion))
else:
logger.die('No local file "' + os.path.basename(full_expansion) + '" found')
def GetExtAttrs(filepath):
"""Fetches extended file attributes.
Args:
filepath: A path to the file.
Yields:
`ExtAttr` pairs.
"""
path = CanonicalPathToLocalPath(filepath)
try:
attr_names = xattr.listxattr(path)
except (IOError, OSError) as error:
msg = "Failed to retrieve extended attributes for '%s': %s"
logging.error(msg, path, error)
return
for attr_name in attr_names:
try:
attr_value = xattr.getxattr(path, attr_name)
except (IOError, OSError) as error:
msg = "Failed to retrieve attribute '%s' for '%s': %s"
logging.error(msg, attr_name, path, error)
continue
yield rdf_client.ExtAttr(name=attr_name, value=attr_value)
def read(path):
"""return string with Finder comment"""
try:
comment = xattr.getxattr(path, kMDItemFinderComment)
return comment.decode("utf-8")
except OSError:
pass
def write_acl(path, acl_type, acl_tuple):
ver, entries = acl_tuple
buf = StringIO()
#write version
buf.write(struct.pack("<I", ver))
#write entries
for e in entries:
buf.write(acl_entry_to_xattr(e))
final = buf.getvalue()
buf.close()
try:
#replace
xattr.getxattr(path, acl_type)
xattr.setxattr(path, acl_type, final, xattr.XATTR_REPLACE)
except:
#create
xattr.setxattr(path, acl_type, final, xattr.XATTR_CREATE)
def unlink(self, path):
try:
if xattr.getxattr(self.db_root + path, 'user.nzbfs.type') == 'nzb':
with self._attr_lock:
self.total_files -= 1
self.total_size -= get_nzf_attr(self.db_root + path, 'size')
finally:
return os.unlink(self.db_root + path)
def copy_xattr(self, src, dest):
attrs = xattr.listxattr(src)
for k in attrs:
try:
val = xattr.getxattr(src, k)
xattr.setxattr(dest, k, val)
except IOError as e:
log.warn(e, extra=self.d)
def read_acl(path, acl_type):
acl = xattr.getxattr(path, acl_type)
ver = struct.unpack("<I", acl[:4])[0]
body = acl[4:]
entries = []
for i in range(len(body) / 8):
entries.append(xattr_to_acl_entry(body[i*8:(i+1)*8]))
return ver, entries
def getxattr (fullpath, attr, default=None):
try:
return xattr.getxattr (fullpath, attr)
except IOError, e:
# Attribute not found
if e.errno == 93:
return default
raise
def comment_get(path):
assert_exists(path)
try:
comment = xattr.getxattr(path, kMDItemFinderComment)
# str/bytes by default
return comment.decode("utf-8") # convert to unicode
except:
pass
def finder_tags(fname):
"""
OS X Finder tags are stored in an extended attribute named
com.apple.metadata:_kMDItemUserTags. Its value is a binary property list
that contains a single array of strings.
"""
pl = getxattr(fname, 'com.apple.metadata:_kMDItemUserTags')
return readPlistFromString(pl)
def get_top_level_info(dirpath):
"""
Temporary function to just get top level info on user and project directories
:param dirpath: path to stash installation
:return: a list with dictionaries containing user/project information
"""
directories = []
if not os.path.isdir(dirpath):
return directories
for entry in os.listdir(os.path.join(dirpath, 'user')):
dir_info = {}
full_path = os.path.join(dirpath, 'user', entry)
if not os.path.isdir(full_path):
continue
dir_info['name'] = full_path
num_files = xattr.getxattr(full_path, 'ceph.dir.rfiles').strip()
# xattr occasionally returns an empty entry
if num_files[:-1] == '':
dir_info['files'] = 0
dir_info['size'] = 0
directories.append(dir_info)
continue
dir_info['files'] = num_files[:-1]
dir_info['size'] = int(xattr.getxattr(full_path,
'ceph.dir.rbytes')[:-1])
directories.append(dir_info)
for entry in os.listdir(os.path.join(dirpath, 'project/')):
full_path = os.path.join(dirpath, 'project', entry)
if not os.path.isdir(full_path):
continue
dir_info['name'] = full_path
num_files = xattr.getxattr(full_path, 'ceph.dir.rfiles').strip()
# xattr occasionally returns an empty entry
if num_files[:-1] == '':
dir_info['files'] = 0
dir_info['size'] = 0
directories.append(dir_info)
continue
dir_info['files'] = num_files[:-1]
dir_info['size'] = int(xattr.getxattr(full_path,
'ceph.dir.rbytes')[:-1])
directories.append(dir_info)
return directories
def get(self, name):
try:
return xattr.getxattr(self._path, name)
except IOError as e:
# TODO: See if this is different on other platforms, such as OS X
if e.errno == errno.ENODATA or e.errno == errno.EOPNOTSUPP or e.errno == errno.ENOENT:
raise KeyError(name)
else:
raise
