def test_purge_lock(self):
resp = self.test_file.write(data='test',
hdrs={'X-Hpss-Purgelock-Status': 'true',
'X-Hpss-Class-Of-Service-Id': '1'},
return_resp=True)
print resp.status
print resp.getheaders()
print resp.read()
test_file_name = os.path.join(self.hpss_dir,
self.account.name,
self.container.name,
'testfile')
print test_file_name
print os.stat(test_file_name)
print xattr.listxattr(test_file_name)
self.assertEqual(xattr.get(test_file_name,
'system.hpss.purgelock'),
'1')
self.test_file.post(hdrs={'X-Hpss-Purgelock-Status': 'false'})
self.assertEqual(xattr.get(test_file_name,
'system.hpss.purgelock'), '0')
def _checkDeprecated(self, item, symlink=False):
"""check deprecated list, set, get operations against an item"""
self.assertEqual(self._ignore(xattr.listxattr(item, symlink)),
[])
self.assertRaises(EnvironmentError, xattr.setxattr, item,
self.USER_ATTR, self.USER_VAL,
XATTR_REPLACE)
try:
xattr.setxattr(item, self.USER_ATTR, self.USER_VAL, 0, symlink)
except IOError:
err = sys.exc_info()[1]
if err.errno == errno.EPERM and symlink:
# symlinks may fail, in which case we abort the rest
# of the test for this case
return
raise
self.assertRaises(EnvironmentError, xattr.setxattr, item,
self.USER_ATTR, self.USER_VAL, XATTR_CREATE)
self.assertEqual(self._ignore(xattr.listxattr(item, symlink)),
[self.USER_ATTR])
self.assertEqual(xattr.getxattr(item, self.USER_ATTR, symlink),
self.USER_VAL)
self.assertEqual(self._ignore_tuples(xattr.get_all(item,
nofollow=symlink)),
[(self.USER_ATTR, self.USER_VAL)])
xattr.removexattr(item, self.USER_ATTR)
self.assertEqual(self._ignore(xattr.listxattr(item, symlink)),
[])
self.assertEqual(self._ignore_tuples(xattr.get_all(item,
nofollow=symlink)),
[])
self.assertRaises(EnvironmentError, xattr.removexattr,
item, self.USER_ATTR)
def testNoXattrDeprecated(self):
"""test no attributes (deprecated functions)"""
fh, fname = self._getfile()
self.assertEqual(xattr.listxattr(fname), [])
self.assertEqual(xattr.get_all(fname), [])
dname = self._getdir()
self.assertEqual(xattr.listxattr(dname), [])
self.assertEqual(xattr.get_all(dname), [])
_, sname = self._getsymlink()
self.assertEqual(xattr.listxattr(sname, True), [])
self.assertEqual(xattr.get_all(sname, nofollow=True), [])
def testMixedAccessDeprecated(self):
"""test mixed access to file (deprecated functions)"""
fh, fname = self._getfile()
fo = os.fdopen(fh)
self.assertEqual(xattr.listxattr(fname), [])
xattr.setxattr(fname, self.USER_ATTR, self.USER_VAL)
self.assertEqual(xattr.listxattr(fh), [self.USER_ATTR])
self.assertEqual(xattr.getxattr(fo, self.USER_ATTR), self.USER_VAL)
self.assertEqual(xattr.get_all(fo), [(self.USER_ATTR, self.USER_VAL)])
self.assertEqual(xattr.get_all(fname), [(self.USER_ATTR, self.USER_VAL)])
fo.close()
def testSymlinkOpsDeprecated(self):
"""test symlink operations (deprecated functions)"""
_, sname = self._getsymlink()
self.assertRaises(EnvironmentError, xattr.listxattr, sname)
self._checkDeprecated(sname, symlink=True)
target, sname = self._getsymlink(dangling=False)
xattr.setxattr(target, self.USER_ATTR, self.USER_VAL)
self.assertEqual(xattr.listxattr(target), [self.USER_ATTR])
self.assertEqual(xattr.listxattr(sname, True), [])
self.assertRaises(EnvironmentError, xattr.removexattr, sname, self.USER_ATTR, True)
xattr.removexattr(sname, self.USER_ATTR, False)
def get_raw_tags(path):
if 'com.apple.metadata:_kMDItemUserTags' in xattr.listxattr(path):
d = xattr.getxattr(path, 'com.apple.metadata:_kMDItemUserTags')
d = biplist.readPlistFromString(d)
return d
else:
return []
def list_xattr(self, filepath):
logger.info("list_xattr - %s" % filepath)
with self._get_lock():
ascii_path = filepath.encode('ascii', 'ignore')
localfs_path = self._make_localfs_path(ascii_path)
return xattr.listxattr(localfs_path)
def __applyxattr(self, fullPath, info, reportProblems= True):
if kXattrAvailable and info[1].has_key('xattr'):
try:
existingAttributes= xattr.listxattr(fullPath)
except KeyboardInterrupt,e:
raise e
except: # Can't list existing attributes on the file
def GetExtAttrs(filepath):
"""Fetches extended file attributes.
Args:
filepath: A path to the file.
Yields:
`ExtAttr` pairs.
"""
path = CanonicalPathToLocalPath(filepath)
try:
attr_names = xattr.listxattr(path)
except (IOError, OSError) as error:
msg = "Failed to retrieve extended attributes for '%s': %s"
logging.error(msg, path, error)
return
for attr_name in attr_names:
try:
attr_value = xattr.getxattr(path, attr_name)
except (IOError, OSError) as error:
msg = "Failed to retrieve attribute '%s' for '%s': %s"
logging.error(msg, attr_name, path, error)
continue
yield rdf_client.ExtAttr(name=attr_name, value=attr_value)
def main():
banner()
p = raw_input("Please provide the BAD back-end file system folder NOT the mountpoint: ")
while not os.path.isdir(p):
p = raw_input("Invalid input %s is not a directory, please enter the correct path: "%p)
a = raw_input("Last chance to back out, are you sure you want me to walk %s and remove all gluster xattrs I find? (y/n): "%p)
while a.lower() not in ('y','n'):
a = raw_input("Invalid input %s, please specify y or n: "%a)
if a.lower() == 'n':
sys.exit(0)
'''
Now we have a valid path, and user concent
'''
for root, dirs, files in os.walk(p):
'''
At the time of writing gluster only sets xarrts on directories
'''
xattrs = xattr.listxattr(root)
if len(xattrs) > 0:
if 'trusted.gfid' in xattrs:
print("Found trusted.gfid set on %s"%root)
xattr.removexattr(root,'trusted.gfid')
for attr in xattrs:
if reAFR.search(attr):
print("Found truster.afr.* set on %s"%root)
xattr.removexattr(root,attr)
def copy_xattr(self, src, dest):
attrs = xattr.listxattr(src)
for k in attrs:
try:
val = xattr.getxattr(src, k)
xattr.setxattr(dest, k, val)
except IOError as e:
log.warn(e, extra=self.d)
def testNoXattrDeprecated(self):
"""test no attributes (deprecated functions)"""
fh, fname = self._getfile()
self.checkList(xattr.listxattr(fname), [])
self.checkTuples(xattr.get_all(fname), [])
self.assertRaises(EnvironmentError, xattr.getxattr, fname,
self.USER_ATTR)
dname = self._getdir()
self.checkList(xattr.listxattr(dname), [])
self.checkTuples(xattr.get_all(dname), [])
self.assertRaises(EnvironmentError, xattr.getxattr, dname,
self.USER_ATTR)
_, sname = self._getsymlink()
self.checkList(xattr.listxattr(sname, True), [])
self.checkTuples(xattr.get_all(sname, nofollow=True), [])
self.assertRaises(EnvironmentError, xattr.getxattr, fname,
self.USER_ATTR, True)
def _message_at_path(self, path, load_content=True):
try:
content = None
if load_content:
f = open(path, "rb")
content = f.read()
f.close()
mtime = os.path.getmtime(path)
directory, filename = os.path.split(path)
directory, subdir = os.path.split(directory)
msgid = None
info = None
parts = filename.split(":")
if len(parts) > 0:
msgid = parts[0]
if len(parts) > 1:
info = parts[1]
msg = Message(content=content, msgid=msgid, info=info, subdir=subdir, mtime=mtime)
if not msg.msg_md5 and self._use_xattrs and load_content:
try:
xattrs = xattr.listxattr(path)
# logging.debug(xattrs)
if XATTR_MD5SUM in xattrs:
msg.msg_md5 = xattr.getxattr(path, XATTR_MD5SUM)
# logging.debug("Read md5: %s", msg.msg_md5)
else:
c = msg.content_hash
if c:
# logging.debug("Setting shasum xattr: %r", c)
xattr.setxattr(path, XATTR_MD5SUM, c)
else:
logging.warning("Could not generate content hash of %s", msgid)
if XATTR_DATE in xattrs:
msg._date = xattr.getxattr(path, XATTR_DATE).decode("utf8")
msg._date = datetime.datetime.fromtimestamp(float(msg._date))
# logging.debug("Read date: %s", msg._date)
else:
d = str(msg.date.timestamp()).encode("utf8")
if d:
# logging.debug("Setting date xattr: %r", d)
xattr.setxattr(path, XATTR_DATE, d)
else:
logging.warning("Could not determine message date of %s", msgid)
except IOError:
# read-only FS, unsupported on FS, etc.
self._use_xattrs = False
log.debug("host filesystem for %s does not support xattrs; disabling" % self.name)
return msg
except OSError:
raise KeyError
def list(self):
try:
return list(xattr.listxattr(self._path))
except IOError as e:
if e.errno == errno.EOPNOTSUPP or e.errno == errno.ENOENT: # No
# xattr support or the file doesn't exist
return []
else:
raise
def get_fs_xattr(fn):
x = {}
for attr in xattr.listxattr(fn):
value = xattr.getxattr(fn, attr)
if value.startswith('bplist'):
x[attr] = pbPlist.PBPlist(value)
else:
x[attr] = value
return x
def set_eas(self, rp, write):
"""Set extended attributes from rp. Tests writing if write is true."""
assert Globals.local_connection is rp.conn
assert rp.lstat()
if Globals.eas_active == 0:
log.Log("Extended attributes test skipped. rdiff-backup run "
"with --no-eas option.", 4)
self.eas = 0
return
try: import xattr
except ImportError:
log.Log("Unable to import module xattr.\nExtended attributes not "
"supported on filesystem at %s" % (rp.path,), 4)
self.eas = 0
return
try:
ver = xattr.__version__
except AttributeError:
ver = 'unknown'
if ver < '0.2.2' or ver == 'unknown':
log.Log("Warning: Your version of pyxattr (%s) has broken support "
"for extended\nattributes on symlinks. If you choose not "
"to upgrade to a more recent version,\nyou may see many "
"warning messages from listattr().\n" % (ver,), 3)
try:
xattr.listxattr(rp.path)
if write:
xattr.setxattr(rp.path, "user.test", "test val")
assert xattr.getxattr(rp.path, "user.test") == "test val"
except IOError:
log.Log("Extended attributes not supported by "
"filesystem at %s" % (rp.path,), 4)
self.eas = 0
except AssertionError:
log.Log("Extended attributes support is broken on filesystem at "
"%s.\nPlease upgrade the filesystem driver, contact the "
"developers,\nor use the --no-eas option to disable "
"extended attributes\nsupport and suppress this message."
% (rp.path,), 1)
self.eas = 0
else: self.eas = 1
def listxattr (path):
global xattr_supported
if xattr_supported:
try:
return xattr.listxattr (path)
except IOError, e:
if e.errno == 95: # Operation not supported
xattr_supported = False
else:
raise
def localCopyOfImageInFolder(imageURLString, imagePageString, targetFolder):
username = None
if imageURLString != None:
fileName = NSString.stringWithString_(imageURLString).lastPathComponent()
filePath = targetFolder + "/" + fileName
if NSFileManager.defaultManager().fileExistsAtPath_(filePath) == False:
imageURL = NSURL.URLWithString_(NSString.stringWithString_(NSString.stringWithString_(imageURLString).stringByAddingPercentEscapesUsingEncoding_(NSUTF8StringEncoding)))
request = NSURLRequest.requestWithURL_(imageURL)
data, response, error = NSURLConnection.sendSynchronousRequest_returningResponse_error_(request, None, None)
if data != None:
if data.writeToFile_atomically_(filePath, True):
print " Image file cached successfully at " + filePath
else:
print " * Failed to write image file at " + filePath
else:
print " * Could not load " + imageURLString + " (" + error.description() + ")"
# get image creator's name
infoURL = NSURL.URLWithString_(NSString.stringWithString_(imagePageString).stringByAddingPercentEscapesUsingEncoding_(NSUTF8StringEncoding))
(infoPageString, error) = NSString.stringWithContentsOfURL_encoding_error_(infoURL, NSUTF8StringEncoding, None)
if infoPageString != None:
(infoXHTML, error) = NSXMLDocument.alloc().initWithXMLString_options_error_(infoPageString, NSXMLDocumentTidyXML, None)
if infoXHTML != None:
nodes, error = infoXHTML.nodesForXPath_error_("""//tr[contains(.,"current")]/td/a[contains(@class,'mw-userlink')]""", None)
if len(nodes) > 0:
wikipediaPath = nodes[0]
username = wikipediaPath.stringValue().split(":")[-1]
xattr.setxattr(filePath, "net.earthlingsoft.imageauthor", username)
print " Image author »" + username + "« stored."
else:
nodes, error = infoXHTML.nodesForXPath_error_("""//tr[contains(.,"aktuell")]/td/a[contains(@class,'mw-userlink')]""", None)
if len(nodes) > 0:
wikipediaPath = nodes[0]
username = wikipediaPath.stringValue().split(":")[-1]
xattr.setxattr(filePath, "net.earthlingsoft.imageauthor", username)
print " Image author »" + username + "« stored."
else:
print " * Could not find author information in info page at " + imagePageString
else:
print " * Could not parse XML of info page at " + imagePageString
else:
print " * Could not download image information at " + imagePageString
else:
xattrs = xattr.listxattr(filePath)
if "net.earthlingsoft.imageauthor" in xattrs:
username = xattr.getxattr(filePath, "net.earthlingsoft.imageauthor")
print " Cached image file available"
newURL = myURL + targetFolder + "/" + fileName
else:
newURL = None
return [newURL, username]
def testManyOpsDeprecated(self):
"""test many ops (deprecated functions)"""
fh, fname = self._getfile()
xattr.setxattr(fh, self.USER_ATTR, self.USER_VAL)
VL = [self.USER_ATTR]
for i in range(self.MANYOPS_COUNT):
self.assertEqual(xattr.listxattr(fh), VL)
for i in range(self.MANYOPS_COUNT):
self.assertEqual(xattr.getxattr(fh, self.USER_ATTR), self.USER_VAL)
for i in range(self.MANYOPS_COUNT):
self.assertEqual(xattr.get_all(fh), [(self.USER_ATTR, self.USER_VAL)])
def process_expire(self, expire):
with self.lock:
self.current_expire = expire
for z in xrange(self.maxz, 2 - 1, -1):
with self.lock:
self.current_expire_zoom = z
for (x, y) in expire.expiredAt(z):
t = Tile(z, x, y)
tile_path = getTilePath(REFERENCE_TILESET, z, x, y)
if path.isfile(tile_path):
if 'user.toposm_dirty' not in xattr.listxattr(tile_path):
xattr.setxattr(tile_path, 'user.toposm_dirty', 'yes')
mt = t.metatile
tile_path = getTilePath(REFERENCE_TILESET, mt.z, mt.x * NTILES[z], mt.y * NTILES[z])
if path.isfile(tile_path) and 'user.toposm_dirty' not in xattr.listxattr(tile_path):
xattr.setxattr(tile_path, 'user.toposm_dirty', 'yes')
self.queue.queue_tile(t, 'zoom', 'expire')
with self.lock:
self.current_expire = None
self.current_expire_zoom = None
def testBinaryPayloadDeprecated(self):
"""test binary values (deprecated functions)"""
fh, fname = self._getfile()
os.close(fh)
BINVAL = "abc" + "\0" + "def"
if PY3K:
BINVAL = BINVAL.encode()
xattr.setxattr(fname, self.USER_ATTR, BINVAL)
self.assertEqual(xattr.listxattr(fname), [self.USER_ATTR])
self.assertEqual(xattr.getxattr(fname, self.USER_ATTR), BINVAL)
self.assertEqual(xattr.get_all(fname), [(self.USER_ATTR, BINVAL)])
xattr.removexattr(fname, self.USER_ATTR)
def read_xattr_metadata(file_list):
"""For the given list of files read us the kMDItemDescription and kMDItemKeywords."""
meta_data = []
for file in file_list:
md = {}
attrs = xattr.listxattr(file)
if 'com.apple.metadata:kMDItemDescription' in attrs:
md['Caption-Abstract'] = biplist.readPlistFromString(xattr.getxattr(file, 'com.apple.metadata:kMDItemDescription'))
if 'com.apple.metadata:kMDItemKeywords' in attrs:
md['Keywords'] = biplist.readPlistFromString(xattr.getxattr(file, 'com.apple.metadata:kMDItemKeywords'))
meta_data.append(md)
return meta_data
def makeDescription(name, titel, jahre, adresse, info):
(htmlString, error) = NSString.stringWithContentsOfFile_encoding_error_("Bubble.html", NSUTF8StringEncoding, None)
htmlString = htmlString.stringByReplacingOccurrencesOfString_withString_("NAME", name)
htmlString = htmlString.stringByReplacingOccurrencesOfString_withString_("TITEL", titel)
htmlString = htmlString.stringByReplacingOccurrencesOfString_withString_("JAHRE", " und ".join(jahre))
htmlString = htmlString.stringByReplacingOccurrencesOfString_withString_("ADRESSE", adresse)
imageTag = ""
if info["imageURL"] != None and info["imagePageURL"] != None:
imagePath = "/".join(NSString.stringWithString_(info["imageURL"]).pathComponents()[-2:])
username = "******"
xattrs = xattr.listxattr(imagePath)
if "net.earthlingsoft.imageauthor" in xattrs:
username = xattr.getxattr(imagePath, "net.earthlingsoft.imageauthor")
imageTag = "<a class='portraitlink' style='display:block;text-decoration:none;color:#999;float:right;width:200px;text-align:right;' href='" + info["imagePageURL"] + "' title='Wikipedia Bildseite'><img class='portrait' style='margin-top:-1em;margin-left:0.5em;max-width:120px;' src='" + NSString.stringWithString_(info["imageURL"]).stringByAddingPercentEscapesUsingEncoding_(NSUTF8StringEncoding) + "' alt='" + name + u"'><br><span style=''>Bild: " + username + u" »</span></a>"
if info["tafelImageURL"] != None and info["tafelImagePageURL"] != None:
imagePath = "/".join(NSString.stringWithString_(info["tafelImageURL"]).pathComponents()[-2:])
username = "******"
xattrs = xattr.listxattr(imagePath)
if "net.earthlingsoft.imageauthor" in xattrs:
username = xattr.getxattr(imagePath, "net.earthlingsoft.imageauthor")
imageTag = imageTag + "\n<a class='tafellink' style='display:block;text-decoration:none;color:#999;float:right;width:200px;text-align:right;' href='" + info["tafelImagePageURL"] + "' title='Wikipedia Bildseite'><img class='portrait' style='margin-top:0;margin-left:0.5em;max-width:200px;' src='" + NSString.stringWithString_(info["tafelImageURL"]).stringByAddingPercentEscapesUsingEncoding_(NSUTF8StringEncoding) + u"' alt='Gedenktafel für " + name + u"'><br><span style=''>Bild: " + username + u" »</span></a>"
htmlString = htmlString.stringByReplacingOccurrencesOfString_withString_("IMAGETAG", imageTag)
wikipediaTag = ""
if info["wikipediaURL"] != None:
wikipediaTag = "<p><a href='" + info["wikipediaURL"] + u"'>Wikipedia Seite »</a></p>"
# wikipediaTag = "<iframe src='" + info["wikipediaURL"] + "'>"
htmlString = htmlString.stringByReplacingOccurrencesOfString_withString_("WIKIPEDIATAG", wikipediaTag)
return htmlString
def get_thumbnail_from_xattr(file, tsize=150):
"""Look for thumbnail in xattr or use ffmpeg to generate one (and store it in xattr).
ffmpeg command from http://stackoverflow.com/questions/14551102/with-ffmpeg-create-thumbnails-proportional-to-the-videos-ratio
e.g. ffmpeg -itsoffset -1 -i TestData/2013-06-29/MVI_0843.AVI -vframes 1 -filter:v scale="min(150\, iw):-1" out.jpg
Note that list form of Popen takes care of the quoting - nothing special needs to be done.
"""
if 'chhobi2:thumbnail' in xattr.listxattr(file):
return xattr.getxattr(file, 'chhobi2:thumbnail')
ftemp = 'chhobi2_thumb_temp.jpg'
Popen(['ffmpeg', '-loglevel', 'panic', '-itsoffset', '-1', '-i', file, '-vframes', '1', '-filter:v', 'scale=min({:d}\, iw):-1'.format(tsize), ftemp]).wait() #This takes a finite amount of time
thumb_data = open(ftemp, 'rb').read()
os.remove(ftemp) #Clean up after ourselves
xattr.setxattr(file, 'chhobi2:thumbnail', thumb_data)
return thumb_data
def rip(self, folderpath, id):
# create a resource fork with the document's first page on it
try:
bigfirstpageiconpath = os.path.join(folderpath, "page-images", "page00001.png")
if os.path.exists(bigfirstpageiconpath):
os.system('/usr/bin/sips -i "%s" > /dev/null' % bigfirstpageiconpath)
if 'com.apple.ResourceFork' in listxattr(bigfirstpageiconpath):
icon = getxattr(bigfirstpageiconpath, 'com.apple.ResourceFork')
resourceforkpath = os.path.join(folderpath, "thumbnails", "mac-icon.rsrc")
fp = open(resourceforkpath, 'wb')
fp.write(icon)
fp.close()
except Exception, x:
note("MacRipper: exception %s encountered.", repr(x))
def getkMDItemWhereFroms(file_path, default):
from subprocess import Popen, PIPE
if u'com.apple.metadata:kMDItemWhereFroms' in xattr.listxattr(file_path):
bplist_data = xattr.getxattr(file_path, 'com.apple.metadata:kMDItemWhereFroms')
args = ["/usr/bin/plutil", "-convert", "json", "-o", "-", "--", "-"]
p = Popen(args, stdin=PIPE, stdout=PIPE)
p.stdin.write(bplist_data)
out, err = p.communicate()
return str(json.loads(out)[0])
else:
return default
def testExtendedAttributesOnMac(self):
"""Verify that Chrome sets the extended attributes on a file.
This test is for mac only.
"""
if not self.IsMac():
logging.info('Skipping testExtendedAttributesOnMac on non-Mac')
return
downloaded_pkg = os.path.join(self.GetDownloadDirectory().value(),
'a_zip_file.zip')
self._ClearLocalDownloadState(downloaded_pkg)
file_url = 'http://src.chromium.org/viewvc/chrome/trunk/src/chrome/'\
'test/data/downloads/a_zip_file.zip'
self.DownloadAndWaitForStart(file_url)
self.WaitForAllDownloadsToComplete()
import xattr
self.assertTrue('com.apple.quarantine' in xattr.listxattr(downloaded_pkg))
def print_xattr(filename):
print filename, " :"
xattr_tuple = xattr.listxattr(filename)
for xattr_key in xattr_tuple:
xattr_value = xattr.getxattr(filename, xattr_key)
if "-d" not in sys.argv:
print "%40s : %s" % (xattr_key, repr(xattr_value))
else:
#xattr_value_128 = (16-len(xattr_value))*'\x00'+ xattr_value
#value_128 = unpack('>QQ', xattr_value_128)
#print "%40s : %s (%d%d)" % (xattr_key, repr(xattr_value), value_128[0], value_128[1])
xattr_len = len(xattr_value)
value_char = unpack('B'*xattr_len, xattr_value)
#print "%40s : %s " % (xattr_key, repr(xattr_value)),
print "%40s : " % xattr_key,
print "0x"+"%02x"*xattr_len % value_char
print "\n"
def write_xattr_metadata(file_list, meta_data):
if meta_data.has_key('caption'): #Simple, just replace all the captions
bipl = biplist.writePlistToString(meta_data['caption'])
for file in file_list:
xattr.setxattr(file, 'com.apple.metadata:kMDItemDescription', bipl)
if meta_data.has_key('keywords'): #A little more involved. For each file, load original keywords,
#then remove or add the relevant keywords
for file in file_list:
if 'com.apple.metadata:kMDItemKeywords' in xattr.listxattr(file):
orig_keywd_list = set(biplist.readPlistFromString(xattr.getxattr(file, 'com.apple.metadata:kMDItemKeywords')))
else:
orig_keywd_list = set([])
for keyword in meta_data['keywords']:
if keyword[0] == '+': orig_keywd_list.add(keyword[1])
if keyword[0] == '-': orig_keywd_list.remove(keyword[1])
xattr.setxattr(file, 'com.apple.metadata:kMDItemKeywords', biplist.writePlistToString(list(orig_keywd_list)))
def __addElement(self, relativePath, doHash, skipAttributes):
try:
fullPath= os.path.join(self.__path, relativePath)
stats= os.lstat(fullPath)
properties= {'path': relativePath}
if stat.S_ISLNK(stats.st_mode):
kind= "link"
properties['target']= os.readlink(fullPath)
elif stat.S_ISDIR(stats.st_mode):
kind= "directory"
elif stat.S_ISREG(stats.st_mode):
kind= "file"
properties['size']= str(stats.st_size)
else:
return None # unknown file type, skip it
properties['modified']= formatDate(stats.st_mtime)
mods= stat.S_IMODE(stats.st_mode)
if mods & (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) == 0:
properties['readonly']= "true"
if mods & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) != 0:
properties['executable']= "true"
bXML.appendText(self.__contents.documentElement, "\n\t")
element= bXML.appendElement(self.__contents.documentElement, kind, properties)
if kXattrAvailable:
try:
attrs= xattr.listxattr(fullPath)
for attr in attrs:
try:
value= xattr.getxattr(fullPath, attr, True)
bXML.appendText(element, "\n\t\t")
tag= bXML.appendElement(element, "xattr", {'name': attr})
bXML.appendText(tag, escape(value))
except: # can't read this attribute
exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
traceback.print_exception(exceptionType, exceptionValue, exceptionTraceback, limit=5, file=sys.stderr)
pass
except: # something went wrong
exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
traceback.print_exception(exceptionType, exceptionValue, exceptionTraceback, limit=5, file=sys.stderr)
pass
if element.firstChild:
bXML.appendText(element, "\n\t")
except: # skip files we can't look at
exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
traceback.print_exception(exceptionType, exceptionValue, exceptionTraceback, limit=5, file=sys.stderr)
pass
def main(ignore_dot_files=False, listen_gitignore=False):
""" Copies files with user.groups, user.individuals into folders, based on groups.json"""
parser = argparse.ArgumentParser()
parser.add_argument('-s', '--source', help='Source folder to traverse.')
parser.add_argument('-g', '--groups', help='Groups definitions json file.')
parser.add_argument('-b', '--base', help='Base directory of source wiki (to trim paths to).')
args = parser.parse_args()
CWD = pathlib.Path(os.getcwd())
SOURCE = pathlib.Path(args.source)
GROUPS = pathlib.Path(args.groups)
BASE = pathlib.Path(args.base or CWD)
for i, p in enumerate([SOURCE, GROUPS, BASE]):
if p.expanduser() == p and not str(p).startswith(p._flavour.sep):
p = pathlib.Path(os.path.join(CWD, p))
if i == 0: SOURCE = p
if i == 1: GROUPS = p
if i == 2: BASE = p
group_defs = json.load(open(GROUPS, 'r'))
indiv = {k: os.path.abspath(os.path.expanduser(os.path.expandvars(group_defs['individuals'][k])))
for k in group_defs['individuals']}
groups = group_defs['groups']
if os.path.isfile(SOURCE):
IS_FILE = True
else:
IS_FILE = False
for dir_name, subdir_list, file_list in os.walk(
not IS_FILE and SOURCE or SOURCE.parent.expanduser()):
if IS_FILE:
dir_name = str(SOURCE.parent.expanduser())
file_list = [SOURCE.name]
dir_path = dir_name.split('/')[1:]
if ignore_dot_files:
if any([name.startswith('.') for name in dir_path]):
continue
for fname in file_list:
path = os.path.join(dir_name, fname)
if ignore_dot_files:
if fname.startswith('.'):
continue
if listen_gitignore:
if os.path.exists('.gitignore'):
patterns = [line[:-1] for line in
open('.gitignore').readlines()
if not line.startswith('#') and line[:-1]]
any_matched = False
for pattern in patterns:
result = re.search(fnmatch.translate(pattern), path)
if result:
any_matched = True
if any_matched:
continue
with open(path, 'r') as f:
attrs = xattr.listxattr(f)
if 'user.to' in attrs:
value = xattr.getxattr(f=f, attr='user.to')
share_to = str(value, 'utf-8').split(',')
else:
share_to = []
# Do set math to figure out groups and set subtractions. #
inds = [it for it in share_to if it in indiv]
grps = list(set(share_to) - set(inds))
indivs_add = set([it for it in inds if not it.startswith('-')])
indivs_rem = set([it[1:] for it in inds if it.startswith('-')])
groups_add = set([it for it in grps if not it.startswith('-')])
groups_rem = set([it[1:] for it in grps if it.startswith('-')])
for grp in groups_add:
members = groups[grp]
indivs_add = indivs_add.union(members)
for grp in groups_rem:
members = groups[grp]
indivs_rem = indivs_rem.union(members)
friends = list(indivs_add - indivs_rem)
for who in friends:
SOURCE = pathlib.Path(path)
SPATH, SFILE = str(SOURCE.parent.expanduser()), SOURCE.name
if SPATH.startswith(str(BASE)): SPATH = SPATH[len(str(BASE)):]
if SPATH.startswith(BASE._flavour.sep): SPATH = SPATH[1:]
DPATH = os.path.join(indiv[who], SPATH)
DESTINATION = os.path.join(DPATH, SFILE)
print(f'COPY: @{who}', SOURCE, '->', DESTINATION)
if not os.path.exists(DPATH):
os.makedirs(DPATH)
shutil.copyfile(SOURCE, DESTINATION)
if IS_FILE:
break
def test_dir_attr(self):
''' set attrs, get attrs and remove attrs for a dir '''
os.mkdir(TESTDIR, 0755)
try:
xattr.setxattr(TESTDIR, DIR_ATTR, DIR_ATTR_VAL)
except IOError, e:
print e
os.rmdir(TESTDIR)
raise e
got_attr = xattr.getxattr(TESTDIR, DIR_ATTR)
self.assertEqual(got_attr, DIR_ATTR_VAL, 'Error: attributes'
'mismatch')
xattr.listxattr(TESTDIR)
xattr.removexattr(TESTDIR, DIR_ATTR)
os.rmdir(TESTDIR)
''' POSIX ACL tests - setfacl, getfacl '''
@unittest.skipUnless(has_posix1e, "requires posix1e module")
def test_file_posix_acl(self):
''' set POSIX acl, get remove ACL on a file '''
f = open(TESTFILE, 'w')
facl = posix1e.ACL(text=FACL_TO_SET)
facl.applyto(TESTFILE)
got_acl = posix1e.ACL(file=TESTFILE)
def has_xattrs(filepath):
return listxattr(filepath)
def _sign(csr, buf, overwrite=False):
# TODO: CRLDistributionPoints, OCSP URL, Certificate URL
assert buf.startswith("-----BEGIN CERTIFICATE REQUEST-----\n")
assert isinstance(csr, CertificationRequest)
csr_pubkey = asymmetric.load_public_key(
csr["certification_request_info"]["subject_pk_info"])
common_name = csr["certification_request_info"]["subject"].native[
"common_name"]
cert_path = os.path.join(config.SIGNED_DIR, "%s.pem" % common_name)
renew = False
attachments = [
(buf, "application/x-pem-file", common_name + ".csr"),
]
revoked_path = None
overwritten = False
# Move existing certificate if necessary
if os.path.exists(cert_path):
with open(cert_path) as fh:
prev_buf = fh.read()
header, _, der_bytes = pem.unarmor(prev_buf)
prev = x509.Certificate.load(der_bytes)
# TODO: assert validity here again?
renew = \
asymmetric.load_public_key(prev["tbs_certificate"]["subject_public_key_info"]) == \
csr_pubkey
# BUGBUG: is this enough?
if overwrite:
# TODO: is this the best approach?
prev_serial_hex = "%x" % prev.serial_number
revoked_path = os.path.join(config.REVOKED_DIR,
"%s.pem" % prev_serial_hex)
os.rename(cert_path, revoked_path)
attachments += [(prev_buf, "application/x-pem-file",
"deprecated.crt" if renew else "overwritten.crt")]
overwritten = True
else:
raise EnvironmentError("Will not overwrite existing certificate")
# Sign via signer process
builder = CertificateBuilder({u'common_name': common_name}, csr_pubkey)
builder.serial_number = random.randint(
0x1000000000000000000000000000000000000000,
0xffffffffffffffffffffffffffffffffffffffff)
now = datetime.utcnow()
builder.begin_date = now - timedelta(minutes=5)
builder.end_date = now + timedelta(
days=config.SERVER_CERTIFICATE_LIFETIME
if server_flags(common_name) else config.CLIENT_CERTIFICATE_LIFETIME)
builder.issuer = certificate
builder.ca = False
builder.key_usage = set([u"digital_signature", u"key_encipherment"])
# OpenVPN uses CN while StrongSwan uses SAN
if server_flags(common_name):
builder.subject_alt_domains = [common_name]
builder.extended_key_usage = set(
[u"server_auth", u"1.3.6.1.5.5.8.2.2", u"client_auth"])
else:
builder.extended_key_usage = set([u"client_auth"])
end_entity_cert = builder.build(private_key)
end_entity_cert_buf = asymmetric.dump_certificate(end_entity_cert)
with open(cert_path + ".part", "wb") as fh:
fh.write(end_entity_cert_buf)
os.rename(cert_path + ".part", cert_path)
attachments.append(
(end_entity_cert_buf, "application/x-pem-file", common_name + ".crt"))
cert_serial_hex = "%x" % end_entity_cert.serial_number
# Create symlink
link_name = os.path.join(config.SIGNED_BY_SERIAL_DIR,
"%x.pem" % end_entity_cert.serial_number)
assert not os.path.exists(
link_name
), "Certificate with same serial number already exists: %s" % link_name
os.symlink("../%s.pem" % common_name, link_name)
# Copy filesystem attributes to newly signed certificate
if revoked_path:
for key in listxattr(revoked_path):
if not key.startswith("user."):
continue
setxattr(cert_path, key, getxattr(revoked_path, key))
# Send mail
if renew: # Same keypair
mailer.send("certificate-renewed.md", **locals())
else: # New keypair
mailer.send("certificate-signed.md", **locals())
url = config.LONG_POLL_PUBLISH % hashlib.sha256(buf).hexdigest()
click.echo("Publishing certificate at %s ..." % url)
requests.post(url,
data=end_entity_cert_buf,
headers={
"User-Agent": "Certidude API",
"Content-Type": "application/x-x509-user-cert"
})
push.publish("request-signed", common_name)
return end_entity_cert, end_entity_cert_buf
def _sign(csr,
buf,
skip_notify=False,
skip_push=False,
overwrite=False,
profile="default",
signer=None):
# TODO: CRLDistributionPoints, OCSP URL, Certificate URL
if profile not in config.PROFILES:
raise ValueError("Invalid profile supplied '%s'" % profile)
assert buf.startswith(b"-----BEGIN ")
assert isinstance(csr, CertificationRequest)
csr_pubkey = asymmetric.load_public_key(
csr["certification_request_info"]["subject_pk_info"])
common_name = csr["certification_request_info"]["subject"].native[
"common_name"]
cert_path = os.path.join(config.SIGNED_DIR, "%s.pem" % common_name)
renew = False
attachments = [
(buf, "application/x-pem-file", common_name + ".csr"),
]
revoked_path = None
overwritten = False
# Move existing certificate if necessary
if os.path.exists(cert_path):
with open(cert_path, "rb") as fh:
prev_buf = fh.read()
header, _, der_bytes = pem.unarmor(prev_buf)
prev = x509.Certificate.load(der_bytes)
# TODO: assert validity here again?
renew = \
asymmetric.load_public_key(prev["tbs_certificate"]["subject_public_key_info"]) == \
csr_pubkey
# BUGBUG: is this enough?
if overwrite:
# TODO: is this the best approach?
prev_serial_hex = "%x" % prev.serial_number
revoked_path = os.path.join(config.REVOKED_DIR,
"%s.pem" % prev_serial_hex)
os.rename(cert_path, revoked_path)
attachments += [(prev_buf, "application/x-pem-file",
"deprecated.crt" if renew else "overwritten.crt")]
overwritten = True
else:
raise FileExistsError("Will not overwrite existing certificate")
# Sign via signer process
dn = {u'common_name': common_name}
profile_server_flags, lifetime, dn[
"organizational_unit_name"], _ = config.PROFILES[profile]
lifetime = int(lifetime)
builder = CertificateBuilder(dn, csr_pubkey)
builder.serial_number = random.randint(
0x1000000000000000000000000000000000000000,
0x7fffffffffffffffffffffffffffffffffffffff)
now = datetime.utcnow()
builder.begin_date = now - timedelta(minutes=5)
builder.end_date = now + timedelta(days=lifetime)
builder.issuer = certificate
builder.ca = False
builder.key_usage = set(["digital_signature", "key_encipherment"])
# If we have FQDN and profile suggests server flags, enable them
if server_flags(common_name) and profile_server_flags:
builder.subject_alt_domains = [
common_name
] # OpenVPN uses CN while StrongSwan uses SAN to match hostname of the server
builder.extended_key_usage = set(
["server_auth", "1.3.6.1.5.5.8.2.2", "client_auth"])
else:
builder.subject_alt_domains = [common_name
] # iOS demands SAN also for clients
builder.extended_key_usage = set(["client_auth"])
end_entity_cert = builder.build(private_key)
end_entity_cert_buf = asymmetric.dump_certificate(end_entity_cert)
with open(cert_path + ".part", "wb") as fh:
fh.write(end_entity_cert_buf)
os.rename(cert_path + ".part", cert_path)
attachments.append(
(end_entity_cert_buf, "application/x-pem-file", common_name + ".crt"))
cert_serial_hex = "%x" % end_entity_cert.serial_number
# Create symlink
link_name = os.path.join(config.SIGNED_BY_SERIAL_DIR,
"%x.pem" % end_entity_cert.serial_number)
assert not os.path.exists(
link_name
), "Certificate with same serial number already exists: %s" % link_name
os.symlink("../%s.pem" % common_name, link_name)
# Copy filesystem attributes to newly signed certificate
if revoked_path:
for key in listxattr(revoked_path):
if not key.startswith(b"user."):
continue
setxattr(cert_path, key, getxattr(revoked_path, key))
# Attach signer username
if signer:
setxattr(cert_path, "user.signature.username", signer)
if not skip_notify:
# Send mail
if renew: # Same keypair
mailer.send("certificate-renewed.md", **locals())
else: # New keypair
mailer.send("certificate-signed.md", **locals())
if not skip_push:
url = config.LONG_POLL_PUBLISH % hashlib.sha256(buf).hexdigest()
click.echo("Publishing certificate at %s ..." % url)
requests.post(url,
data=end_entity_cert_buf,
headers={
"User-Agent": "Certidude API",
"Content-Type": "application/x-x509-user-cert"
})
push.publish("request-signed", common_name)
return end_entity_cert, end_entity_cert_buf
def __addElement(self, relativePath, doHash, skipAttributes):
try:
fullPath = os.path.join(self.__path, relativePath)
stats = os.lstat(fullPath)
properties = {'path': relativePath}
if stat.S_ISLNK(stats.st_mode):
kind = "link"
properties['target'] = os.readlink(fullPath)
elif stat.S_ISDIR(stats.st_mode):
kind = "directory"
elif stat.S_ISREG(stats.st_mode):
kind = "file"
properties['size'] = str(stats.st_size)
else:
return None # unknown file type, skip it
properties['modified'] = formatDate(stats.st_mtime)
mods = stat.S_IMODE(stats.st_mode)
if mods & (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) == 0:
properties['readonly'] = "true"
if mods & (stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) != 0:
properties['executable'] = "true"
bXML.appendText(self.__contents.documentElement, "\n\t")
element = bXML.appendElement(self.__contents.documentElement, kind,
properties)
if kXattrAvailable:
try:
attrs = xattr.listxattr(fullPath)
for attr in attrs:
try:
value = xattr.getxattr(fullPath, attr, True)
bXML.appendText(element, "\n\t\t")
tag = bXML.appendElement(element, "xattr",
{'name': attr})
bXML.appendText(tag, escape(value))
except: # can't read this attribute
exceptionType, exceptionValue, exceptionTraceback = sys.exc_info(
)
traceback.print_exception(exceptionType,
exceptionValue,
exceptionTraceback,
limit=5,
file=sys.stderr)
pass
except: # something went wrong
exceptionType, exceptionValue, exceptionTraceback = sys.exc_info(
)
traceback.print_exception(exceptionType,
exceptionValue,
exceptionTraceback,
limit=5,
file=sys.stderr)
pass
if element.firstChild:
bXML.appendText(element, "\n\t")
except: # skip files we can't look at
exceptionType, exceptionValue, exceptionTraceback = sys.exc_info()
traceback.print_exception(exceptionType,
exceptionValue,
exceptionTraceback,
limit=5,
file=sys.stderr)
pass
def _sign(csr,
buf,
profile,
skip_notify=False,
skip_push=False,
overwrite=False,
signer=None):
# TODO: CRLDistributionPoints, OCSP URL, Certificate URL
assert buf.startswith(b"-----BEGIN ")
assert isinstance(csr, CertificationRequest)
csr_pubkey = asymmetric.load_public_key(
csr["certification_request_info"]["subject_pk_info"])
common_name = csr["certification_request_info"]["subject"].native[
"common_name"]
cert_path = os.path.join(config.SIGNED_DIR, "%s.pem" % common_name)
renew = False
attachments = [
(buf, "application/x-pem-file", common_name + ".csr"),
]
revoked_path = None
overwritten = False
# Move existing certificate if necessary
if os.path.exists(cert_path):
with open(cert_path, "rb") as fh:
prev_buf = fh.read()
header, _, der_bytes = pem.unarmor(prev_buf)
prev = x509.Certificate.load(der_bytes)
# TODO: assert validity here again?
renew = \
asymmetric.load_public_key(prev["tbs_certificate"]["subject_public_key_info"]) == \
csr_pubkey
# BUGBUG: is this enough?
if overwrite:
# TODO: is this the best approach?
prev_serial_hex = "%040x" % prev.serial_number
revoked_path = os.path.join(config.REVOKED_DIR,
"%s.pem" % prev_serial_hex)
os.rename(cert_path, revoked_path)
attachments += [(prev_buf, "application/x-pem-file",
"deprecated.crt" if renew else "overwritten.crt")]
overwritten = True
else:
raise FileExistsError("Will not overwrite existing certificate")
builder = CertificateBuilder(
cn_to_dn(common_name,
const.FQDN,
o=certificate["tbs_certificate"]["subject"].native.get(
"organization_name"),
ou=profile.ou), csr_pubkey)
builder.serial_number = generate_serial()
now = datetime.utcnow()
builder.begin_date = now - timedelta(minutes=5)
builder.end_date = now + timedelta(days=profile.lifetime)
builder.issuer = certificate
builder.ca = profile.ca
builder.key_usage = profile.key_usage
builder.extended_key_usage = profile.extended_key_usage
builder.subject_alt_domains = [common_name]
end_entity_cert = builder.build(private_key)
end_entity_cert_buf = asymmetric.dump_certificate(end_entity_cert)
with open(cert_path + ".part", "wb") as fh:
fh.write(end_entity_cert_buf)
os.rename(cert_path + ".part", cert_path)
attachments.append(
(end_entity_cert_buf, "application/x-pem-file", common_name + ".crt"))
cert_serial_hex = "%040x" % end_entity_cert.serial_number
# Create symlink
link_name = os.path.join(config.SIGNED_BY_SERIAL_DIR,
"%040x.pem" % end_entity_cert.serial_number)
assert not os.path.exists(
link_name
), "Certificate with same serial number already exists: %s" % link_name
os.symlink("../%s.pem" % common_name, link_name)
# Copy filesystem attributes to newly signed certificate
if revoked_path:
for key in listxattr(revoked_path):
if not key.startswith(b"user."):
continue
setxattr(cert_path, key, getxattr(revoked_path, key))
# Attach signer username
if signer:
setxattr(cert_path, "user.signature.username", signer)
if not skip_notify:
# Send mail
if renew: # Same keypair
mailer.send("certificate-renewed.md", **locals())
else: # New keypair
mailer.send("certificate-signed.md", **locals())
if not skip_push:
url = config.LONG_POLL_PUBLISH % hashlib.sha256(buf).hexdigest()
click.echo("Publishing certificate at %s ..." % url)
requests.post(url,
data=end_entity_cert_buf,
headers={
"User-Agent": "Certidude API",
"Content-Type": "application/x-x509-user-cert"
})
if renew:
# TODO: certificate-renewed event
push.publish("certificate-revoked", common_name)
push.publish("request-signed", common_name)
else:
push.publish("request-signed", common_name)
return end_entity_cert, end_entity_cert_buf
def serialize_certificates(g):
for common_name, path, buf, cert, signed, expires in g():
# Extract certificate tags from filesystem
try:
tags = []
for tag in getxattr(
path, "user.xdg.tags").decode("utf-8").split(","):
if "=" in tag:
k, v = tag.split("=", 1)
else:
k, v = "other", tag
tags.append(dict(id=tag, key=k, value=v))
except IOError: # No such attribute(s)
tags = None
attributes = {}
for key in listxattr(path):
if key.startswith(b"user.machine."):
attributes[key[13:].decode("ascii")] = getxattr(
path, key).decode("ascii")
# Extract lease information from filesystem
try:
last_seen = datetime.strptime(
getxattr(path, "user.lease.last_seen").decode("ascii"),
"%Y-%m-%dT%H:%M:%S.%fZ")
lease = dict(
inner_address=getxattr(
path, "user.lease.inner_address").decode("ascii"),
outer_address=getxattr(
path, "user.lease.outer_address").decode("ascii"),
last_seen=last_seen,
age=datetime.utcnow() - last_seen)
except IOError: # No such attribute(s)
lease = None
try:
signer_username = getxattr(
path, "user.signature.username").decode("ascii")
except IOError:
signer_username = None
# TODO: dedup
yield dict(
serial="%x" % cert.serial_number,
organizational_unit=cert.subject.native.get(
"organizational_unit_name"),
common_name=common_name,
# TODO: key type, key length, key exponent, key modulo
signed=signed,
expires=expires,
sha256sum=hashlib.sha256(buf).hexdigest(),
signer=signer_username,
lease=lease,
tags=tags,
attributes=attributes or None,
extensions=dict([
(e["extn_id"].native, e["extn_value"].native)
for e in cert["tbs_certificate"]["extensions"]
if e["extn_id"].native in ("extended_key_usage", )
]))
def listxattr(self, path):
return xattr.listxattr(self.db_root + path)
sys.exit(1)
args[0] = os.path.abspath(args[0])
if option.name:
print_header(args[0], option.absnames)
try:
getfattr(args[0], option)
except KeyError as err:
print(("Invalid key %s" % err))
sys.exit(1)
except IOError as err:
print(err)
sys.exit(1)
if option.pattern:
print_header(args[0], option.absnames)
try:
xattrs = xattr.listxattr(args[0])
for attr in xattrs:
if option.dump:
option.name = attr.encode('utf-8')
getfattr(args[0], option)
else:
option.name = attr.encode('utf-8')
print_getfattr(args[0], option, None)
except IOError as err:
print(err)
sys.exit(1)
def clear_cache(self):
for name in xattr.listxattr(self.name):
if name.startswith('user.pyanidb.'):
xattr.removexattr(self.name, name)
def getxattr(pathname, attr):
"""Get a named xattr from a file. Return None if not present"""
if attr in xattr.listxattr(pathname):
return xattr.getxattr(pathname, attr)
else:
return None
# ext3 needs to be mounted with option user_xattr
import xattr
xattr.listxattr("file.txt")
xattr.setxattr("file.txt", "user.comment", "Simple text file")
xattr.getxattr("file.txt", "user.comment")
def is_nzf(path):
if os.path.isfile(path):
return 'user.nzbfs.size' in xattr.listxattr(path)
else:
return False
def _listxattr(path, symlink=True):
if _python_has_xattr_lib:
return os.listxattr(path, follow_symlinks=symlink)
else:
return xattr.listxattr(path, symlink=symlink)
def getxattr(self, attr):
"""Get a named xattr from a file. Return None if not present."""
if attr in xattr.listxattr(self.env["pathname"]):
return xattr.getxattr(self.env["pathname"], attr).decode()
return None
