def normalize(ref, base=None):
"""Normalize a url string into a reference and fix windows shenanigans"""
if not isinstance(ref, Reference):
if ref.startswith('file:'):
# URLs always use /, so change windows path separators to forward
# slashes
try:
ref = unicode(ref)
except UnicodeDecodeError:
try:
ref = str(ref).decode(sys.getfilesystemencoding())
except UnicodeDecodeError:
ref = str(ref).decode('utf-8')
#dprint(repr(ref))
if os.path.sep == '\\':
ref = ref.replace(os.path.sep, '/')
ref = get_reference(ref)
# If the reference is absolute (i.e. contains a scheme), we return;
# otherwise we assume it's a file:// URI
#dprint(str(ref))
if ref.scheme:
return ref
# Default to the current working directory
if base is None:
try:
base = os.getcwd().decode(sys.getfilesystemencoding())
except UnicodeDecodeError:
base = os.getcwd().decode('utf-8')
# URLs always use /
if os.path.sep == '\\':
base = base.replace(os.path.sep, '/')
#dprint(base)
# Check windows drive letters and add extra slash for correct URL syntax
if len(base) > 1 and base[1] == ':':
base = "/%s:%s" % (base[0].lower(), base[2:])
baseref = get_reference(u'file://%s/' % base)
try:
path = unicode(ref.path)
except UnicodeDecodeError:
try:
path = str(ref.path).decode(sys.getfilesystemencoding())
except UnicodeDecodeError:
path = str(ref.path).decode('utf-8')
#dprint(repr(path))
# Add the query string and fragment if they exist
newref = baseref.resolve(path)
newref.query = ref.query
newref.fragment = ref.fragment
#dprint(newref)
return newref
def open_numpy_mmap(ref):
if not isinstance(ref, Reference):
ref = get_reference(ref)
fs = get_file_system(ref.scheme)
if hasattr(fs, 'open_numpy_mmap'):
return fs.open_numpy_mmap(ref)
raise IOError('%s not supported for mmap access' % str(ref))
def test_username(self):
"""username and password specified before host."""
ref = get_reference(
'http://*****:*****@example.org:8888/test.html')
self.assertEqual(ref.scheme, 'http')
self.assertEqual(ref.authority.userinfo, 'username:passwd')
self.assertEqual(ref.authority.host, 'example.org')
def get_dirname(ref):
if not isinstance(ref, Reference):
ref = get_reference(ref)
return Reference(ref.scheme,
pycopy.copy(ref.authority),
ref.path.resolve2('../'),
pycopy.copy(ref.query),
ref.fragment)
def get_file_reference(path):
"""Get a relative filename reference based on the path, ignoring # or ?
characters special meaning in URLs.
Because filesystem files aren't required to obey the URL character rules
about # or ?, force the passed in path into a URL L{Reference} object
without being interpreted as a query string or fragment.
"""
import urllib
return get_reference(urllib.quote(path))
def get_metadata(ref):
if not isinstance(ref, Reference):
ref = get_reference(ref)
fs = get_file_system(ref.scheme)
if hasattr(fs, 'get_metadata'):
return fs.get_metadata(ref)
return {
'mimetype': fs.get_mimetype(ref),
'description': '',
'mtime': fs.get_mtime(ref),
'size': fs.get_size(ref),
}
def open_write(ref):
"""Open a file for writing, and if it exists, truncate it"""
if not isinstance(ref, Reference):
ref = get_reference(ref)
if exists(ref):
if is_file(ref):
fh = open(ref, WRITE)
else:
raise OSError(u"%s exists and is a directory; can't save as file" % ref)
else:
fh = make_file(ref)
return fh
def reference_with_new_extension(ref, ext):
"""Returns copy of reference with new extension
If the URL has a extension, it is replaced by the new extension. If
it doesn't have an extension, the new extension is appended. The new
extension may be specified with or without the leading "." character.
"""
if not isinstance(ref, Reference):
ref = get_reference(ref)
path = pycopy.copy(ref.path)
last = path.pop()
if "." in last:
prefix, oldext = os.path.splitext(last)
else:
prefix = last
while ext.startswith("."):
ext = ext[1:]
last = "%s.%s" % (prefix, ext)
return Reference(ref.scheme,
pycopy.copy(ref.authority),
ref.path.resolve2('../%s' % last),
pycopy.copy(ref.query),
pycopy.copy(ref.fragment))
def test_http(self):
"""http references are generic."""
ref = get_reference('http://ikaaro.org')
self.assert_(isinstance(ref, Reference))
def test_mailto(self):
"""Test if mailto references are detected."""
ref = get_reference('mailto:[email protected]')
self.assert_(isinstance(ref, Mailto))
def get_filename(ref):
if not isinstance(ref, Reference):
ref = get_reference(ref)
"""Convenience method to return the filename component of the URL"""
return ref.path[-1]
def test_ftp(self):
"""references with unknow scheme are generic."""
ref = get_reference('http://ikaaro.org')
self.assert_(isinstance(ref, Reference))
def test_username(self):
"""username specified before host."""
ref = get_reference('http://[email protected]')
self.assertEqual(ref.scheme, 'http')
self.assertEqual(ref.authority.userinfo, 'username')
self.assertEqual(ref.authority.host, 'example.org')
def test_mailto(self):
"""Test if mailto references are detected."""
ref = get_reference('mailto:[email protected]')
self.assert_(isinstance(ref, Mailto))
def test_username(self):
"""username and password specified before host."""
ref = get_reference('http://*****:*****@example.org:8888/test.html')
self.assertEqual(ref.scheme, 'http')
self.assertEqual(ref.authority.userinfo, 'username:passwd')
self.assertEqual(ref.authority.host, 'example.org')
def test_username(self):
"""username specified before host."""
ref = get_reference('http://[email protected]')
self.assertEqual(ref.scheme, 'http')
self.assertEqual(ref.authority.userinfo, 'username')
self.assertEqual(ref.authority.host, 'example.org')
def test_no_scheme(self):
"""references with no scheme are generic."""
ref = get_reference('logo.png')
self.assert_(isinstance(ref, Reference))
def test_ftp(self):
"""references with unknow scheme are generic."""
ref = get_reference('http://ikaaro.org')
self.assert_(isinstance(ref, Reference))
def test_no_scheme(self):
"""references with no scheme are generic."""
ref = get_reference('logo.png')
self.assert_(isinstance(ref, Reference))
def decode(value):
return get_reference(value)
def decode(value):
return get_reference(value)
def test_http(self):
"""http references are generic."""
ref = get_reference('http://ikaaro.org')
self.assert_(isinstance(ref, Reference))