def _get_target(self):
if self._target is None:
asdffile = self._asdffile().read_external(self._uri)
parts = urlparse.urlparse(self._uri)
fragment = parts.fragment
self._target = resolve_fragment(asdffile.tree, fragment)
return self._target
def _get_target(self):
if self._target is None:
asdffile = self._asdffile().read_external(self._uri)
parts = urlparse.urlparse(self._uri)
fragment = parts.fragment
self._target = resolve_fragment(asdffile.tree, fragment)
return self._target
def get_source(self, block):
"""
Get a source identifier for a given block.
Parameters
----------
block : Block
Returns
-------
source_id : str
May be an integer for an internal block, or a URI for an
external block.
"""
for i, internal_block in enumerate(self.internal_blocks):
if block == internal_block:
if internal_block.array_storage == 'streamed':
return -1
return i
for i, external_block in enumerate(self.external_blocks):
if block == external_block:
if self._asdffile().uri is None:
raise ValueError(
"Can't write external blocks, since URI of main file is "
"unknown.")
parts = list(urlparse.urlparse(self._asdffile().uri))
path = parts[2]
filename = os.path.basename(path)
return self.get_external_filename(filename, i)
raise ValueError("block not found.")
def get_source(self, block):
"""
Get a source identifier for a given block.
Parameters
----------
block : Block
Returns
-------
source_id : str
May be an integer for an internal block, or a URI for an
external block.
"""
for i, internal_block in enumerate(self.internal_blocks):
if block == internal_block:
if internal_block.array_storage == 'streamed':
return -1
return i
for i, external_block in enumerate(self.external_blocks):
if block == external_block:
if self._asdffile().uri is None:
raise ValueError(
"Can't write external blocks, since URI of main file is "
"unknown.")
parts = list(urlparse.urlparse(self._asdffile().uri))
path = parts[2]
filename = os.path.basename(path)
return self.get_external_filename(filename, i)
raise ValueError("block not found.")
def _get_target(self, do_not_fill_defaults=False):
if self._target is None:
asdffile = self._asdffile().read_external(
self._uri, do_not_fill_defaults=do_not_fill_defaults)
parts = urlparse.urlparse(self._uri)
fragment = parts.fragment
self._target = resolve_fragment(asdffile.tree, fragment)
return self._target
def relative_uri(source, target):
"""
Make a relative URI from source to target.
"""
su = urlparse.urlparse(source)
tu = urlparse.urlparse(target)
extra = list(tu[3:])
relative = None
if tu[0] == '' and tu[1] == '':
if tu[2] == su[2]:
relative = ''
elif not tu[2].startswith('/'):
relative = tu[2]
elif su[0:2] != tu[0:2]:
return target
if relative is None:
relative = os.path.relpath(tu[2], os.path.dirname(su[2]))
if relative == '.':
relative = ''
relative = urlparse.urlunparse(["", "", relative] + extra)
return relative
def resolve_uri(base, uri):
"""
Resolve a URI against a base URI.
"""
if base is None:
if uri == '':
return ''
parsed = urlparse.urlparse(uri)
if parsed.path.startswith('/'):
return uri
raise ValueError(
"Can not resolve relative URLs since the base is unknown.")
return urlparse.urljoin(base, uri)
def relative_uri(source, target):
"""
Make a relative URI from source to target.
"""
su = urlparse.urlparse(source)
tu = urlparse.urlparse(target)
extra = list(tu[3:])
relative = None
if tu[0] == '' and tu[1] == '':
if tu[2] == su[2]:
relative = ''
elif not tu[2].startswith('/'):
relative = tu[2]
elif su[0:2] != tu[0:2]:
return target
if relative is None:
relative = os.path.relpath(tu[2], su[2])
if relative == '.':
relative = ''
relative = urlparse.urlunparse(["", "", relative] + extra)
return relative
def resolve_uri(base, uri):
"""
Resolve a URI against a base URI.
"""
if base is None:
if uri == '':
return ''
parsed = urlparse.urlparse(uri)
if parsed.path.startswith('/'):
return uri
raise ValueError(
"Can not resolve relative URLs since the base is unknown.")
return urlparse.urljoin(base, uri)
def get_external_uri(self, uri, index):
"""
Given a main URI and an index number, return a new URI for
saving an external block.
"""
if uri is None:
uri = ''
parts = list(urlparse.urlparse(uri))
path = parts[2]
dirname, filename = os.path.split(path)
filename = self.get_external_filename(filename, index)
path = os.path.join(dirname, filename)
parts[2] = path
return urlparse.urlunparse(parts)
def get_external_uri(self, uri, index):
"""
Given a main URI and an index number, return a new URI for
saving an external block.
"""
if uri is None:
uri = ''
parts = list(urlparse.urlparse(uri))
path = parts[2]
dirname, filename = os.path.split(path)
filename = self.get_external_filename(filename, index)
path = os.path.join(dirname, filename)
parts[2] = path
return urlparse.urlunparse(parts)
def _make_http_connection(init, mode, uri=None):
"""
Creates a HTTPConnection instance if the HTTP server supports
Range requests, otherwise falls back to a generic InputStream.
"""
from astropy.extern.six.moves import http_client
parsed = urlparse.urlparse(init)
connection = http_client.HTTPConnection(parsed.netloc)
connection.connect()
# We request a range of everything ("0-") just to check if the
# server understands that header entry.
headers = {'Range': 'bytes=0-'}
connection.request('GET', parsed.path, headers=headers)
response = connection.getresponse()
if response.status // 100 != 2:
raise IOError("HTTP failed: {0} {1}".format(
response.status, response.reason))
# Status 206 means a range was returned. If it's anything else
# that indicates the server probably doesn't support Range
# headers.
if (response.status != 206 or
response.getheader('accept-ranges', None) != 'bytes' or
response.getheader('content-range', None) is None or
response.getheader('content-length', None) is None):
# Fall back to a regular input stream, but we don't
# need to open a new connection.
response.close = connection.close
return InputStream(response, mode, uri=uri or init, close=True)
# Since we'll be requesting chunks, we can't read at all with the
# current request (because we can't abort it), so just close and
# start over
size = int(response.getheader('content-length'))
connection.close()
connection.connect()
return HTTPConnection(connection, size, parsed.path, uri or init)
def _make_http_connection(init, mode, uri=None):
"""
Creates a HTTPConnection instance if the HTTP server supports
Range requests, otherwise falls back to a generic InputStream.
"""
from astropy.extern.six.moves import http_client
parsed = urlparse.urlparse(init)
connection = http_client.HTTPConnection(parsed.netloc)
connection.connect()
# We request a range of everything ("0-") just to check if the
# server understands that header entry.
headers = {'Range': 'bytes=0-'}
connection.request('GET', parsed.path, headers=headers)
response = connection.getresponse()
if response.status // 100 != 2:
raise IOError("HTTP failed: {0} {1}".format(
response.status, response.reason))
# Status 206 means a range was returned. If it's anything else
# that indicates the server probably doesn't support Range
# headers.
if (response.status != 206 or
response.getheader('accept-ranges', None) != 'bytes' or
response.getheader('content-range', None) is None or
response.getheader('content-length', None) is None):
# Fall back to a regular input stream, but we don't
# need to open a new connection.
response.close = connection.close
return InputStream(response, mode, uri=uri or init, close=True)
# Since we'll be requesting chunks, we can't read at all with the
# current request (because we can't abort it), so just close and
# start over
size = int(response.getheader('content-length'))
connection.close()
connection.connect()
return HTTPConnection(connection, size, parsed.path, uri or init)
def list_name_is_url(name):
may_be_url = urlparse.urlparse(name)
return (may_be_url.scheme and may_be_url.netloc)
def get_base_uri(uri):
"""
For a given URI, return the part without any fragment.
"""
parts = urlparse.urlparse(uri)
return urlparse.urlunparse(list(parts[:5]) + [''])
def list_name_is_url(name):
may_be_url = urlparse.urlparse(name)
return (may_be_url.scheme and may_be_url.netloc)
def get_file(init, mode='r', uri=None):
"""
Returns a `GenericFile` instance suitable for wrapping the given
object `init`.
If passed an already open file-like object, it must be opened for
reading/writing in binary mode. It is the caller's responsibility
to close it.
Parameters
----------
init : object
`init` may be:
- A `bytes` or `unicode` file path or ``file:`` or ``http:``
url.
- A Python 2 `file` object.
- An `io.IOBase` object (the default file object on Python 3).
- A ducktyped object that looks like a file object. If `mode`
is ``"r"``, it must have a ``read`` method. If `mode` is
``"w"``, it must have a ``write`` method. If `mode` is
``"rw"`` it must have the ``read``, ``write``, ``tell`` and
``seek`` methods.
- A `GenericFile` instance, in which case it is wrapped in a
`GenericWrapper` instance, so that the file is closed when
only when the final layer is unwrapped.
mode : str
Must be one of ``"r"``, ``"w"`` or ``"rw"``.
uri : str
Sets the base URI of the file object. This will be used to
resolve any relative URIs contained in the file. This is
redundant if `init` is a `bytes` or `unicode` object (since it
will be the uri), and it may be determined automatically if
`init` refers to a regular filesystem file. It is not required
if URI resolution is not used in the file.
Returns
-------
fd : GenericFile
Raises
------
ValueError, TypeError, IOError
"""
if mode not in ('r', 'w', 'rw'):
raise ValueError("mode must be 'r', 'w' or 'rw'")
# Special case for sys.stdout on Python 3, since it takes unicode
# by default, but we need to write to it with bytes
if six.PY3 and init in (sys.stdout, sys.stdin, sys.stderr):
init = init.buffer
if isinstance(init, (GenericFile, GenericWrapper)):
if mode not in init.mode:
raise ValueError(
"File is opened as '{0}', but '{1}' was requested".format(
init.mode, mode))
return GenericWrapper(init, uri=uri)
elif isinstance(init, six.string_types):
parsed = urlparse.urlparse(init)
if parsed.scheme == 'http':
if mode == 'w':
raise ValueError(
"HTTP connections can not be opened for writing")
return _make_http_connection(init, mode, uri=uri)
elif parsed.scheme in ('', 'file'):
if mode == 'rw':
realmode = 'r+b'
else:
realmode = mode + 'b'
return RealFile(
open(parsed.path, realmode), mode, close=True,
uri=uri or parsed.path)
elif isinstance(init, io.BytesIO):
return MemoryIO(init, mode, uri=uri)
elif isinstance(init, io.StringIO):
raise TypeError(
"io.StringIO objects are not supported. Use io.BytesIO instead.")
elif six.PY2 and isinstance(init, file):
if not mode in init.mode:
raise ValueError(
"File is opened as '{0}', but '{1}' was requested".format(
init.mode, mode))
try:
init.tell()
except IOError:
if mode == 'w':
return OutputStream(init, uri=uri)
elif mode == 'r':
return InputStream(init, mode, uri=uri)
else:
raise ValueError(
"File '{0}' could not be opened in 'rw' mode".format(init))
else:
return RealFile(init, mode, uri=uri)
elif isinstance(init, io.IOBase):
if sys.version_info[:2] == (2, 6):
raise ValueError("io.open file objects are not supported on Python 2.6")
if (('r' in mode and not init.readable()) or
('w' in mode and not init.writable())):
raise ValueError(
"File is opened as '{0}', but '{1}' was requested".format(
init.mode, mode))
if init.seekable():
if isinstance(init, (io.BufferedReader,
io.BufferedWriter,
io.BufferedRandom)):
init2 = init.raw
else:
init2 = init
if isinstance(init2, io.RawIOBase):
result = RealFile(init2, mode, uri=uri)
else:
result = MemoryIO(init2, mode, uri=uri)
result._secondary_fd = init
return result
else:
if mode == 'w':
return OutputStream(init, uri=uri)
elif mode == 'r':
return InputStream(init, mode, uri=uri)
else:
raise ValueError(
"File '{0}' could not be opened in 'rw' mode".format(init))
elif 'w' in mode and (
hasattr(init, 'write') and
hasattr(init, 'seek') and
hasattr(init, 'tell')):
return MemoryIO(init, mode, uri=uri)
elif 'r' in mode and (
hasattr(init, 'read') and
hasattr(init, 'seek') and
hasattr(init, 'tell')):
return MemoryIO(init, mode, uri=uri)
elif 'w' in mode and hasattr(init, 'write'):
return OutputStream(init, uri=uri)
elif 'r' in mode and hasattr(init, 'read'):
return InputStream(init, mode, uri=uri)
raise ValueError("Can't handle '{0}' as a file for mode '{1}'".format(
init, mode))
def get_base_uri(uri):
"""
For a given URI, return the part without any fragment.
"""
parts = urlparse.urlparse(uri)
return urlparse.urlunparse(list(parts[:5]) + [''])
def get_file(init, mode='r', uri=None):
"""
Returns a `GenericFile` instance suitable for wrapping the given
object `init`.
If passed an already open file-like object, it must be opened for
reading/writing in binary mode. It is the caller's responsibility
to close it.
Parameters
----------
init : object
`init` may be:
- A `bytes` or `unicode` file path or ``file:`` or ``http:``
url.
- A Python 2 `file` object.
- An `io.IOBase` object (the default file object on Python 3).
- A ducktyped object that looks like a file object. If `mode`
is ``"r"``, it must have a ``read`` method. If `mode` is
``"w"``, it must have a ``write`` method. If `mode` is
``"rw"`` it must have the ``read``, ``write``, ``tell`` and
``seek`` methods.
- A `GenericFile` instance, in which case it is wrapped in a
`GenericWrapper` instance, so that the file is closed when
only when the final layer is unwrapped.
mode : str
Must be one of ``"r"``, ``"w"`` or ``"rw"``.
uri : str
Sets the base URI of the file object. This will be used to
resolve any relative URIs contained in the file. This is
redundant if `init` is a `bytes` or `unicode` object (since it
will be the uri), and it may be determined automatically if
`init` refers to a regular filesystem file. It is not required
if URI resolution is not used in the file.
Returns
-------
fd : GenericFile
Raises
------
ValueError, TypeError, IOError
"""
if mode not in ('r', 'w', 'rw'):
raise ValueError("mode must be 'r', 'w' or 'rw'")
# Special case for sys.stdout on Python 3, since it takes unicode
# by default, but we need to write to it with bytes
if six.PY3 and init in (sys.stdout, sys.stdin, sys.stderr):
init = init.buffer
if isinstance(init, (GenericFile, GenericWrapper)):
if mode not in init.mode:
raise ValueError(
"File is opened as '{0}', but '{1}' was requested".format(
init.mode, mode))
return GenericWrapper(init, uri=uri)
elif isinstance(init, six.string_types):
parsed = urlparse.urlparse(init)
if parsed.scheme == 'http':
if mode == 'w':
raise ValueError(
"HTTP connections can not be opened for writing")
return _make_http_connection(init, mode, uri=uri)
elif parsed.scheme in _local_file_schemes:
if mode == 'rw':
realmode = 'r+b'
else:
realmode = mode + 'b'
realpath = url2pathname(parsed.path)
return RealFile(
open(realpath, realmode), mode, close=True,
uri=uri)
elif isinstance(init, io.BytesIO):
return MemoryIO(init, mode, uri=uri)
elif isinstance(init, io.StringIO):
raise TypeError(
"io.StringIO objects are not supported. Use io.BytesIO instead.")
elif six.PY2 and isinstance(init, file):
if not mode in init.mode:
raise ValueError(
"File is opened as '{0}', but '{1}' was requested".format(
init.mode, mode))
try:
init.tell()
except IOError:
if mode == 'w':
return OutputStream(init, uri=uri)
elif mode == 'r':
return InputStream(init, mode, uri=uri)
else:
raise ValueError(
"File '{0}' could not be opened in 'rw' mode".format(init))
else:
return RealFile(init, mode, uri=uri)
elif isinstance(init, io.IOBase):
if sys.version_info[:2] == (2, 6):
raise ValueError("io.open file objects are not supported on Python 2.6")
if (('r' in mode and not init.readable()) or
('w' in mode and not init.writable())):
raise ValueError(
"File is opened as '{0}', but '{1}' was requested".format(
init.mode, mode))
if init.seekable():
if isinstance(init, (io.BufferedReader,
io.BufferedWriter,
io.BufferedRandom)):
init2 = init.raw
else:
init2 = init
if isinstance(init2, io.RawIOBase):
result = RealFile(init2, mode, uri=uri)
else:
result = MemoryIO(init2, mode, uri=uri)
result._secondary_fd = init
return result
else:
if mode == 'w':
return OutputStream(init, uri=uri)
elif mode == 'r':
return InputStream(init, mode, uri=uri)
else:
raise ValueError(
"File '{0}' could not be opened in 'rw' mode".format(init))
elif 'w' in mode and (
hasattr(init, 'write') and
hasattr(init, 'seek') and
hasattr(init, 'tell')):
return MemoryIO(init, mode, uri=uri)
elif 'r' in mode and (
hasattr(init, 'read') and
hasattr(init, 'seek') and
hasattr(init, 'tell')):
return MemoryIO(init, mode, uri=uri)
elif 'w' in mode and hasattr(init, 'write'):
return OutputStream(init, uri=uri)
elif 'r' in mode and hasattr(init, 'read'):
return InputStream(init, mode, uri=uri)
raise ValueError("Can't handle '{0}' as a file for mode '{1}'".format(
init, mode))
