def _get_target(self): if self._target is None: asdffile = self._asdffile().read_external(self._uri) parts = urlparse.urlparse(self._uri) fragment = parts.fragment self._target = resolve_fragment(asdffile.tree, fragment) return self._target
def _get_target(self): if self._target is None: asdffile = self._asdffile().read_external(self._uri) parts = urlparse.urlparse(self._uri) fragment = parts.fragment self._target = resolve_fragment(asdffile.tree, fragment) return self._target
def get_source(self, block): """ Get a source identifier for a given block. Parameters ---------- block : Block Returns ------- source_id : str May be an integer for an internal block, or a URI for an external block. """ for i, internal_block in enumerate(self.internal_blocks): if block == internal_block: if internal_block.array_storage == 'streamed': return -1 return i for i, external_block in enumerate(self.external_blocks): if block == external_block: if self._asdffile().uri is None: raise ValueError( "Can't write external blocks, since URI of main file is " "unknown.") parts = list(urlparse.urlparse(self._asdffile().uri)) path = parts[2] filename = os.path.basename(path) return self.get_external_filename(filename, i) raise ValueError("block not found.")
def get_source(self, block): """ Get a source identifier for a given block. Parameters ---------- block : Block Returns ------- source_id : str May be an integer for an internal block, or a URI for an external block. """ for i, internal_block in enumerate(self.internal_blocks): if block == internal_block: if internal_block.array_storage == 'streamed': return -1 return i for i, external_block in enumerate(self.external_blocks): if block == external_block: if self._asdffile().uri is None: raise ValueError( "Can't write external blocks, since URI of main file is " "unknown.") parts = list(urlparse.urlparse(self._asdffile().uri)) path = parts[2] filename = os.path.basename(path) return self.get_external_filename(filename, i) raise ValueError("block not found.")
def _get_target(self, do_not_fill_defaults=False): if self._target is None: asdffile = self._asdffile().read_external( self._uri, do_not_fill_defaults=do_not_fill_defaults) parts = urlparse.urlparse(self._uri) fragment = parts.fragment self._target = resolve_fragment(asdffile.tree, fragment) return self._target
def relative_uri(source, target): """ Make a relative URI from source to target. """ su = urlparse.urlparse(source) tu = urlparse.urlparse(target) extra = list(tu[3:]) relative = None if tu[0] == '' and tu[1] == '': if tu[2] == su[2]: relative = '' elif not tu[2].startswith('/'): relative = tu[2] elif su[0:2] != tu[0:2]: return target if relative is None: relative = os.path.relpath(tu[2], os.path.dirname(su[2])) if relative == '.': relative = '' relative = urlparse.urlunparse(["", "", relative] + extra) return relative
def resolve_uri(base, uri): """ Resolve a URI against a base URI. """ if base is None: if uri == '': return '' parsed = urlparse.urlparse(uri) if parsed.path.startswith('/'): return uri raise ValueError( "Can not resolve relative URLs since the base is unknown.") return urlparse.urljoin(base, uri)
def relative_uri(source, target): """ Make a relative URI from source to target. """ su = urlparse.urlparse(source) tu = urlparse.urlparse(target) extra = list(tu[3:]) relative = None if tu[0] == '' and tu[1] == '': if tu[2] == su[2]: relative = '' elif not tu[2].startswith('/'): relative = tu[2] elif su[0:2] != tu[0:2]: return target if relative is None: relative = os.path.relpath(tu[2], su[2]) if relative == '.': relative = '' relative = urlparse.urlunparse(["", "", relative] + extra) return relative
def resolve_uri(base, uri): """ Resolve a URI against a base URI. """ if base is None: if uri == '': return '' parsed = urlparse.urlparse(uri) if parsed.path.startswith('/'): return uri raise ValueError( "Can not resolve relative URLs since the base is unknown.") return urlparse.urljoin(base, uri)
def get_external_uri(self, uri, index): """ Given a main URI and an index number, return a new URI for saving an external block. """ if uri is None: uri = '' parts = list(urlparse.urlparse(uri)) path = parts[2] dirname, filename = os.path.split(path) filename = self.get_external_filename(filename, index) path = os.path.join(dirname, filename) parts[2] = path return urlparse.urlunparse(parts)
def get_external_uri(self, uri, index): """ Given a main URI and an index number, return a new URI for saving an external block. """ if uri is None: uri = '' parts = list(urlparse.urlparse(uri)) path = parts[2] dirname, filename = os.path.split(path) filename = self.get_external_filename(filename, index) path = os.path.join(dirname, filename) parts[2] = path return urlparse.urlunparse(parts)
def _make_http_connection(init, mode, uri=None): """ Creates a HTTPConnection instance if the HTTP server supports Range requests, otherwise falls back to a generic InputStream. """ from astropy.extern.six.moves import http_client parsed = urlparse.urlparse(init) connection = http_client.HTTPConnection(parsed.netloc) connection.connect() # We request a range of everything ("0-") just to check if the # server understands that header entry. headers = {'Range': 'bytes=0-'} connection.request('GET', parsed.path, headers=headers) response = connection.getresponse() if response.status // 100 != 2: raise IOError("HTTP failed: {0} {1}".format( response.status, response.reason)) # Status 206 means a range was returned. If it's anything else # that indicates the server probably doesn't support Range # headers. if (response.status != 206 or response.getheader('accept-ranges', None) != 'bytes' or response.getheader('content-range', None) is None or response.getheader('content-length', None) is None): # Fall back to a regular input stream, but we don't # need to open a new connection. response.close = connection.close return InputStream(response, mode, uri=uri or init, close=True) # Since we'll be requesting chunks, we can't read at all with the # current request (because we can't abort it), so just close and # start over size = int(response.getheader('content-length')) connection.close() connection.connect() return HTTPConnection(connection, size, parsed.path, uri or init)
def _make_http_connection(init, mode, uri=None): """ Creates a HTTPConnection instance if the HTTP server supports Range requests, otherwise falls back to a generic InputStream. """ from astropy.extern.six.moves import http_client parsed = urlparse.urlparse(init) connection = http_client.HTTPConnection(parsed.netloc) connection.connect() # We request a range of everything ("0-") just to check if the # server understands that header entry. headers = {'Range': 'bytes=0-'} connection.request('GET', parsed.path, headers=headers) response = connection.getresponse() if response.status // 100 != 2: raise IOError("HTTP failed: {0} {1}".format( response.status, response.reason)) # Status 206 means a range was returned. If it's anything else # that indicates the server probably doesn't support Range # headers. if (response.status != 206 or response.getheader('accept-ranges', None) != 'bytes' or response.getheader('content-range', None) is None or response.getheader('content-length', None) is None): # Fall back to a regular input stream, but we don't # need to open a new connection. response.close = connection.close return InputStream(response, mode, uri=uri or init, close=True) # Since we'll be requesting chunks, we can't read at all with the # current request (because we can't abort it), so just close and # start over size = int(response.getheader('content-length')) connection.close() connection.connect() return HTTPConnection(connection, size, parsed.path, uri or init)
def list_name_is_url(name): may_be_url = urlparse.urlparse(name) return (may_be_url.scheme and may_be_url.netloc)
def get_base_uri(uri): """ For a given URI, return the part without any fragment. """ parts = urlparse.urlparse(uri) return urlparse.urlunparse(list(parts[:5]) + [''])
def list_name_is_url(name): may_be_url = urlparse.urlparse(name) return (may_be_url.scheme and may_be_url.netloc)
def get_file(init, mode='r', uri=None): """ Returns a `GenericFile` instance suitable for wrapping the given object `init`. If passed an already open file-like object, it must be opened for reading/writing in binary mode. It is the caller's responsibility to close it. Parameters ---------- init : object `init` may be: - A `bytes` or `unicode` file path or ``file:`` or ``http:`` url. - A Python 2 `file` object. - An `io.IOBase` object (the default file object on Python 3). - A ducktyped object that looks like a file object. If `mode` is ``"r"``, it must have a ``read`` method. If `mode` is ``"w"``, it must have a ``write`` method. If `mode` is ``"rw"`` it must have the ``read``, ``write``, ``tell`` and ``seek`` methods. - A `GenericFile` instance, in which case it is wrapped in a `GenericWrapper` instance, so that the file is closed when only when the final layer is unwrapped. mode : str Must be one of ``"r"``, ``"w"`` or ``"rw"``. uri : str Sets the base URI of the file object. This will be used to resolve any relative URIs contained in the file. This is redundant if `init` is a `bytes` or `unicode` object (since it will be the uri), and it may be determined automatically if `init` refers to a regular filesystem file. It is not required if URI resolution is not used in the file. Returns ------- fd : GenericFile Raises ------ ValueError, TypeError, IOError """ if mode not in ('r', 'w', 'rw'): raise ValueError("mode must be 'r', 'w' or 'rw'") # Special case for sys.stdout on Python 3, since it takes unicode # by default, but we need to write to it with bytes if six.PY3 and init in (sys.stdout, sys.stdin, sys.stderr): init = init.buffer if isinstance(init, (GenericFile, GenericWrapper)): if mode not in init.mode: raise ValueError( "File is opened as '{0}', but '{1}' was requested".format( init.mode, mode)) return GenericWrapper(init, uri=uri) elif isinstance(init, six.string_types): parsed = urlparse.urlparse(init) if parsed.scheme == 'http': if mode == 'w': raise ValueError( "HTTP connections can not be opened for writing") return _make_http_connection(init, mode, uri=uri) elif parsed.scheme in ('', 'file'): if mode == 'rw': realmode = 'r+b' else: realmode = mode + 'b' return RealFile( open(parsed.path, realmode), mode, close=True, uri=uri or parsed.path) elif isinstance(init, io.BytesIO): return MemoryIO(init, mode, uri=uri) elif isinstance(init, io.StringIO): raise TypeError( "io.StringIO objects are not supported. Use io.BytesIO instead.") elif six.PY2 and isinstance(init, file): if not mode in init.mode: raise ValueError( "File is opened as '{0}', but '{1}' was requested".format( init.mode, mode)) try: init.tell() except IOError: if mode == 'w': return OutputStream(init, uri=uri) elif mode == 'r': return InputStream(init, mode, uri=uri) else: raise ValueError( "File '{0}' could not be opened in 'rw' mode".format(init)) else: return RealFile(init, mode, uri=uri) elif isinstance(init, io.IOBase): if sys.version_info[:2] == (2, 6): raise ValueError("io.open file objects are not supported on Python 2.6") if (('r' in mode and not init.readable()) or ('w' in mode and not init.writable())): raise ValueError( "File is opened as '{0}', but '{1}' was requested".format( init.mode, mode)) if init.seekable(): if isinstance(init, (io.BufferedReader, io.BufferedWriter, io.BufferedRandom)): init2 = init.raw else: init2 = init if isinstance(init2, io.RawIOBase): result = RealFile(init2, mode, uri=uri) else: result = MemoryIO(init2, mode, uri=uri) result._secondary_fd = init return result else: if mode == 'w': return OutputStream(init, uri=uri) elif mode == 'r': return InputStream(init, mode, uri=uri) else: raise ValueError( "File '{0}' could not be opened in 'rw' mode".format(init)) elif 'w' in mode and ( hasattr(init, 'write') and hasattr(init, 'seek') and hasattr(init, 'tell')): return MemoryIO(init, mode, uri=uri) elif 'r' in mode and ( hasattr(init, 'read') and hasattr(init, 'seek') and hasattr(init, 'tell')): return MemoryIO(init, mode, uri=uri) elif 'w' in mode and hasattr(init, 'write'): return OutputStream(init, uri=uri) elif 'r' in mode and hasattr(init, 'read'): return InputStream(init, mode, uri=uri) raise ValueError("Can't handle '{0}' as a file for mode '{1}'".format( init, mode))
def get_base_uri(uri): """ For a given URI, return the part without any fragment. """ parts = urlparse.urlparse(uri) return urlparse.urlunparse(list(parts[:5]) + [''])
def get_file(init, mode='r', uri=None): """ Returns a `GenericFile` instance suitable for wrapping the given object `init`. If passed an already open file-like object, it must be opened for reading/writing in binary mode. It is the caller's responsibility to close it. Parameters ---------- init : object `init` may be: - A `bytes` or `unicode` file path or ``file:`` or ``http:`` url. - A Python 2 `file` object. - An `io.IOBase` object (the default file object on Python 3). - A ducktyped object that looks like a file object. If `mode` is ``"r"``, it must have a ``read`` method. If `mode` is ``"w"``, it must have a ``write`` method. If `mode` is ``"rw"`` it must have the ``read``, ``write``, ``tell`` and ``seek`` methods. - A `GenericFile` instance, in which case it is wrapped in a `GenericWrapper` instance, so that the file is closed when only when the final layer is unwrapped. mode : str Must be one of ``"r"``, ``"w"`` or ``"rw"``. uri : str Sets the base URI of the file object. This will be used to resolve any relative URIs contained in the file. This is redundant if `init` is a `bytes` or `unicode` object (since it will be the uri), and it may be determined automatically if `init` refers to a regular filesystem file. It is not required if URI resolution is not used in the file. Returns ------- fd : GenericFile Raises ------ ValueError, TypeError, IOError """ if mode not in ('r', 'w', 'rw'): raise ValueError("mode must be 'r', 'w' or 'rw'") # Special case for sys.stdout on Python 3, since it takes unicode # by default, but we need to write to it with bytes if six.PY3 and init in (sys.stdout, sys.stdin, sys.stderr): init = init.buffer if isinstance(init, (GenericFile, GenericWrapper)): if mode not in init.mode: raise ValueError( "File is opened as '{0}', but '{1}' was requested".format( init.mode, mode)) return GenericWrapper(init, uri=uri) elif isinstance(init, six.string_types): parsed = urlparse.urlparse(init) if parsed.scheme == 'http': if mode == 'w': raise ValueError( "HTTP connections can not be opened for writing") return _make_http_connection(init, mode, uri=uri) elif parsed.scheme in _local_file_schemes: if mode == 'rw': realmode = 'r+b' else: realmode = mode + 'b' realpath = url2pathname(parsed.path) return RealFile( open(realpath, realmode), mode, close=True, uri=uri) elif isinstance(init, io.BytesIO): return MemoryIO(init, mode, uri=uri) elif isinstance(init, io.StringIO): raise TypeError( "io.StringIO objects are not supported. Use io.BytesIO instead.") elif six.PY2 and isinstance(init, file): if not mode in init.mode: raise ValueError( "File is opened as '{0}', but '{1}' was requested".format( init.mode, mode)) try: init.tell() except IOError: if mode == 'w': return OutputStream(init, uri=uri) elif mode == 'r': return InputStream(init, mode, uri=uri) else: raise ValueError( "File '{0}' could not be opened in 'rw' mode".format(init)) else: return RealFile(init, mode, uri=uri) elif isinstance(init, io.IOBase): if sys.version_info[:2] == (2, 6): raise ValueError("io.open file objects are not supported on Python 2.6") if (('r' in mode and not init.readable()) or ('w' in mode and not init.writable())): raise ValueError( "File is opened as '{0}', but '{1}' was requested".format( init.mode, mode)) if init.seekable(): if isinstance(init, (io.BufferedReader, io.BufferedWriter, io.BufferedRandom)): init2 = init.raw else: init2 = init if isinstance(init2, io.RawIOBase): result = RealFile(init2, mode, uri=uri) else: result = MemoryIO(init2, mode, uri=uri) result._secondary_fd = init return result else: if mode == 'w': return OutputStream(init, uri=uri) elif mode == 'r': return InputStream(init, mode, uri=uri) else: raise ValueError( "File '{0}' could not be opened in 'rw' mode".format(init)) elif 'w' in mode and ( hasattr(init, 'write') and hasattr(init, 'seek') and hasattr(init, 'tell')): return MemoryIO(init, mode, uri=uri) elif 'r' in mode and ( hasattr(init, 'read') and hasattr(init, 'seek') and hasattr(init, 'tell')): return MemoryIO(init, mode, uri=uri) elif 'w' in mode and hasattr(init, 'write'): return OutputStream(init, uri=uri) elif 'r' in mode and hasattr(init, 'read'): return InputStream(init, mode, uri=uri) raise ValueError("Can't handle '{0}' as a file for mode '{1}'".format( init, mode))