def load(url_or_handle, cache=None, encoding="utf-8"): """Load a file. File format is inferred from url. File retrieval strategy is inferred from URL. Returned object type is inferred from url extension. Args: url_or_handle: a (reachable) URL, or an already open file handle Raises: RuntimeError: If file extension or URL is not supported. """ ext = get_extension(url_or_handle) try: loader = loaders[ext.lower()] message = "Using inferred loader '%s' due to passed file extension '%s'." log.debug(message, loader.__name__[6:], ext) return load_using_loader(url_or_handle, loader, cache, encoding) except KeyError: log.warning("Unknown extension '%s', attempting to load as image.", ext) try: with read_handle(url_or_handle, cache=cache) as handle: result = _load_img(handle) except Exception as e: message = "Could not load resource %s as image. Supported extensions: %s" log.error(message, url_or_handle, list(loaders)) raise RuntimeError(message.format(url_or_handle, list(loaders))) else: log.info("Unknown extension '%s' successfully loaded as image.", ext) return result
def load_using_loader(url_or_handle, loader, cache, encoding): if is_handle(url_or_handle): result = loader(url_or_handle, encoding=encoding) else: url = url_or_handle try: with read_handle(url, cache=cache) as handle: result = loader(handle, encoding=encoding) except (DecodeError, ValueError): log.warning( "While loading '%s' an error occurred. Purging cache once and trying again; if this fails we will raise an Exception!", url) # since this may have been cached, it's our responsibility to try again once # since we use a handle here, the next DecodeError should propagate upwards with read_handle(url, cache='purge') as handle: result = load_using_loader(handle, loader, cache, encoding) return result
def load(url_or_handle, allow_unsafe_formats=False, cache=None, **kwargs): """Load a file. File format is inferred from url. File retrieval strategy is inferred from URL. Returned object type is inferred from url extension. Args: url_or_handle: a (reachable) URL, or an already open file handle allow_unsafe_formats: set to True to allow saving unsafe formats (eg. pickles) cache: whether to attempt caching the resource. Defaults to True only if the given URL specifies a remote resource. Raises: RuntimeError: If file extension or URL is not supported. """ # handle lists of URLs in a performant manner if isinstance(url_or_handle, (list, tuple)): return _load_urls(url_or_handle, cache=cache, **kwargs) ext, decompressor_ext = _get_extension(url_or_handle) try: ext = ext.lower() if ext in loaders: loader = loaders[ext] elif ext in unsafe_loaders: if not allow_unsafe_formats: raise ValueError( f"{ext} is considered unsafe, you must explicitly allow its use by passing allow_unsafe_formats=True" ) loader = unsafe_loaders[ext] else: raise KeyError(f'no loader found for {ext}') decompressor = decompressors[ decompressor_ext] if decompressor_ext is not None else nullcontext message = "Using inferred loader '%s' due to passed file extension '%s'." log.debug(message, loader.__name__[6:], ext) return load_using_loader(url_or_handle, decompressor, loader, cache, **kwargs) except KeyError: log.warning("Unknown extension '%s', attempting to load as image.", ext) try: with read_handle(url_or_handle, cache=cache) as handle: result = _load_img(handle) except Exception as e: message = "Could not load resource %s as image. Supported extensions: %s" log.error(message, url_or_handle, list(loaders)) raise RuntimeError(message.format(url_or_handle, list(loaders))) else: log.info("Unknown extension '%s' successfully loaded as image.", ext) return result
def load_using_loader(url_or_handle, decompressor, loader, cache, **kwargs): if is_handle(url_or_handle): with decompressor(url_or_handle) as decompressor_handle: result = loader(decompressor_handle, **kwargs) else: url = url_or_handle try: with read_handle(url, cache=cache) as handle: with decompressor(handle) as decompressor_handle: result = loader(decompressor_handle, **kwargs) except (DecodeError, ValueError): log.warning( "While loading '%s' an error occurred. Purging cache once and trying again; if this fails we will raise an Exception! Current io scopes: %r", url, current_io_scopes(), ) # since this may have been cached, it's our responsibility to try again once # since we use a handle here, the next DecodeError should propagate upwards with read_handle(url, cache="purge") as handle: result = load_using_loader(handle, decompressor, loader, cache, **kwargs) return result
def load(url, cache=None, encoding='utf-8'): """Load a file. File format is inferred from url. File retrieval strategy is inferred from URL. Returned object type is inferred from url extension. Args: path: a (reachable) URL Raises: RuntimeError: If file extension or URL is not supported. """ _, ext = os.path.splitext(url) if not ext: raise RuntimeError("No extension in URL: " + url) ext = ext.lower() if ext in loaders: loader = loaders[ext] message = "Using inferred loader '%s' due to passed file extension '%s'." log.debug(message, loader.__name__[6:], ext) with read_handle(url, cache=cache) as handle: result = loader(handle, encoding=encoding) return result else: log.warn("Unknown extension '%s', attempting to load as image.", ext) try: with read_handle(url, cache=cache) as handle: result = _load_img(handle) except Exception as e: message = "Could not load resource %s as image. Supported extensions: %s" log.error(message, url, list(loaders)) raise RuntimeError(message.format(url, list(loaders))) else: log.info("Unknown extension '%s' successfully loaded as image.", ext) return result
def test_read_handle_behaves_like_file(): with read_handle(path) as handle: content1 = handle.read() handle.seek(0) content2 = handle.read() assert content1 == content2
def test_read_handle_txt_file(): with read_handle(path) as handle: content1 = handle.read().decode('utf-8') assert content1 == string
def load_json_from_url(url,cache=None,encoding='utf-8'): with read_handle(url,cache=cache) as handle: res = handle.read().decode(encoding=encoding) return json.loads(res)
def compile_html(input_path, html_path=None, *, props=None, precision=None, title=None, div_id=None, inline_js=None, svelte_to_js=None, js_path=None, js_name=None, js_lint=None): """Compile Svelte or JavaScript to HTML. Arguments: input_path: path to input Svelte or JavaScript file html_path: path to output HTML file defaults to input_path with a new .html suffix props: JSON-serializable object to pass to Svelte script defaults to an empty object precision: number of significant figures to round numpy arrays to defaults to no rounding title: title of HTML page defaults to html_path filename without suffix div_id: HTML id of div containing Svelte component defaults to _default_div_id inline_js: whether to insert the JavaScript into the HTML page inline defaults to svelte_to_js svelte_to_js: whether to first compile from Svelte to JavaScript defaults to whether input_path doesn't have a .js suffix js_path: path to output JavaScript file if compiling from Svelte and not inserting the JavaScript inline defaults to compile_js default js_name: name of JavaScript global variable should match existing name if compiling from JavaScript defaults to _default_js_name js_lint: whether to use eslint if compiling from Svelte defaults to compile_js default """ if html_path is None: html_path = replace_file_extension(input_path, ".html") if props is None: props = {} if title is None: title = os.path.basename(html_path).rsplit(".", 1)[0] if div_id is None: div_id = _default_div_id if svelte_to_js is None: svelte_to_js = not input_path.endswith(".js") if inline_js is None: inline_js = svelte_to_js if svelte_to_js: if inline_js: if js_path is None: js_path = replace_file_extension(input_path, ".js") prefix = "svelte_" + os.path.basename(js_path) if prefix.endswith(".js"): prefix = prefix[:-3] _, js_path = tempfile.mkstemp(suffix=".js", prefix=prefix + "_", dir=_temp_config_dir, text=True) try: compile_js_result = compile_js(input_path, js_path, js_name=js_name, js_lint=js_lint) except CompileError as exn: raise CompileError( "Unable to compile Svelte source.\n" "See the above advice or try supplying pre-compiled JavaScript." ) from exn js_path = compile_js_result["js_path"] js_name = compile_js_result["js_name"] command_output = compile_js_result["command_output"] else: js_path = input_path if js_name is None: js_name = _default_js_name command_output = None if inline_js: with read_handle(js_path, cache=False, mode="r") as js_file: js_code = js_file.read().rstrip("\n") js_html = "<script>\n" + js_code + "\n </script>" js_path = None else: js_relpath = os.path.relpath(js_path, start=os.path.dirname(html_path)) js_html = '<script src="' + js_relpath + '"></script>' with write_handle(html_path, "w") as html_file: html_file.write("""<!DOCTYPE html> <html> <head> <meta charset="utf-8"> <title>""" + title + '''</title> </head> <body> <div id="''' + div_id + """"></div> """ + js_html + """ <script> var app = new """ + js_name + """({ target: document.querySelector("#""" + div_id + """"), props: """ + json.dumps(props, cls=encoder(precision=precision)) + """ }); </script> </body> </html>""") return { "html_path": html_path, "js_path": js_path if svelte_to_js else None, "title": title, "div_id": div_id, "js_name": js_name, "command_output": command_output, }