__author__ = 'Marius Lindauer, Matthias Feurer, Katharina Eggensperger, Joshua Marben, André Biedenkapp, Aaron Klein,'\ 'Stefan Falkner and Frank Hutter' with open(os.path.join(os.path.dirname(__file__), 'requirements.txt')) as fh: dependencies.verify_packages(fh.read()) with open(os.path.join(os.path.dirname(__file__), 'extras_require.json')) as fh: extras_require = json.load(fh) extras_installed = set() for name, requirements in extras_require.items(): if name in ['documentation', 'test']: continue if dependencies.are_valid_packages(requirements): extras_installed.add(name) for requirement in requirements: package_name = dependencies.RE_PATTERN.match(requirement).group( 'name') # type: ignore[union-attr] # noqa F821 if package_name == 'scikit-optimize': package_name = 'skopt' lazy_import.lazy_module(package_name) if sys.version_info < (3, 5, 2): raise ValueError("SMAC requires Python 3.5.2 or newer.") if os.name != 'posix': print( 'Detected unsupported operating system: %s. Please be aware, that SMAC might not run on this system.' % sys.platform)
import futsu import lazy_import import os fgcpstorage = lazy_import.lazy_module('futsu.gcp.storage') gcstorage = futsu.env_lazy_import('FUTSU_GCP_ENABLE', 'google.cloud.storage') fs3 = lazy_import.lazy_module('futsu.aws.s3') ffs = lazy_import.lazy_module('futsu.fs') urllib_request = lazy_import.lazy_module('urllib.request') shutil = lazy_import.lazy_module('shutil') def local_to_path(dst, src): if fgcpstorage.is_blob_path(dst): gcs_client = gcstorage.client.Client() fgcpstorage.file_to_blob(dst, src, gcs_client) return if fs3.is_blob_path(dst): client = fs3.create_client() fs3.file_to_blob(dst, src, client) return ffs.cp(dst, src) def path_to_local(dst, src): if fgcpstorage.is_blob_path(src): gcs_client = gcstorage.client.Client() fgcpstorage.blob_to_file(dst, src, gcs_client) return if fs3.is_blob_path(src): client = fs3.create_client()
import warnings import os import re from io import BytesIO from collections import OrderedDict, defaultdict from six import string_types import numpy as np import traitlets as tl import pandas as pd # Core dependency of xarray # Helper utility for optional imports from lazy_import import lazy_module # Optional dependencies bs4 = lazy_module('bs4') # Not used directly, but used indirectly by bs4 so want to check if it's available lxml = lazy_module('lxml') pydap = lazy_module('pydap') lazy_module('pydap.client') lazy_module('pydap.model') rasterio = lazy_module('rasterio') h5py = lazy_module('h5py') boto3 = lazy_module('boto3') requests = lazy_module('requests') # esri RasterToNumPyArray = lazy_module('arcpy.RasterToNumPyArray') urllib3 = lazy_module('urllib3') certifi = lazy_module('certifi') # Internal dependencies
from __future__ import division, unicode_literals, print_function, absolute_import from collections import OrderedDict import io from podpac.core.coordinates.array_coordinates1d import ArrayCoordinates1d import re from six import string_types import traitlets as tl import numpy as np import pyproj import logging from lazy_import import lazy_module rasterio = lazy_module("rasterio") boto3 = lazy_module("boto3") from podpac.core.utils import common_doc, cached_property from podpac.core.coordinates import UniformCoordinates1d, Coordinates, merge_dims from podpac.core.data.datasource import COMMON_DATA_DOC, DATA_DOC from podpac.core.data.file_source import BaseFileSource from podpac.core.authentication import S3Mixin from podpac.core.interpolation.interpolation import InterpolationMixin _logger = logging.getLogger(__name__) @common_doc(COMMON_DATA_DOC) class RasterioRaw(S3Mixin, BaseFileSource): """Create a DataSource using rasterio.
import lazy_import import binascii import base64 import codecs import html import base58 import ujson yaml = lazy_import.lazy_module("yaml") import regex as re import hexdump from ast import literal_eval from typing import Union from urllib.parse import quote_plus as _urllib_quote_plus from urllib.parse import unquote_plus as _urllib_unquote_plus from ..core import ChepyCore, ChepyDecorators from chepy.modules.internal.constants import Encoding class DataFormat(ChepyCore): def __init__(self, *data): super().__init__(*data) @ChepyDecorators.call_stack def eval_state(self): """Eval state as python. Handy when converting string representation of objects. Returns: Chepy: The Chepy object
from six import string_types from dateutil import parser from io import StringIO try: import cPickle # Python 2.7 except: import _pickle as cPickle import numpy as np import traitlets as tl # Optional dependencies from lazy_import import lazy_module bs4 = lazy_module("bs4") import podpac from podpac.core.utils import _get_from_url, cached_property from podpac.data import DataSource from podpac.compositor import TileCompositorRaw from podpac.interpolators import InterpolationMixin _logger = logging.getLogger(__name__) def _convert_str_to_vals(properties): IGNORE_KEYS = ["sitenumber"] for k, v in properties.items(): if not isinstance(v, string_types) or k in IGNORE_KEYS: continue
- Create a "Service" within that network. e.g. "apache-public". - Easily control network connections and firewalls using "Paths". The goal is to provide an intuitive abstraction that is powerful enough to build on, so that building other layers on top is easy and anything built on it is automatically cross cloud. The main entry point to this module is through the `cloudless.Client` object, and all calls that interact with a backing cloud provider go through this object. """ import logging import lazy_import # Lazily import these so the import and command line is fast unless we are actually creating a # client. # pylint:disable=invalid-name network = lazy_import.lazy_module("cloudless.network") service = lazy_import.lazy_module("cloudless.service") paths = lazy_import.lazy_module("cloudless.paths") image = lazy_import.lazy_module("cloudless.image") def set_level(level): """ Set log level for this module. """ logging.basicConfig() logger = logging.getLogger(__name__) # pylint: disable=locally-disabled, invalid-name logger.setLevel(level) def set_global_level(level):
""" from typing import Optional import lazy_import from pathlib import PurePosixPath, Path import os, threading, sys from lazydata.config.config import Config from lazydata.storage.hash import calculate_file_sha256 from lazydata.storage.local import LocalStorage from pySmartDL import SmartDL boto3 = lazy_import.lazy_module("boto3") botocore = lazy_import.lazy_module("botocore") from urllib.parse import urlparse import logging logging.getLogger('boto3').setLevel(logging.CRITICAL) logging.getLogger('botocore').setLevel(logging.CRITICAL) class RemoteStorage: """ A storage backend abstraction layer """ @staticmethod def get_from_url(remote_url: str, endpoint_url: str): if remote_url.startswith("s3://"):
import logging import requests from six import string_types import numpy as np import xarray as xr import traitlets as tl # Set up logging _logger = logging.getLogger(__name__) # Helper utility for optional imports from lazy_import import lazy_module # Optional dependencies bs4 = lazy_module('bs4') boto3 = lazy_module('boto3') # fixing problem with older versions of numpy if not hasattr(np, 'isnat'): def isnat(a): return a.astype(str) == 'None' np.isnat = isnat # Internal dependencies import podpac from podpac.core.coordinates import Coordinates, union, merge_dims, concat from podpac.core.data import types as datatype from podpac.core import authentication from podpac.core.utils import common_doc from podpac.core.data.datasource import COMMON_DATA_DOC
from functools import reduce import traitlets as tl import pyproj from podpac.core.utils import common_doc, cached_property, resolve_bbox_order from podpac.core.data.datasource import DataSource from podpac.core.interpolation.interpolation import InterpolationMixin, InterpolationTrait from podpac.core.node import NodeException from podpac.core.coordinates import Coordinates from podpac.core.coordinates import UniformCoordinates1d, ArrayCoordinates1d, Coordinates1d, StackedCoordinates # Optional dependencies from lazy_import import lazy_module, lazy_class bs4 = lazy_module("bs4") lxml = lazy_module("lxml") # used by bs4 so want to check if it's available owslib_wcs = lazy_module("owslib.wcs") owslib_util = lazy_module("owslib.util") rasterio = lazy_module("rasterio") logger = logging.getLogger(__name__) class MockWCSClient(tl.HasTraits): source = tl.Unicode() version = tl.Enum(["1.0.0"], default_value="1.0.0") headers = None cookies = None auth = tl.Any()
""" Acquire : (C) Christopher Woods 2019 This module provides the stubs around modules that are not installed or available on this system, e.g. for license incompatibility reasons (i.e. while Acquire is Apache licensed, some users may want to use it in systems where GPL modules are installed) """ try: import lazy_import lazy_import.logging.disable(lazy_import.logging.DEBUG) except: # lazy_import is not available, e.g. because we want the Apache # licensed version of this code - import the non-lazy wrapper from ._lazy_import import * requests = lazy_import.lazy_module("requests")
from .gaisser_hillas_fit import gaisser_hillas from .gaisser_hillas_fit import gaisser_hillas_log from .gaisser_hillas_fit import gaisser_hillas_fit from .running_mean import running_mean import os import lazy_import if "lazy_import" in os.environ: lazy_import.lazy_module("src.analysis.histogram") from . import histogram
__build_platform__ = 'n/a' __all__ = ["gui", "misc", "visual", "core", "event", "data", "sound", "microphone"] # for developers the following allows access to the current git sha from # their repository if __git_sha__ == 'n/a': import subprocess # see if we're in a git repo and fetch from there try: thisFileLoc = os.path.split(__file__)[0] output = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=thisFileLoc, stderr=subprocess.PIPE) except Exception: output = False if output: __git_sha__ = output.strip() # remove final linefeed # update preferences and the user paths if 'installing' not in locals(): from psychopy.preferences import prefs for pathName in prefs.general['paths']: sys.path.append(pathName) from psychopy.tools.versionchooser import useVersion, ensureMinimal # lazy imports for things we only want to import when needed import lazy_import git = lazy_import.lazy_module("git")
import lazy_import as _lazy_import from Acquire.Service import call_function as _call_function from Acquire.Service import pack_arguments as _pack_arguments from Acquire.Service import unpack_arguments as _unpack_arguments from Acquire.Service import Service as _Service from Acquire.ObjectStore import bytes_to_string as _bytes_to_string from Acquire.ObjectStore import string_to_bytes as _string_to_bytes from Acquire.Crypto import PrivateKey as _PrivateKey from Acquire.Crypto import OTP as _OTP from ._errors import LoginError _pyotp = _lazy_import.lazy_module("pyotp") __all__ = ["ServiceWallet"] class ServiceWallet: """This class holds a wallet that can be used to simplify sending passwords and one-time-password (OTP) codes to administer one of the Acquire services This holds a wallet of passwords and (optionally) OTP secrets that are encrypted using a local keypair that is unlocked by a password supplied by the user locally. """ def __init__(self): self._wallet_key = None
""" Algorithm Summary """ from __future__ import division, unicode_literals, print_function, absolute_import from collections import OrderedDict import inspect import numpy as np import xarray as xr import traitlets as tl from lazy_import import lazy_module # Optional dependencies ne = lazy_module('numexpr') # Internal dependencies from podpac.core.coordinates import Coordinates, union from podpac.core.units import UnitsDataArray from podpac.core.node import Node from podpac.core.node import NodeException from podpac.core.node import COMMON_NODE_DOC from podpac.core.node import node_eval from podpac.core.utils import common_doc COMMON_DOC = COMMON_NODE_DOC.copy() class Algorithm(Node): """Base class for algorithm and computation nodes.
# coding: utf-8 # Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved. import lazy_import as _lazy_import chardet = _lazy_import.lazy_module("oci._vendor.chardet") httpsig_cffi = _lazy_import.lazy_module("oci._vendor.httpsig_cffi") jwt = _lazy_import.lazy_module("oci._vendor.jwt") idna = _lazy_import.lazy_module("oci._vendor.idna") six = _lazy_import.lazy_module("oci._vendor.six") urllib3 = _lazy_import.lazy_module("oci._vendor.urllib3") requests = _lazy_import.lazy_module("oci._vendor.requests")
import pathlib from abc import ABC, abstractmethod from typing import Any, Optional, Type, NoReturn, Union import lazy_import nx = lazy_import.lazy_module('networkx') gtall = lazy_import.lazy_module('graph_tool.all') NODE_T = Type[Any] EDGE_T = Type[Any] GRAPH_T = Optional[Any] class KnowledgeGraphBase(ABC): LOAD_FORMAT = [] def __init__(self, graph_path: Union[str, pathlib.Path], is_lazy: bool, *args, **kwargs): self._graph: GRAPH_T = None self.graph_path: pathlib.Path = self._check_path(graph_path) if not is_lazy: a = self._graph @property def graph(self): if self._graph is None: self._graph = self.load_graph(self._check_path(self.graph_path)) return self._graph @graph.setter
from inspect import signature import math from typing import Optional import numpy as np import sklearn.gaussian_process.kernels import scipy.optimize import scipy.spatial.distance import scipy.special from lazy_import import lazy_module kernels = lazy_module('skopt.learning.gaussian_process.kernels') # This file contains almost no type annotations to simplify comparing it to the original scikit-learn version! def get_conditional_hyperparameters(X: np.ndarray, Y: Optional[np.ndarray]) -> np.ndarray: # Taking care of conditional hyperparameters according to Levesque et al. X_cond = X <= -1 if Y is not None: Y_cond = Y <= -1 else: Y_cond = X <= -1 active = ~((np.expand_dims(X_cond, axis=1) != Y_cond).any(axis=2)) return active class MagicMixin: prior = None
General-purpose Algorithm Nodes. """ from __future__ import division, unicode_literals, print_function, absolute_import import sys import warnings import numpy as np import xarray as xr import traitlets as tl # Optional dependencies from lazy_import import lazy_module ne = lazy_module("numexpr") from podpac import settings from podpac import Coordinates from podpac.core.node import Node, NodeException from podpac.core.utils import NodeTrait from podpac.core.algorithm.algorithm import Algorithm if sys.version_info.major == 2: class PermissionError(OSError): pass class GenericInputs(Algorithm): """Base class for Algorithms that accept generic named inputs."""
import lazy_import lazy_import.lazy_module("configupdater") lazy_import.lazy_module("cerberus") lazy_import.lazy_module("mfutil") opinionated_configparser = lazy_import.lazy_module("opinionated_configparser")
# Copyright (c) Microsoft # Licensed under the MIT License. # Written by Bin Xiao ([email protected]) # ------------------------------------------------------------------------------ from __future__ import absolute_import from __future__ import division from __future__ import print_function from collections import defaultdict from collections import OrderedDict import logging import os import lazy_import COCO = lazy_import.lazy_module("pycocotools.coco") COCOeval = lazy_import.lazy_module("pycocotools.cocoeval") import json import numpy as np from .estimation import EstiamtionDataset from ..ops.nms.nms import oks_nms from ..ops.nms.nms import soft_oks_nms logger = logging.getLogger(__name__) class COCODataset(EstiamtionDataset): ''' "keypoints": { 0: "nose",
def app(): main = lazy_import.lazy_module('main') yield main.app importlib.reload(main)
"""Package that handles all data processing.""" from .path import get_path, set_path from .clear_run import clear_run from .config import numpy_config from .config import numpy_particle_label from .config import numpy_data_layout from .config import numpy_label_layout from .caching import prepare_cache from .caching import cache_dataset import os import lazy_import if "lazy_import" in os.environ: lazy_import.lazy_module("src.data.raw") lazy_import.lazy_module("src.data.interim") lazy_import.lazy_module("src.data.processed") lazy_import.lazy_module("src.data.external") lazy_import.lazy_module("src.data.random") else: lazy_import.lazy_module("src.data.processed") lazy_import.lazy_module("src.data.random") from . import raw from . import interim from . import processed from . import external from . import random
import os from termcolor import colored import lazy_import np = lazy_import.lazy_module("numpy") yaml = lazy_import.lazy_module("yaml") shutil = lazy_import.lazy_module("shutil") from mantraml.core.cloud.AWS import AWS from .consts import METADATA_FILE_NAME, SHORT_HASH_INT, DEFAULT_EPOCHS, DEFAULT_BATCH_SIZE class MantraModel: """ By inheriting from this MantraModel class, a deep learning model gets access to Mantra integration, including cloud integration, model monitoring and more. The core methods below link the model with configuration and training metadata. """ # default training arguments batch_size = 64 epochs = 200 def configure_core_arguments(self, args): """ This method adds core training attributes from an argument parser namespace (args) such as the number of epochs, the batch size, and more Parameters ----------- args - Argument parser namespace Containing training arguments such as batch size, number of epochs
Supports access to: - Landsat 8 on AWS OpenData: https://registry.opendata.aws/landsat-8/ - Sentinel 2 """ import logging import datetime import os import numpy as np import traitlets as tl from lazy_import import lazy_module satsearch = lazy_module("satsearch") # Internal dependencies import podpac from podpac.compositor import TileCompositor from podpac.core.data.rasterio_source import RasterioRaw from podpac.core.units import UnitsDataArray from podpac.authentication import S3Mixin from podpac import settings _logger = logging.getLogger(__name__) def _get_asset_info(item, name): """ for forwards/backwards compatibility, convert B0x to/from Bx as needed """
# coding: utf-8 from __future__ import annotations from abc import ABCMeta, abstractmethod from dataclasses import dataclass, field from io import BytesIO from PIL import Image, ImageOps, ImageEnhance from lazy_import import lazy_module from bookworm import typehints as t from bookworm.image_io import ImageIO from bookworm.logger import logger np = lazy_module("numpy") cv2 = lazy_module("cv2") cv2_utils = lazy_module("bookworm.ocr_engines.cv2_utils") log = logger.getChild(__name__) @dataclass class ImageProcessingPipeline(metaclass=ABCMeta): """Preprocesses images before running OCR.""" images: t.Tuple[ImageIO] ocr_request: "OcrRequest" args: dict = field(default_factory=dict) run_order: t.ClassVar[int] = 0 @abstractmethod def should_process(self) -> bool: """Should this pipeline be applied given the arguments."""
import logging import datetime import traitlets as tl import numpy as np # Helper utility for optional imports from lazy_import import lazy_module # Internal imports import podpac from podpac import Coordinates from podpac.utils import cached_property intake = lazy_module("intake") # lazy_module('intake.catalog.local.LocalCatalogEntry') class IntakeCatalog(podpac.data.DataSource): """ Support for Intake Catalogs (https://intake.readthedocs.io/en/latest/index.html) This primarily supports CSV data sources while we expand for Intake Catalogs. Parameters ---------- uri : str, required Intake Catalog uri (local path to catalog yml file, or remote uri) See https://intake.readthedocs.io/en/latest/catalog.html#local-catalogs source : str, required Intake Catalog source
import datetime import importlib from unittest.mock import call import pytest import freezegun import lazy_import mongo_lock = lazy_import.lazy_module('src.mongo_lock') const = lazy_import.lazy_module('src.const') @pytest.fixture def MongoThrottling(): yield mongo_lock.MongoThrottling importlib.reload(mongo_lock) @pytest.fixture def mocked_mongo(mocker): mongo_lock.MongoClient = mocker.MagicMock() collection = mocker.MagicMock() mongo_lock.MongoClient.return_value = { const.MONGODB_DB_NAME: { const.MONGODB_COLLECTION_NAME: collection } } yield mongo_lock.MongoClient, collection class TestMongoThrottlingLock:
from __future__ import division, unicode_literals, print_function, absolute_import import datetime import traitlets as tl import numpy as np from lazy_import import lazy_module s3fs = lazy_module("s3fs") # Internal imports from podpac.core.data.rasterio_source import RasterioRaw from podpac.core.authentication import S3Mixin from podpac.coordinates import Coordinates from podpac.utils import cached_property, DiskCacheMixin from podpac.compositor import TileCompositor BUCKET = "noaa-gfs-pds" class GFSSourceRaw(DiskCacheMixin, RasterioRaw): """Raw GFS data from S3 Attributes ---------- parameter : str parameter, e.g. 'SOIM'. level : str depth, e.g. "0-10 m DPTH" date : str
from __future__ import division, print_function, absolute_import import fnmatch from lazy_import import lazy_module boto3 = lazy_module("boto3") import podpac from podpac.core.settings import settings from podpac.core.cache.utils import CacheException, CacheWildCard from podpac.core.cache.file_cache_store import FileCacheStore class S3CacheStore(FileCacheStore): # pragma: no cover cache_mode = "s3" cache_modes = set(["s3", "all"]) _limit_setting = "S3_CACHE_MAX_BYTES" _delim = "/" def __init__(self, s3_bucket=None, aws_region_name=None, aws_access_key_id=None, aws_secret_access_key=None): """Initialize a cache that uses a folder on a local disk file system. Parameters ---------- max_size : None, optional Maximum allowed size of the cache store in bytes. Defaults to podpac 'S3_CACHE_MAX_BYTES' setting, or no limit if this setting does not exist.
import lazy_import RSA = lazy_import.lazy_module("Crypto.PublicKey.RSA") OpenSSL = lazy_import.lazy_module("OpenSSL") from ..core import ChepyCore, ChepyDecorators class Publickey(ChepyCore): def __init__(self, *data): super().__init__(*data) def _convert_cert_to_obj(self, cert): issuer = cert.get_issuer() subject = cert.get_subject() pubkey = cert.get_pubkey() info = { "version": cert.get_version(), "serial": cert.get_serial_number(), "algo": cert.get_signature_algorithm(), "before": cert.get_notBefore(), "after": cert.get_notAfter(), "issuer": { "C": issuer.C, "ST": issuer.ST, "L": issuer.L, "O": issuer.O, "OU": issuer.OU, "CN": issuer.CN, "email": issuer.emailAddress, }, "subject": {
#!/usr/bin/env python # -*- coding: utf-8 -*- # Part of the PsychoPy library # Copyright (C) 2018 Jonathan Peirce # Distributed under the terms of the GNU General Public License (GPL). # DONE: add+commit before push # DONE: add .gitignore file. Added when opening a repo without one # DONE: fork+sync doesn't yet fork the project first # DONE: rather than clone into a folder with files we should init/add/push # # TODO: after clone, remember this folder for next file-open call # TODO: user dlg could/should be local not a browser # TODO: syncProject() doesn't handle case of a local git pushing to new gitlab # TODO: if more than one remote then offer options from psychopy.projects import pavlovia from .functions import * from .project import ProjectEditor, syncProject from ._base import PavloviaMiniBrowser from . import menu, project, search, toolbar # lazy import git import lazy_import sync = lazy_import.lazy_module("psychopy.app.pavlovia_ui.sync")