__author__ = 'Marius Lindauer, Matthias Feurer, Katharina Eggensperger, Joshua Marben, André Biedenkapp, Aaron Klein,'\
'Stefan Falkner and Frank Hutter'
with open(os.path.join(os.path.dirname(__file__), 'requirements.txt')) as fh:
dependencies.verify_packages(fh.read())
with open(os.path.join(os.path.dirname(__file__),
'extras_require.json')) as fh:
extras_require = json.load(fh)
extras_installed = set()
for name, requirements in extras_require.items():
if name in ['documentation', 'test']:
continue
if dependencies.are_valid_packages(requirements):
extras_installed.add(name)
for requirement in requirements:
package_name = dependencies.RE_PATTERN.match(requirement).group(
'name') # type: ignore[union-attr] # noqa F821
if package_name == 'scikit-optimize':
package_name = 'skopt'
lazy_import.lazy_module(package_name)
if sys.version_info < (3, 5, 2):
raise ValueError("SMAC requires Python 3.5.2 or newer.")
if os.name != 'posix':
print(
'Detected unsupported operating system: %s. Please be aware, that SMAC might not run on this system.'
% sys.platform)
import futsu
import lazy_import
import os
fgcpstorage = lazy_import.lazy_module('futsu.gcp.storage')
gcstorage = futsu.env_lazy_import('FUTSU_GCP_ENABLE', 'google.cloud.storage')
fs3 = lazy_import.lazy_module('futsu.aws.s3')
ffs = lazy_import.lazy_module('futsu.fs')
urllib_request = lazy_import.lazy_module('urllib.request')
shutil = lazy_import.lazy_module('shutil')
def local_to_path(dst, src):
if fgcpstorage.is_blob_path(dst):
gcs_client = gcstorage.client.Client()
fgcpstorage.file_to_blob(dst, src, gcs_client)
return
if fs3.is_blob_path(dst):
client = fs3.create_client()
fs3.file_to_blob(dst, src, client)
return
ffs.cp(dst, src)
def path_to_local(dst, src):
if fgcpstorage.is_blob_path(src):
gcs_client = gcstorage.client.Client()
fgcpstorage.blob_to_file(dst, src, gcs_client)
return
if fs3.is_blob_path(src):
client = fs3.create_client()
import warnings
import os
import re
from io import BytesIO
from collections import OrderedDict, defaultdict
from six import string_types
import numpy as np
import traitlets as tl
import pandas as pd # Core dependency of xarray
# Helper utility for optional imports
from lazy_import import lazy_module
# Optional dependencies
bs4 = lazy_module('bs4')
# Not used directly, but used indirectly by bs4 so want to check if it's available
lxml = lazy_module('lxml')
pydap = lazy_module('pydap')
lazy_module('pydap.client')
lazy_module('pydap.model')
rasterio = lazy_module('rasterio')
h5py = lazy_module('h5py')
boto3 = lazy_module('boto3')
requests = lazy_module('requests')
# esri
RasterToNumPyArray = lazy_module('arcpy.RasterToNumPyArray')
urllib3 = lazy_module('urllib3')
certifi = lazy_module('certifi')
# Internal dependencies
from __future__ import division, unicode_literals, print_function, absolute_import
from collections import OrderedDict
import io
from podpac.core.coordinates.array_coordinates1d import ArrayCoordinates1d
import re
from six import string_types
import traitlets as tl
import numpy as np
import pyproj
import logging
from lazy_import import lazy_module
rasterio = lazy_module("rasterio")
boto3 = lazy_module("boto3")
from podpac.core.utils import common_doc, cached_property
from podpac.core.coordinates import UniformCoordinates1d, Coordinates, merge_dims
from podpac.core.data.datasource import COMMON_DATA_DOC, DATA_DOC
from podpac.core.data.file_source import BaseFileSource
from podpac.core.authentication import S3Mixin
from podpac.core.interpolation.interpolation import InterpolationMixin
_logger = logging.getLogger(__name__)
@common_doc(COMMON_DATA_DOC)
class RasterioRaw(S3Mixin, BaseFileSource):
"""Create a DataSource using rasterio.
import lazy_import
import binascii
import base64
import codecs
import html
import base58
import ujson
yaml = lazy_import.lazy_module("yaml")
import regex as re
import hexdump
from ast import literal_eval
from typing import Union
from urllib.parse import quote_plus as _urllib_quote_plus
from urllib.parse import unquote_plus as _urllib_unquote_plus
from ..core import ChepyCore, ChepyDecorators
from chepy.modules.internal.constants import Encoding
class DataFormat(ChepyCore):
def __init__(self, *data):
super().__init__(*data)
@ChepyDecorators.call_stack
def eval_state(self):
"""Eval state as python.
Handy when converting string representation
of objects.
Returns:
Chepy: The Chepy object
from six import string_types
from dateutil import parser
from io import StringIO
try:
import cPickle # Python 2.7
except:
import _pickle as cPickle
import numpy as np
import traitlets as tl
# Optional dependencies
from lazy_import import lazy_module
bs4 = lazy_module("bs4")
import podpac
from podpac.core.utils import _get_from_url, cached_property
from podpac.data import DataSource
from podpac.compositor import TileCompositorRaw
from podpac.interpolators import InterpolationMixin
_logger = logging.getLogger(__name__)
def _convert_str_to_vals(properties):
IGNORE_KEYS = ["sitenumber"]
for k, v in properties.items():
if not isinstance(v, string_types) or k in IGNORE_KEYS:
continue
- Create a "Service" within that network. e.g. "apache-public".
- Easily control network connections and firewalls using "Paths".
The goal is to provide an intuitive abstraction that is powerful enough to build
on, so that building other layers on top is easy and anything built on it is
automatically cross cloud.
The main entry point to this module is through the `cloudless.Client` object, and all calls that
interact with a backing cloud provider go through this object.
"""
import logging
import lazy_import
# Lazily import these so the import and command line is fast unless we are actually creating a
# client.
# pylint:disable=invalid-name
network = lazy_import.lazy_module("cloudless.network")
service = lazy_import.lazy_module("cloudless.service")
paths = lazy_import.lazy_module("cloudless.paths")
image = lazy_import.lazy_module("cloudless.image")
def set_level(level):
"""
Set log level for this module.
"""
logging.basicConfig()
logger = logging.getLogger(__name__) # pylint: disable=locally-disabled, invalid-name
logger.setLevel(level)
def set_global_level(level):
"""
from typing import Optional
import lazy_import
from pathlib import PurePosixPath, Path
import os, threading, sys
from lazydata.config.config import Config
from lazydata.storage.hash import calculate_file_sha256
from lazydata.storage.local import LocalStorage
from pySmartDL import SmartDL
boto3 = lazy_import.lazy_module("boto3")
botocore = lazy_import.lazy_module("botocore")
from urllib.parse import urlparse
import logging
logging.getLogger('boto3').setLevel(logging.CRITICAL)
logging.getLogger('botocore').setLevel(logging.CRITICAL)
class RemoteStorage:
"""
A storage backend abstraction layer
"""
@staticmethod
def get_from_url(remote_url: str, endpoint_url: str):
if remote_url.startswith("s3://"):
import logging
import requests
from six import string_types
import numpy as np
import xarray as xr
import traitlets as tl
# Set up logging
_logger = logging.getLogger(__name__)
# Helper utility for optional imports
from lazy_import import lazy_module
# Optional dependencies
bs4 = lazy_module('bs4')
boto3 = lazy_module('boto3')
# fixing problem with older versions of numpy
if not hasattr(np, 'isnat'):
def isnat(a):
return a.astype(str) == 'None'
np.isnat = isnat
# Internal dependencies
import podpac
from podpac.core.coordinates import Coordinates, union, merge_dims, concat
from podpac.core.data import types as datatype
from podpac.core import authentication
from podpac.core.utils import common_doc
from podpac.core.data.datasource import COMMON_DATA_DOC
from functools import reduce
import traitlets as tl
import pyproj
from podpac.core.utils import common_doc, cached_property, resolve_bbox_order
from podpac.core.data.datasource import DataSource
from podpac.core.interpolation.interpolation import InterpolationMixin, InterpolationTrait
from podpac.core.node import NodeException
from podpac.core.coordinates import Coordinates
from podpac.core.coordinates import UniformCoordinates1d, ArrayCoordinates1d, Coordinates1d, StackedCoordinates
# Optional dependencies
from lazy_import import lazy_module, lazy_class
bs4 = lazy_module("bs4")
lxml = lazy_module("lxml") # used by bs4 so want to check if it's available
owslib_wcs = lazy_module("owslib.wcs")
owslib_util = lazy_module("owslib.util")
rasterio = lazy_module("rasterio")
logger = logging.getLogger(__name__)
class MockWCSClient(tl.HasTraits):
source = tl.Unicode()
version = tl.Enum(["1.0.0"], default_value="1.0.0")
headers = None
cookies = None
auth = tl.Any()
"""
Acquire : (C) Christopher Woods 2019
This module provides the stubs around modules that are not installed
or available on this system, e.g. for license incompatibility reasons
(i.e. while Acquire is Apache licensed, some users may want to use it
in systems where GPL modules are installed)
"""
try:
import lazy_import
lazy_import.logging.disable(lazy_import.logging.DEBUG)
except:
# lazy_import is not available, e.g. because we want the Apache
# licensed version of this code - import the non-lazy wrapper
from ._lazy_import import *
requests = lazy_import.lazy_module("requests")
from .gaisser_hillas_fit import gaisser_hillas
from .gaisser_hillas_fit import gaisser_hillas_log
from .gaisser_hillas_fit import gaisser_hillas_fit
from .running_mean import running_mean
import os
import lazy_import
if "lazy_import" in os.environ:
lazy_import.lazy_module("src.analysis.histogram")
from . import histogram
__build_platform__ = 'n/a'
__all__ = ["gui", "misc", "visual", "core",
"event", "data", "sound", "microphone"]
# for developers the following allows access to the current git sha from
# their repository
if __git_sha__ == 'n/a':
import subprocess
# see if we're in a git repo and fetch from there
try:
thisFileLoc = os.path.split(__file__)[0]
output = subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'],
cwd=thisFileLoc, stderr=subprocess.PIPE)
except Exception:
output = False
if output:
__git_sha__ = output.strip() # remove final linefeed
# update preferences and the user paths
if 'installing' not in locals():
from psychopy.preferences import prefs
for pathName in prefs.general['paths']:
sys.path.append(pathName)
from psychopy.tools.versionchooser import useVersion, ensureMinimal
# lazy imports for things we only want to import when needed
import lazy_import
git = lazy_import.lazy_module("git")
import lazy_import as _lazy_import
from Acquire.Service import call_function as _call_function
from Acquire.Service import pack_arguments as _pack_arguments
from Acquire.Service import unpack_arguments as _unpack_arguments
from Acquire.Service import Service as _Service
from Acquire.ObjectStore import bytes_to_string as _bytes_to_string
from Acquire.ObjectStore import string_to_bytes as _string_to_bytes
from Acquire.Crypto import PrivateKey as _PrivateKey
from Acquire.Crypto import OTP as _OTP
from ._errors import LoginError
_pyotp = _lazy_import.lazy_module("pyotp")
__all__ = ["ServiceWallet"]
class ServiceWallet:
"""This class holds a wallet that can be used to simplify
sending passwords and one-time-password (OTP) codes
to administer one of the Acquire services
This holds a wallet of passwords and (optionally)
OTP secrets that are encrypted using a local keypair
that is unlocked by a password supplied by the user locally.
"""
def __init__(self):
self._wallet_key = None
"""
Algorithm Summary
"""
from __future__ import division, unicode_literals, print_function, absolute_import
from collections import OrderedDict
import inspect
import numpy as np
import xarray as xr
import traitlets as tl
from lazy_import import lazy_module
# Optional dependencies
ne = lazy_module('numexpr')
# Internal dependencies
from podpac.core.coordinates import Coordinates, union
from podpac.core.units import UnitsDataArray
from podpac.core.node import Node
from podpac.core.node import NodeException
from podpac.core.node import COMMON_NODE_DOC
from podpac.core.node import node_eval
from podpac.core.utils import common_doc
COMMON_DOC = COMMON_NODE_DOC.copy()
class Algorithm(Node):
"""Base class for algorithm and computation nodes.
# coding: utf-8
# Copyright (c) 2016, 2018, Oracle and/or its affiliates. All rights reserved.
import lazy_import as _lazy_import
chardet = _lazy_import.lazy_module("oci._vendor.chardet")
httpsig_cffi = _lazy_import.lazy_module("oci._vendor.httpsig_cffi")
jwt = _lazy_import.lazy_module("oci._vendor.jwt")
idna = _lazy_import.lazy_module("oci._vendor.idna")
six = _lazy_import.lazy_module("oci._vendor.six")
urllib3 = _lazy_import.lazy_module("oci._vendor.urllib3")
requests = _lazy_import.lazy_module("oci._vendor.requests")
import pathlib
from abc import ABC, abstractmethod
from typing import Any, Optional, Type, NoReturn, Union
import lazy_import
nx = lazy_import.lazy_module('networkx')
gtall = lazy_import.lazy_module('graph_tool.all')
NODE_T = Type[Any]
EDGE_T = Type[Any]
GRAPH_T = Optional[Any]
class KnowledgeGraphBase(ABC):
LOAD_FORMAT = []
def __init__(self, graph_path: Union[str, pathlib.Path], is_lazy: bool,
*args, **kwargs):
self._graph: GRAPH_T = None
self.graph_path: pathlib.Path = self._check_path(graph_path)
if not is_lazy:
a = self._graph
@property
def graph(self):
if self._graph is None:
self._graph = self.load_graph(self._check_path(self.graph_path))
return self._graph
@graph.setter
from inspect import signature
import math
from typing import Optional
import numpy as np
import sklearn.gaussian_process.kernels
import scipy.optimize
import scipy.spatial.distance
import scipy.special
from lazy_import import lazy_module
kernels = lazy_module('skopt.learning.gaussian_process.kernels')
# This file contains almost no type annotations to simplify comparing it to the original scikit-learn version!
def get_conditional_hyperparameters(X: np.ndarray,
Y: Optional[np.ndarray]) -> np.ndarray:
# Taking care of conditional hyperparameters according to Levesque et al.
X_cond = X <= -1
if Y is not None:
Y_cond = Y <= -1
else:
Y_cond = X <= -1
active = ~((np.expand_dims(X_cond, axis=1) != Y_cond).any(axis=2))
return active
class MagicMixin:
prior = None
General-purpose Algorithm Nodes.
"""
from __future__ import division, unicode_literals, print_function, absolute_import
import sys
import warnings
import numpy as np
import xarray as xr
import traitlets as tl
# Optional dependencies
from lazy_import import lazy_module
ne = lazy_module("numexpr")
from podpac import settings
from podpac import Coordinates
from podpac.core.node import Node, NodeException
from podpac.core.utils import NodeTrait
from podpac.core.algorithm.algorithm import Algorithm
if sys.version_info.major == 2:
class PermissionError(OSError):
pass
class GenericInputs(Algorithm):
"""Base class for Algorithms that accept generic named inputs."""
import lazy_import
lazy_import.lazy_module("configupdater")
lazy_import.lazy_module("cerberus")
lazy_import.lazy_module("mfutil")
opinionated_configparser = lazy_import.lazy_module("opinionated_configparser")
# Copyright (c) Microsoft # Licensed under the MIT License. # Written by Bin Xiao ([email protected]) # ------------------------------------------------------------------------------ from __future__ import absolute_import from __future__ import division from __future__ import print_function from collections import defaultdict from collections import OrderedDict import logging import os import lazy_import COCO = lazy_import.lazy_module("pycocotools.coco") COCOeval = lazy_import.lazy_module("pycocotools.cocoeval") import json import numpy as np from .estimation import EstiamtionDataset from ..ops.nms.nms import oks_nms from ..ops.nms.nms import soft_oks_nms logger = logging.getLogger(__name__) class COCODataset(EstiamtionDataset): ''' "keypoints": { 0: "nose",
def app():
main = lazy_import.lazy_module('main')
yield main.app
importlib.reload(main)
"""Package that handles all data processing."""
from .path import get_path, set_path
from .clear_run import clear_run
from .config import numpy_config
from .config import numpy_particle_label
from .config import numpy_data_layout
from .config import numpy_label_layout
from .caching import prepare_cache
from .caching import cache_dataset
import os
import lazy_import
if "lazy_import" in os.environ:
lazy_import.lazy_module("src.data.raw")
lazy_import.lazy_module("src.data.interim")
lazy_import.lazy_module("src.data.processed")
lazy_import.lazy_module("src.data.external")
lazy_import.lazy_module("src.data.random")
else:
lazy_import.lazy_module("src.data.processed")
lazy_import.lazy_module("src.data.random")
from . import raw
from . import interim
from . import processed
from . import external
from . import random
import os
from termcolor import colored
import lazy_import
np = lazy_import.lazy_module("numpy")
yaml = lazy_import.lazy_module("yaml")
shutil = lazy_import.lazy_module("shutil")
from mantraml.core.cloud.AWS import AWS
from .consts import METADATA_FILE_NAME, SHORT_HASH_INT, DEFAULT_EPOCHS, DEFAULT_BATCH_SIZE
class MantraModel:
"""
By inheriting from this MantraModel class, a deep learning model gets access to Mantra integration, including cloud integration, model monitoring
and more. The core methods below link the model with configuration and training metadata.
"""
# default training arguments
batch_size = 64
epochs = 200
def configure_core_arguments(self, args):
"""
This method adds core training attributes from an argument parser namespace (args) such as the number of epochs, the batch size, and more
Parameters
-----------
args - Argument parser namespace
Containing training arguments such as batch size, number of epochs
Supports access to:
- Landsat 8 on AWS OpenData: https://registry.opendata.aws/landsat-8/
- Sentinel 2
"""
import logging
import datetime
import os
import numpy as np
import traitlets as tl
from lazy_import import lazy_module
satsearch = lazy_module("satsearch")
# Internal dependencies
import podpac
from podpac.compositor import TileCompositor
from podpac.core.data.rasterio_source import RasterioRaw
from podpac.core.units import UnitsDataArray
from podpac.authentication import S3Mixin
from podpac import settings
_logger = logging.getLogger(__name__)
def _get_asset_info(item, name):
""" for forwards/backwards compatibility, convert B0x to/from Bx as needed """
# coding: utf-8
from __future__ import annotations
from abc import ABCMeta, abstractmethod
from dataclasses import dataclass, field
from io import BytesIO
from PIL import Image, ImageOps, ImageEnhance
from lazy_import import lazy_module
from bookworm import typehints as t
from bookworm.image_io import ImageIO
from bookworm.logger import logger
np = lazy_module("numpy")
cv2 = lazy_module("cv2")
cv2_utils = lazy_module("bookworm.ocr_engines.cv2_utils")
log = logger.getChild(__name__)
@dataclass
class ImageProcessingPipeline(metaclass=ABCMeta):
"""Preprocesses images before running OCR."""
images: t.Tuple[ImageIO]
ocr_request: "OcrRequest"
args: dict = field(default_factory=dict)
run_order: t.ClassVar[int] = 0
@abstractmethod
def should_process(self) -> bool:
"""Should this pipeline be applied given the arguments."""
import logging
import datetime
import traitlets as tl
import numpy as np
# Helper utility for optional imports
from lazy_import import lazy_module
# Internal imports
import podpac
from podpac import Coordinates
from podpac.utils import cached_property
intake = lazy_module("intake")
# lazy_module('intake.catalog.local.LocalCatalogEntry')
class IntakeCatalog(podpac.data.DataSource):
"""
Support for Intake Catalogs (https://intake.readthedocs.io/en/latest/index.html)
This primarily supports CSV data sources while we expand for Intake Catalogs.
Parameters
----------
uri : str, required
Intake Catalog uri (local path to catalog yml file, or remote uri)
See https://intake.readthedocs.io/en/latest/catalog.html#local-catalogs
source : str, required
Intake Catalog source
import datetime
import importlib
from unittest.mock import call
import pytest
import freezegun
import lazy_import
mongo_lock = lazy_import.lazy_module('src.mongo_lock')
const = lazy_import.lazy_module('src.const')
@pytest.fixture
def MongoThrottling():
yield mongo_lock.MongoThrottling
importlib.reload(mongo_lock)
@pytest.fixture
def mocked_mongo(mocker):
mongo_lock.MongoClient = mocker.MagicMock()
collection = mocker.MagicMock()
mongo_lock.MongoClient.return_value = {
const.MONGODB_DB_NAME: {
const.MONGODB_COLLECTION_NAME: collection
}
}
yield mongo_lock.MongoClient, collection
class TestMongoThrottlingLock:
from __future__ import division, unicode_literals, print_function, absolute_import
import datetime
import traitlets as tl
import numpy as np
from lazy_import import lazy_module
s3fs = lazy_module("s3fs")
# Internal imports
from podpac.core.data.rasterio_source import RasterioRaw
from podpac.core.authentication import S3Mixin
from podpac.coordinates import Coordinates
from podpac.utils import cached_property, DiskCacheMixin
from podpac.compositor import TileCompositor
BUCKET = "noaa-gfs-pds"
class GFSSourceRaw(DiskCacheMixin, RasterioRaw):
"""Raw GFS data from S3
Attributes
----------
parameter : str
parameter, e.g. 'SOIM'.
level : str
depth, e.g. "0-10 m DPTH"
date : str
from __future__ import division, print_function, absolute_import
import fnmatch
from lazy_import import lazy_module
boto3 = lazy_module("boto3")
import podpac
from podpac.core.settings import settings
from podpac.core.cache.utils import CacheException, CacheWildCard
from podpac.core.cache.file_cache_store import FileCacheStore
class S3CacheStore(FileCacheStore): # pragma: no cover
cache_mode = "s3"
cache_modes = set(["s3", "all"])
_limit_setting = "S3_CACHE_MAX_BYTES"
_delim = "/"
def __init__(self,
s3_bucket=None,
aws_region_name=None,
aws_access_key_id=None,
aws_secret_access_key=None):
"""Initialize a cache that uses a folder on a local disk file system.
Parameters
----------
max_size : None, optional
Maximum allowed size of the cache store in bytes. Defaults to podpac 'S3_CACHE_MAX_BYTES' setting, or no limit if this setting does not exist.
import lazy_import
RSA = lazy_import.lazy_module("Crypto.PublicKey.RSA")
OpenSSL = lazy_import.lazy_module("OpenSSL")
from ..core import ChepyCore, ChepyDecorators
class Publickey(ChepyCore):
def __init__(self, *data):
super().__init__(*data)
def _convert_cert_to_obj(self, cert):
issuer = cert.get_issuer()
subject = cert.get_subject()
pubkey = cert.get_pubkey()
info = {
"version": cert.get_version(),
"serial": cert.get_serial_number(),
"algo": cert.get_signature_algorithm(),
"before": cert.get_notBefore(),
"after": cert.get_notAfter(),
"issuer": {
"C": issuer.C,
"ST": issuer.ST,
"L": issuer.L,
"O": issuer.O,
"OU": issuer.OU,
"CN": issuer.CN,
"email": issuer.emailAddress,
},
"subject": {
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Part of the PsychoPy library
# Copyright (C) 2018 Jonathan Peirce
# Distributed under the terms of the GNU General Public License (GPL).
# DONE: add+commit before push
# DONE: add .gitignore file. Added when opening a repo without one
# DONE: fork+sync doesn't yet fork the project first
# DONE: rather than clone into a folder with files we should init/add/push
#
# TODO: after clone, remember this folder for next file-open call
# TODO: user dlg could/should be local not a browser
# TODO: syncProject() doesn't handle case of a local git pushing to new gitlab
# TODO: if more than one remote then offer options
from psychopy.projects import pavlovia
from .functions import *
from .project import ProjectEditor, syncProject
from ._base import PavloviaMiniBrowser
from . import menu, project, search, toolbar
# lazy import git
import lazy_import
sync = lazy_import.lazy_module("psychopy.app.pavlovia_ui.sync")
