import enum
import typing
from PyQt5.QtCore import QUrl
Url = typing.TypeVar('Url', str, QUrl)
LoadEvent = enum.Enum("LoadEvent", ["FINISHED", "STARTED", "BEFORE_LOAD"])
# Copyright 2009-2022 Joshua Bronson. All Rights Reserved.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""Provide typing-related objects."""
import typing as t
from enum import Enum
KT = t.TypeVar('KT')
VT = t.TypeVar('VT')
IterItems = t.Iterable[t.Tuple[KT, VT]]
MapOrIterItems = t.Union[t.Mapping[KT, VT], IterItems[KT, VT]]
class MissingT(Enum):
"""Sentinel used to represent none/missing when None itself can't be used."""
MISSING = 'MISSING'
def __repr__(self) -> str:
return '<MISSING>'
MISSING = MissingT.MISSING
OKT = t.Union[KT, MissingT] #: optional key type
OVT = t.Union[VT, MissingT] #: optional value type
DT = t.TypeVar('DT') #: for default arguments
ODT = t.Union[DT, MissingT]
import typing
import numpy as np
import xarray as xr
from glotaran.parameter import ParameterGroup
from .irf import IrfMultiGaussian
T_KineticImageModel = typing.TypeVar("glotaran.builtin.models.kinetic_image.KineticImageModel")
def finalize_kinetic_image_result(
model: T_KineticImageModel,
global_indices: typing.List[typing.List[object]],
reduced_clp_labels: typing.Union[typing.Dict[str, typing.List[str]], np.ndarray],
reduced_clps: typing.Union[typing.Dict[str, np.ndarray], np.ndarray],
parameter: ParameterGroup,
data: typing.Dict[str, xr.Dataset],
):
for label in model.dataset:
dataset = data[label]
dataset_descriptor = model.dataset[label].fill(model, parameter)
if not dataset_descriptor.get_k_matrices():
continue
retrieve_species_assocatiated_data(model, dataset, dataset_descriptor, "images")
retrieve_decay_assocatiated_data(model, dataset, dataset_descriptor, "images")
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import abc
import contextlib
import typing
import tbot
from . import connector
from .. import linux, channel
from ..linux import auth
Self = typing.TypeVar("Self", bound="SSHConnector")
class SSHConnector(connector.Connector):
"""
Connect to remote using ``ssh`` by starting off from an existing machine.
An :py:class:`SSHConnector` is different from a
:py:class:`ParamikoConnector` as it requires an existing machine to start
the connection from. This allows jumping via one host to a second.
**Example**:
.. code-block:: python
import tbot
import functools
import sys
import typing
import warnings
import anyio
if sys.version_info >= (3, 10): # pragma: no cover
from typing import ParamSpec
else: # pragma: no cover
from typing_extensions import ParamSpec
T = typing.TypeVar("T")
P = ParamSpec("P")
async def run_until_first_complete(
*args: typing.Tuple[typing.Callable, dict]) -> None:
warnings.warn(
"run_until_first_complete is deprecated "
"and will be removed in a future version.",
DeprecationWarning,
)
async with anyio.create_task_group() as task_group:
async def run(func: typing.Callable[[], typing.Coroutine]) -> None:
await func()
task_group.cancel_scope.cancel()
for func, kwargs in args:
import cg_json
from cg_dt_utils import DatetimeWithTimezone
from cg_sqlalchemy_helpers import hybrid_property, hybrid_expression
from cg_sqlalchemy_helpers.types import DbColumn, ColumnProxy
from . import Base, db
from . import work as work_models
from .. import helpers
from ..registry import rubric_row_types
from ..exceptions import APICodes, APIException
if t.TYPE_CHECKING: # pragma: no cover
# pylint: disable=unused-import
from . import auto_test as auto_test_models
T = t.TypeVar('T', bound=t.Type['RubricRowBase'])
_ALL_RUBRIC_ROW_TYPES = sorted(['normal', 'continuous'])
def _register(cls: T) -> T:
name = cls.__mapper_args__['polymorphic_identity']
assert isinstance(name, str)
assert name in _ALL_RUBRIC_ROW_TYPES
assert rubric_row_types.get(name) is None
rubric_row_types.register(name)(cls)
return cls
class WorkRubricItem(helpers.NotEqualMixin, Base):
return set(self.__field_factory__(element) for element in value)
class VectorField(_SequenceField):
__type__ = 'vector'
def cast(self, value):
return list(self.__field_factory__(element) for element in value)
###############################################################################
# Bro logging types
# internal typings
_bro_string = typing.TypeVar(
'bro_string', bound=StringField) # _bro_string.__bound__ == StringField
_bro_port = typing.TypeVar('bro_port', bound=PortField)
_bro_enum = typing.TypeVar('bro_enum', bound=EnumField)
_bro_interval = typing.TypeVar('bro_interval', bound=IntervalField)
_bro_addr = typing.TypeVar('bro_addr', bound=AddrField)
_bro_subnet = typing.TypeVar('bro_subnet', bound=SubnetField)
_bro_int = typing.TypeVar('bro_int', bound=IntField)
_bro_count = typing.TypeVar('bro_count', bound=CountField)
_bro_time = typing.TypeVar('bro_time', bound=TimeField)
_bro_double = typing.TypeVar('bro_double', bound=DoubleField)
_bro_bool = typing.TypeVar('bro_bool', bound=BoolField)
_bro_record = typing.TypeVar('bro_record', bound=RecordField)
_bro_set = typing.TypeVar('bro_set', bound=SetField)
_bro_vector = typing.TypeVar('bro_vector', bound=VectorField)
_bro_type = typing.TypeVar(
'bro_type', # _bro_type.__constraints__ == (...)
"""Utility definitions for internal use. Not part of the public API."""
import functools
import itertools
import typing as t
import weakref
if t.TYPE_CHECKING:
from nnf import NNF # noqa: F401
Name = t.Hashable
Model = t.Dict[Name, bool]
T = t.TypeVar("T")
U = t.TypeVar("U")
T_NNF = t.TypeVar("T_NNF", bound="NNF")
U_NNF = t.TypeVar("U_NNF", bound="NNF")
T_NNF_co = t.TypeVar("T_NNF_co", bound="NNF", covariant=True)
# Bottom type with no values
# This works in mypy but not pytype
# t.Union[()] works too but not at runtime
# NoReturn doesn't exist in some Python releases, hence the guard
if t.TYPE_CHECKING:
Bottom = t.NoReturn
else:
Bottom = None
memoize = t.cast(t.Callable[[T], T], functools.lru_cache(maxsize=None))
import logging
import threading
import typing
from functools import wraps
from os import environ
from pkg_resources import iter_entry_points
from opentelemetry.context.context import Context, _RuntimeContext
from opentelemetry.environment_variables import OTEL_PYTHON_CONTEXT
logger = logging.getLogger(__name__)
_RUNTIME_CONTEXT = None # type: typing.Optional[_RuntimeContext]
_RUNTIME_CONTEXT_LOCK = threading.Lock()
_F = typing.TypeVar("_F", bound=typing.Callable[..., typing.Any])
def _load_runtime_context(func: _F) -> _F:
"""A decorator used to initialize the global RuntimeContext
Returns:
A wrapper of the decorated method.
"""
@wraps(func) # type: ignore
def wrapper(
*args: typing.Tuple[typing.Any, typing.Any],
**kwargs: typing.Dict[typing.Any, typing.Any]
) -> typing.Optional[typing.Any]:
global _RUNTIME_CONTEXT # pylint: disable=global-statement
import abc
import json
import logging
import pathlib
import typing
__all__ = ["ConfigBase"]
USER_CONFIG_PATH = pathlib.Path.home() / ".config" / "python-livy.json"
CONFIG_LOAD_ORDER = [
pathlib.Path(__file__).resolve().parent.parent /
"default-configuration.json",
USER_CONFIG_PATH,
]
_T = typing.TypeVar("_T")
class ConfigBase(abc.ABC):
"""Base class for setting configurations. Inspired by
`pydantic <https://pydantic-docs.helpmanual.io/>`_.
Thought this, please just treat this class as a simplified version of
:py:mod:`dataclasses`.
This class does lake of validation, it is created to perform dictionary
updating liked config merge and json transform.
To use this base, inherit this class and use
`type annotation <https://docs.python.org/3/glossary.html#term-variable-annotation>`_
to define the fields. Fields would be automatically created during
:py:meth:`__init__`.
if g.startswith('setup/always/'))))
self.needs_target = tuple(
sorted(
set(
g.split('/')[2] for g in groups
if g.startswith('needs/target/'))))
@staticmethod
def _remove_group(groups, group):
return [
g for g in groups if g != group and not g.startswith('%s/' % group)
]
class TargetPatternsNotMatched(ApplicationError):
"""One or more targets were not matched when a match was required."""
def __init__(self, patterns): # type: (t.Set[str]) -> None
self.patterns = sorted(patterns)
if len(patterns) > 1:
message = 'Target patterns not matched:\n%s' % '\n'.join(
self.patterns)
else:
message = 'Target pattern not matched: %s' % self.patterns[0]
super().__init__(message)
TCompletionTarget = t.TypeVar('TCompletionTarget', bound=CompletionTarget)
TIntegrationTarget = t.TypeVar('TIntegrationTarget', bound=IntegrationTarget)
def open_in_browser(self, file_type: str = ".html") -> bool:
fd, path = tempfile.mkstemp(file_type)
if self._body:
os.write(fd, self._body)
os.close(fd)
return webbrowser.open("file://" + path)
class CustomNoneType:
"""Different with "None" obj ("null" in json)
"""
pass
Item = typing.TypeVar("Item")
def set_value_to_item(item: Item, key: str, value: typing.Any):
if isinstance(item, MutableMapping):
item[key] = value
else:
setattr(item, key, value)
def get_value_from_item(item: Item, key: str):
try:
if isinstance(item, MutableMapping):
return item[key]
else:
return getattr(item, key)
run_support_container,
)
from .connections import (
Connection,
DockerConnection,
LocalConnection,
SshConnection,
)
from .become import (
Su,
Sudo,
)
TControllerHostConfig = t.TypeVar('TControllerHostConfig',
bound=ControllerHostConfig)
THostConfig = t.TypeVar('THostConfig', bound=HostConfig)
TPosixConfig = t.TypeVar('TPosixConfig', bound=PosixConfig)
TRemoteConfig = t.TypeVar('TRemoteConfig', bound=RemoteConfig)
@dataclasses.dataclass(frozen=True)
class Inventory:
"""Simple representation of an Ansible inventory."""
host_groups: t.Dict[str, t.Dict[str, t.Dict[str, str]]]
extra_groups: t.Optional[t.Dict[str, t.List[str]]] = None
@staticmethod
def create_single_host(
name, variables): # type: (str, t.Dict[str, str]) -> Inventory
"""Return an inventory instance created from the given hostname and variables."""
configure_pypi_proxy, )
from ...inventory import (
create_controller_inventory,
create_windows_inventory,
create_network_inventory,
create_posix_inventory,
)
from .filters import (
get_target_filter, )
from .coverage import (
CoverageManager, )
THostProfile = t.TypeVar('THostProfile', bound=HostProfile)
def generate_dependency_map(
integration_targets: list[IntegrationTarget]
) -> dict[str, set[IntegrationTarget]]:
"""Analyze the given list of integration test targets and return a dictionary expressing target names and the targets on which they depend."""
targets_dict = dict(
(target.name, target) for target in integration_targets)
target_dependencies = analyze_integration_target_dependencies(
integration_targets)
dependency_map: dict[str, set[IntegrationTarget]] = {}
invalid_targets = set()
for dependency, dependents in target_dependencies.items():
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import typing
import getpass
from tbot.machine import linux
from tbot.machine import channel
from .machine import LabHost
LLH = typing.TypeVar("LLH", bound="LocalLabHost")
class LocalLabHost(LabHost):
"""
LabHost on the host TBot is running on.
Makes use of the :class:`~tbot.machine.channel.SubprocessChannel`.
``LocalLabHost`` can be instanciated as is, but if you need customization,
you should subclass it.
"""
name = "local"
@property
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
"""A base class for implementing extraction handlers."""
import abc
import typing
import attr
_HandlerBaseType = typing.TypeVar("T", bound="Repository")
@attr.s
class HandlerBase(object):
"""Handle extracting packages from build logs."""
# Ignore PycodestyleBear (E701)
handlers: typing.ClassVar[typing.List[_HandlerBaseType]] = []
@classmethod
def register(cls, handler_instance: _HandlerBaseType) -> None:
"""Register a handler instance to be used."""
cls.handlers.append(handler_instance)
@classmethod
# Author: Pavel Kirienko <*****@*****.**>
#
from __future__ import annotations
import abc
import sys
import typing
import struct
import base64
import numpy
# We must use uint8 instead of ubyte because uint8 is platform-invariant whereas (u)byte is platform-dependent.
_Byte = numpy.uint8
# noinspection PyShadowingBuiltins
_T = typing.TypeVar('_T')
_PrimitiveType = typing.Union[typing.Type[numpy.integer], typing.Type[numpy.inexact]]
class Deserializer(abc.ABC):
"""
The deserializer class is used for deconstruction of serialized representations of DSDL objects into Python objects.
It implements the implicit zero extension rule as described in the Specification.
"""
class FormatError(ValueError):
"""
This exception class is used when an auto-generated deserialization routine is supplied with invalid input data;
in other words, input that is not a valid serialized representation of its data type.
Deserialization logic (auto-generated or manually written) may use this exception type.
from .. import coredata
from .. import mlog
from .. import mesonlib
from ..linkers import LinkerEnvVarsMixin
from ..mesonlib import (EnvironmentException, MachineChoice, MesonException,
Popen_safe, split_args)
from ..envconfig import (Properties, get_env_var)
from ..arglist import CompilerArgs
if T.TYPE_CHECKING:
from ..coredata import OptionDictType
from ..envconfig import MachineInfo
from ..environment import Environment
from ..linkers import DynamicLinker # noqa: F401
CompilerType = T.TypeVar('CompilerType', bound=Compiler)
"""This file contains the data files of all compilers Meson knows
about. To support a new compiler, add its information below.
Also add corresponding autodetection code in environment.py."""
header_suffixes = ('h', 'hh', 'hpp', 'hxx', 'H', 'ipp', 'moc', 'vapi', 'di')
obj_suffixes = ('o', 'obj', 'res')
lib_suffixes = ('a', 'lib', 'dll', 'dll.a', 'dylib', 'so')
# Mapping of language to suffixes of files that should always be in that language
# This means we can't include .h headers here since they could be C, C++, ObjC, etc.
lang_suffixes = {
'c': ('c', ),
'cpp': ('cpp', 'cc', 'cxx', 'c++', 'hh', 'hpp', 'ipp', 'hxx', 'ino'),
'cuda': ('cu', ),
# f90, f95, f03, f08 are for free-form fortran ('f90' recommended)
# f, for, ftn, fpp are for fixed-form fortran ('f' or 'for' recommended)
"dotted_getattr",
"qualified_name",
"name_lookup",
"type_lookup",
"type_lookup_",
"asmodule",
"check_type",
"check_arg",
"check_subclass",
"unique",
"assocv",
"assocf",
]
if typing.TYPE_CHECKING:
H = typing.TypeVar("H", bound=typing.Hashable)
C = typing.TypeVar("C")
K = typing.TypeVar("K")
V = typing.TypeVar("V")
KV = Tuple[K, V]
def dotted_getattr(obj, name):
# type: (Any, str) -> Any
"""
`getattr` like function accepting a dotted name for attribute lookup.
"""
return reduce(getattr, name.split("."), obj)
def qualified_name(obj):
if self.count() == 0:
return
elif self.count() == 1:
self.__pages[0].button.position = QStyleOptionToolBox.OnlyOneTab
else:
self.__pages[0].button.position = QStyleOptionToolBox.Beginning
self.__pages[-1].button.position = QStyleOptionToolBox.End
for p in self.__pages[1:-1]:
p.button.position = QStyleOptionToolBox.Middle
for p in self.__pages:
p.button.update()
if typing.TYPE_CHECKING:
A = typing.TypeVar("A")
B = typing.TypeVar("B")
C = typing.TypeVar("C")
def identity(arg):
return arg
def find(iterable, what, key=identity, predicate=eq):
# type: (Iterable[A], B, Callable[[A], C], Callable[[C, B], bool]) -> A
"""
find(iterable, what, [key=None, [predicate=operator.eq]])
"""
for item in iterable:
item_key = key(item)
# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import typing as tp
from .core import Dict as Dict # Dict needs to be implemented in core since it's used in the base class
from . import core
Ins = tp.TypeVar("Ins", bound="Instrumentation")
ArgsKwargs = tp.Tuple[tp.Tuple[tp.Any, ...], tp.Dict[str, tp.Any]]
class Tuple(Dict):
"""Tuple-valued parameter. This Parameter can contain other Parameters,
its value is tuple which values are either directly the provided values
if they are not Parameter instances, or the value of those Parameters.
It also implements a getter to access the Parameters directly if need be.
Parameters
----------
**parameters: Any
the objects or Parameter which will provide values for the tuple
Note
----
This is the base structure for all container Parameters, and it is
used to hold the subparameters for all Parameter classes.
"""
ret = method(state, args, kwargs)
if isinstance(ret, ModuleReturnValue):
self.interpreter.process_new_values(ret.new_objects)
ret = ret.return_value
return ret
class MutableModuleObjectHolder(ModuleObjectHolder, MutableInterpreterObject):
def __deepcopy__(self, memo: T.Dict[int,
T.Any]) -> 'MutableModuleObjectHolder':
# Deepcopy only held object, not interpreter
modobj = copy.deepcopy(self.held_object, memo)
return MutableModuleObjectHolder(modobj, self.interpreter)
_BuildTarget = T.TypeVar('_BuildTarget',
bound=T.Union[build.BuildTarget, build.BothLibraries])
class BuildTargetHolder(ObjectHolder[_BuildTarget]):
def __init__(self, target: _BuildTarget, interp: 'Interpreter'):
super().__init__(target, interp)
self.methods.update({
'extract_objects':
self.extract_objects_method,
'extract_all_objects':
self.extract_all_objects_method,
'name':
self.name_method,
'get_id':
self.get_id_method,
'outdir':
import pprint
import six
import typing
from seqann import util
T = typing.TypeVar('T')
class Model(object):
# swaggerTypes: The key is attribute name and the
# value is attribute type.
swagger_types = {}
# attributeMap: The key is attribute name and the
# value is json key in definition.
attribute_map = {}
@classmethod
def from_dict(cls: typing.Type[T], dikt) -> T:
"""Returns the dict as a model"""
return util.deserialize_model(dikt, cls)
def to_dict(self):
"""Returns the model properties as a dict
:rtype: dict
"""
result = {}
setattr(filter_func, CONTEXT_FILTER_ATTRIBUTE_NAME, True)
return filter_func
def template_volatile_filter(
filter_func: typing.Callable) -> typing.Callable[..., str]:
"""
Decorator for marking a filter as volatile therefore disabling optimizations for the
frame it appears within.
An opaque object will be passed to the filter as the first argument.
"""
setattr(filter_func, CONTEXT_FILTER_ATTRIBUTE_NAME, True)
return filter_func
LanguageFilterReturnType = typing.TypeVar("LanguageFilterReturnType")
class GenericTemplateLanguageFilter(typing.Generic[LanguageFilterReturnType]):
"""
Decorator for marking template filters that take a :class:`nunavut.lang.Language` object
as the first argument with a generic return type.
"""
def __init__(self, language_name_or_module: str):
self._language_name_or_module = language_name_or_module
def __call__(
self, filter_func: typing.Callable[..., LanguageFilterReturnType]
) -> typing.Callable[..., LanguageFilterReturnType]:
self._annotate_function(filter_func)
return filter_func
def test_nonfunction_task_and_df_input():
@reference_task(
project="flytesnacks",
domain="development",
name="ref_t1",
version="fast56d8ce2e373baf011f4d3532e45f0a9b",
)
def ref_t1(
dataframe: pd.DataFrame,
imputation_method: str = "median",
) -> pd.DataFrame:
...
@reference_task(
project="flytesnacks",
domain="development",
name="ref_t2",
version="aedbd6fe44051c171fd966c280c5c3036f658831",
)
def ref_t2(
dataframe: pd.DataFrame,
split_mask: int,
num_features: int,
) -> pd.DataFrame:
...
wb = Workflow(name="core.feature_engineering.workflow.fe_wf")
wb.add_workflow_input("sqlite_archive",
FlyteFile[typing.TypeVar("sqlite")])
sql_task = SQLite3Task(
name="dummy.sqlite.task",
query_template="select * from data",
inputs=kwtypes(),
output_schema_type=FlyteSchema,
task_config=SQLite3Config(
uri="https://sample/data",
compressed=True,
),
)
node_sql = wb.add_task(sql_task)
node_t1 = wb.add_task(ref_t1,
dataframe=node_sql.outputs["results"],
imputation_method="mean")
node_t2 = wb.add_task(
ref_t2,
dataframe=node_t1.outputs["o0"],
split_mask=24,
num_features=15,
)
wb.add_workflow_output("output_from_t3",
node_t2.outputs["o0"],
python_type=pd.DataFrame)
wf_spec = get_serializable(OrderedDict(), serialization_settings, wb)
assert len(wf_spec.template.nodes) == 3
assert len(wf_spec.template.interface.inputs) == 1
assert wf_spec.template.interface.inputs[
"sqlite_archive"].type.blob is not None
assert len(wf_spec.template.interface.outputs) == 1
assert wf_spec.template.interface.outputs[
"output_from_t3"].type.schema is not None
class LanguageEnvironment:
"""
Data structure defining stuff contributed to a template environment for a given :class:`Language`.
"""
TEST_NAME_PREFIX = "is_"
FILTER_NAME_PREFIX = "filter_"
USES_QUERY_PREFIX = "uses_"
@classmethod
def is_test_name(cls, callable_name: typing.Optional[str]) -> bool:
return (callable_name is not None
and len(callable_name) >= len(cls.TEST_NAME_PREFIX)
and callable_name.startswith(cls.TEST_NAME_PREFIX))
@classmethod
def is_filter_name(cls, callable_name: typing.Optional[str]) -> bool:
return (callable_name is not None
and len(callable_name) >= len(cls.FILTER_NAME_PREFIX)
and callable_name.startswith(cls.FILTER_NAME_PREFIX))
@classmethod
def is_uses_query_name(cls, callable_name: typing.Optional[str]) -> bool:
return (callable_name is not None
and len(callable_name) >= len(cls.USES_QUERY_PREFIX)
and callable_name.startswith(cls.USES_QUERY_PREFIX))
LanguageListT = typing.TypeVar("LanguageListT",
typing.AbstractSet[nunavut.lang.Language],
typing.ValuesView[nunavut.lang.Language])
def __init__(self, language_name: str) -> None:
self._language_name = language_name
self._tests = dict() # type: typing.Dict[str, typing.Callable]
self._filters = dict() # type: typing.Dict[str, typing.Callable]
self._uses_queries = dict() # type: typing.Dict[str, typing.Callable]
@property
def language_name(self) -> str:
return self._language_name
@property
def tests(self) -> typing.Mapping[str, typing.Callable]:
return self._tests
@property
def filters(self) -> typing.Mapping[str, typing.Callable]:
return self._filters
@property
def uses_queries(self) -> typing.Mapping[str, typing.Callable]:
return self._uses_queries
@classmethod
def _parse_callable_name(
cls,
callable: typing.Callable,
callable_name: typing.Optional[str] = None
) -> typing.Tuple[typing.Optional[str], str]:
if callable_name is None:
if isinstance(callable, functools.partial):
callable_name = callable.func.__name__
else:
callable_name = callable.__name__
if cls.is_test_name(callable_name):
prefix = cls.TEST_NAME_PREFIX # type: typing.Optional[str]
method_name = callable_name[len(cls.TEST_NAME_PREFIX):]
elif cls.is_filter_name(callable_name):
prefix = cls.FILTER_NAME_PREFIX
method_name = callable_name[len(cls.FILTER_NAME_PREFIX):]
elif cls.is_uses_query_name(callable_name):
prefix = cls.USES_QUERY_PREFIX
method_name = callable_name[len(cls.USES_QUERY_PREFIX):]
else:
prefix = None
method_name = callable_name
return (prefix, method_name)
@classmethod
def handle_conventional_methods(
cls,
callable: typing.Callable,
callable_name: typing.Optional[str] = None,
supported_languages: typing.Optional[LanguageListT] = None,
) -> typing.Tuple[typing.Optional[str], str, typing.Callable]:
"""
Processes method objects that utilize the nunavut convention of ``is_``, ``filter_``, or ``uses_`` prefixes.
Also wraps the method in a partial if it requested the language as the first argument.
:param str callable_name: If provided this is the name used to process the callable otherwise
the ``__name__`` property is used from the callable itself.
:return: A 3-tuple with the prefix, method name without prefix, and the method or partial. If the first
element is ``None`` then the callable was not a conventional method.
"""
prefix, method_name = cls._parse_callable_name(callable, callable_name)
resolved_callable = None # type: typing.Optional[typing.Callable]
if hasattr(callable, LANGUAGE_FILTER_ATTRIBUTE_NAME):
callable_language_name = getattr(callable,
LANGUAGE_FILTER_ATTRIBUTE_NAME)
if supported_languages is not None:
for language in supported_languages:
if language.get_templates_package_name(
) == callable_language_name:
resolved_callable = functools.partial(
callable, language)
break
if resolved_callable is None:
raise RuntimeWarning(
'Language callable "{}", required an unsupported language({})'
.format(method_name, callable_language_name))
else:
resolved_callable = callable
return (prefix, method_name, resolved_callable)
# +---------------------------------------------------------------------------------------------------------------+
# | DATA MODEL
# +---------------------------------------------------------------------------------------------------------------+
def __getitem__(self, key: str) -> typing.Dict[str, typing.Callable]:
key_case_insensitive = key.lower()
if key_case_insensitive.startswith("is"):
return self._tests
elif key_case_insensitive.startswith("filter"):
return self._filters
elif key_case_insensitive.startswith("uses"):
return self._uses_queries
else:
raise KeyError(
"Key {} was not supported by this object.".format(key))
# +---------------------------------------------------------------------------------------------------------------+
# | FACTORY
# +---------------------------------------------------------------------------------------------------------------+
@classmethod
def find_all_conventional_methods_in_language_module(
cls, language: nunavut.lang.Language, all_languages: LanguageListT,
language_module: "types.ModuleType") -> "LanguageEnvironment":
results = LanguageEnvironment(language.name)
callables = inspect.getmembers(language_module, inspect.isfunction)
for function_tuple in callables:
result = cls.handle_conventional_methods(
function_tuple[1], supported_languages=all_languages)
if result[0] is not None:
results[result[0]][result[1]] = result[2]
return results
"""Defining abstract loss classes for actor critic models."""
import abc
import typing
from typing import Dict, Tuple
import torch
from allenact.algorithms.onpolicy_sync.policy import ObservationType
from allenact.base_abstractions.misc import Loss, Memory
ModelType = typing.TypeVar("ModelType")
class AbstractOffPolicyLoss(typing.Generic[ModelType], Loss):
"""Abstract class representing an off-policy loss function used to train a
model."""
@abc.abstractmethod
def loss( # type: ignore
self,
model: ModelType,
batch: ObservationType,
memory: Memory,
*args,
**kwargs,
) -> Tuple[torch.FloatTensor, Dict[str, float], Memory, int]:
"""Computes the loss.
Loss after processing a batch of data with (part of) a model (possibly with memory).
# Parameters
from bravado_core.validate import validate_schema_object
from msgpack import unpackb
from bravado.config import bravado_config_from_config_dict
from bravado.config import BravadoConfig
from bravado.config import CONFIG_DEFAULTS
from bravado.config import RequestConfig
from bravado.exception import BravadoConnectionError
from bravado.exception import BravadoTimeoutError
from bravado.exception import ForcedFallbackResultError
from bravado.exception import HTTPServerError
from bravado.exception import make_http_exception
from bravado.response import BravadoResponse
FuncType = typing.Callable[..., typing.Any]
F = typing.TypeVar('F', bound=FuncType)
T = typing.TypeVar('T')
log = logging.getLogger(__name__)
FALLBACK_EXCEPTIONS = (
BravadoTimeoutError,
BravadoConnectionError,
HTTPServerError,
)
class _SENTINEL(object):
pass
SENTINEL = _SENTINEL()
Returns:
`True` if component can handle specific language, `False` otherwise.
"""
# if language_list is set to `None` it means: support all languages
if language is None or (cls.supported_language_list is None
and cls.not_supported_language_list is None):
return True
language_list = cls.supported_language_list or []
not_supported_language_list = cls.not_supported_language_list or []
return language in language_list or language not in not_supported_language_list
C = typing.TypeVar("C", bound=Component)
class ComponentBuilder:
"""Creates trainers and interpreters based on configurations.
Caches components for reuse.
"""
def __init__(self, use_cache: bool = True) -> None:
self.use_cache = use_cache
# Reuse nlp and featurizers where possible to save memory,
# every component that implements a cache-key will be cached
self.component_cache = {}
def __get_cached_component(
self, component_meta: Dict[Text, Any], model_metadata: "Metadata"
from __future__ import annotations
import weakref
import typing as t
import dataclasses
T = t.TypeVar("T")
class Undefined:
def __repr__(self) -> str:
return "<UNDEFINED>"
UNDEFINED = Undefined()
POOL = weakref.WeakKeyDictionary()
class Prop(t.Generic[T]):
def __init__(self, *, name: t.Optional[str] = None) -> None:
self.name = name
self.store_name = "_"
def __repr__(self) -> str:
cls = self.__class__
return f"<{cls.__module__}.{cls.__name__} object at {hex(id(self))}, field={self.name!r}>"
def __get__( # noqa:F811
self,
ob: t.Optional[t.Any],
typ: t.Optional[t.Type[t.Any]] = None) -> t.Union[T, Undefined]:
