import functools import json from itertools import chain from time import time as utc from typing import Dict, List, Callable, Tuple, Any, Optional, Union import requests from mypy_extensions import DefaultNamedArg from .config_logging import log log = log.getChild('TwitchAPI') RF = Callable[[ str, DefaultNamedArg(Optional[Dict], 'params'), DefaultNamedArg(Optional[Dict[str, str]], 'headers') ], requests.Response] def timed_cache(func: Callable[..., Dict]) -> Callable[..., Dict]: cache: Dict[str, Tuple[Dict, int]] = {} @functools.wraps(func) def wrapper(*args: Any, **kwargs: Any) -> Dict: _, video_id, *_ = args if video_id not in cache or cache[video_id][1] < utc(): token = func(*args, **kwargs) cache[video_id] = token, json.loads(token['token'])['expires'] return cache[video_id][0]
elif byteorder == 'little': bytes = bytes[::-1] else: raise ValueError("byteorder must be either 'little' or 'big'") hex_str = codecs.encode(bytes, 'hex') # type: ignore return int(hex_str, 16) # Make this cast since Python 2 doesn't have syntax for default # named arguments. Hence, must cast so Mypy thinks it matches the # original function. int_from_bytes = cast( Callable[[ Arg(Sequence[int], 'bytes'), Arg(str, 'byteorder'), DefaultNamedArg(bool, 'signed') ], int], _int_from_bytes) def re_compile_full(pattern, flags=0): # type: (AnyStr, int) -> Pattern """ Create compiled regular expression such that it matches the entire string. Calling re.match on the output of this function is equivalent to calling re.fullmatch on its input. This is needed to support Python 2. (On Python 3, we would just call re.fullmatch.) Kudos: https://stackoverflow.com/a/30212799 :param pattern: The pattern to compile. :param flags: Regular expression flags. Refer to Python
from faust.exceptions import KeyDecodeError, ValueDecodeError from faust.types.app import AppT from faust.types.codecs import CodecArg from faust.types.core import K, OpenHeadersArg, V from faust.types.events import EventT from faust.types.models import ModelArg from faust.types.serializers import KT, VT, SchemaT from faust.types.tuples import Message __all__ = ["Schema"] if typing.TYPE_CHECKING: # pragma: no cover from mypy_extensions import DefaultNamedArg DecodeFunction = Callable[ [Message, DefaultNamedArg(bool, "propagate")], # noqa: F821 Awaitable[EventT], ] else: DecodeFunction = Callable[..., Awaitable[EventT]] OnKeyDecodeErrorFun = Callable[[Exception, Message], Awaitable[None]] OnValueDecodeErrorFun = Callable[[Exception, Message], Awaitable[None]] async def _noop_decode_error(exc: Exception, message: Message) -> None: ... class Schema(SchemaT): def __init__( self,
for A, B in zip(training_metrics["A"], training_metrics["B"]): assert_equivalent_metrics(A, B) assert len(validation_metrics["A"]) == len(validation_metrics["B"]) for A, B in zip(validation_metrics["A"], validation_metrics["B"]): assert_equivalent_metrics(A, B) return ( (training_metrics["A"], validation_metrics["A"]), (training_metrics["B"], validation_metrics["B"]), ) RestorableMakeControllerFn = Callable[ [workload.Stream, DefaultNamedArg(Optional[pathlib.Path], "load_path")], # noqa: F821 det.TrialController, ] def checkpointing_and_restoring_test( make_trial_controller_fn: RestorableMakeControllerFn, tmp_path: Path ) -> Tuple[Sequence[Dict[str, Any]], Sequence[Dict[str, Any]]]: """ Tests if a trial controller of any framework can checkpoint and restore from that checkpoint without state changes. This test runs two trials. 1) Trial A runs for one steps of 100 batches, checkpoints itself, and restores from that checkpoint. 2) Trial B runs for two steps of 100 batches.
assert len(training_metrics["A"]) == len(training_metrics["B"]) for A, B in zip(training_metrics["A"], training_metrics["B"]): assert_equivalent_metrics(A, B) assert len(validation_metrics["A"]) == len(validation_metrics["B"]) for A, B in zip(validation_metrics["A"], validation_metrics["B"]): assert_equivalent_metrics(A, B) return ( (training_metrics["A"], validation_metrics["A"]), (training_metrics["B"], validation_metrics["B"]), ) RestorableMakeControllerFn = Callable[ [workload.Stream, DefaultNamedArg(Optional[pathlib.Path], "load_path")], # noqa: F821 det.TrialController, ] def checkpointing_and_restoring_test( make_trial_controller_fn: RestorableMakeControllerFn, tmp_path: Path ) -> Tuple[Sequence[Dict[str, Any]], Sequence[Dict[str, Any]]]: """ Tests if a trial controller of any framework can checkpoint and restore from that checkpoint without state changes. This test runs two trials. 1) Trial A runs for one steps of 100 batches, checkpoints itself, and restores from that checkpoint. 2) Trial B runs for two steps of 100 batches.
RecordMetadata, SchemaT, TP, V, ) from .types.core import OpenHeadersArg from .types.topics import ChannelT, TopicT from .types.transports import ProducerT if typing.TYPE_CHECKING: # pragma: no cover from mypy_extensions import DefaultNamedArg from .app import App DecodeFunction = Callable[ [Message, DefaultNamedArg(bool, 'propagate')], Awaitable[EventT]] else: class App: ... # noqa DecodeFunction = Callable[..., Awaitable[EventT]] __all__ = ['Topic'] logger = get_logger(__name__) class Topic(Channel, TopicT): """Define new topic description.
# pylint: disable=cyclic-import import bs4 # noqa import gluetool # noqa import gluetool.sentry # noqa # Type definitions # pylint: disable=invalid-name ExceptionInfoType = Union[ Tuple[Optional[type], Optional[BaseException], Optional[TracebackType]], # returned by sys.exc_info() Tuple[None, None, None] ] LoggingFunctionType = Callable[ [ Arg(str), DefaultNamedArg(ExceptionInfoType, 'exc_info'), DefaultNamedArg(Dict[str, Any], 'extra'), DefaultNamedArg(bool, 'sentry') ], None ] ContextInfoType = Tuple[int, Any] BLOB_HEADER = '---v---v---v---v---v---' BLOB_FOOTER = '---^---^---^---^---^---' # Default log level is logging.INFO or logging.DEBUG if GLUETOOL_DEBUG environment variable is set DEFAULT_LOG_LEVEL = logging.DEBUG if os.getenv('GLUETOOL_DEBUG') else logging.INFO
MessageSentCallback, ModelArg, PendingMessage, RecordMetadata, TP, V, ) from .types.topics import ChannelT, TopicT from .types.transports import ProducerT if typing.TYPE_CHECKING: # pragma: no cover from mypy_extensions import DefaultNamedArg from .app import App DecodeFunction = Callable[[Message, DefaultNamedArg(bool, 'propagate')], Awaitable[EventT]] else: class App: ... # noqa DecodeFunction = Callable[..., Awaitable[EventT]] __all__ = ['Topic'] logger = get_logger(__name__) class Topic(Channel, TopicT): """Define new topic description.
from paasta_tools.utils import InvalidInstanceConfig from paasta_tools.utils import InvalidJobNameError from paasta_tools.utils import load_system_paasta_config from paasta_tools.utils import NoConfigurationForServiceError from paasta_tools.utils import NoDeploymentsAvailable from paasta_tools.utils import NoDockerImageError from paasta_tools.utils import SPACER from paasta_tools.utils import SystemPaastaConfig # Marathon REST API: # https://github.com/mesosphere/marathon/blob/master/REST.md#post-v2apps log = logging.getLogger(__name__) LogDeployError = Callable[ [Arg(str, 'errormsg'), DefaultNamedArg(str, 'level')], None] LogBounceAction = Callable[ [Arg(str, 'line'), DefaultNamedArg(str, 'level')], None] def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description='Creates marathon jobs.') parser.add_argument( 'service_instance_list', nargs='+', help="The list of marathon service instances to create or update", metavar="SERVICE%sINSTANCE" % SPACER, ) parser.add_argument( '-d',
segment.decode("utf-8", errors='replace') for segment in request.prepath), request.method, enctype, "utf-8", fieldValues.prevalidationValues, fieldValues.validationErrors, ) return Element(TagLoader(renderable)) _requirerFunctionWithForm = Any _routeCallable = Any _routeDecorator = Callable[ [_routeCallable, DefaultNamedArg(Any, '__session__')], _routeCallable] _validationFailureHandler = Callable[ [Optional[object], IRequest, 'Form', Dict[str, str]], Element] validationFailureHandlerAttribute = "__kleinFormValidationFailureHandlers__" class IProtoForm(Interface): """ Marker interface for L{ProtoForm}. """ class IForm(Interface): """ Marker interface for form attached to dependency injection components.
passback_grades = _passback_grades_1.delay # pylint: disable=invalid-name lint_instances = _lint_instances_1.delay # pylint: disable=invalid-name add = _add_1.delay # pylint: disable=invalid-name send_done_mail = _send_done_mail_1.delay # pylint: disable=invalid-name send_grader_status_mail = _send_grader_status_mail_1.delay # pylint: disable=invalid-name run_plagiarism_control = _run_plagiarism_control_1.delay # pylint: disable=invalid-name notify_broker_of_new_job = _notify_broker_of_new_job_1.delay # pylint: disable=invalid-name notify_broker_end_of_job = _notify_broker_end_of_job_1.delay # pylint: disable=invalid-name notify_broker_kill_single_runner = _notify_broker_kill_single_runner_1.delay # pylint: disable=invalid-name adjust_amount_runners = _adjust_amount_runners_1.delay # pylint: disable=invalid-name send_reminder_mails: t.Callable[ [t. Tuple[int], NamedArg(t.Optional[datetime.datetime], 'eta')], t.Any] = _send_reminder_mails_1.apply_async # pylint: disable=invalid-name stop_auto_test_run: t.Callable[ [t. Tuple[int], NamedArg(t.Optional[datetime.datetime], 'eta')], t.Any] = _stop_auto_test_run_1.apply_async # pylint: disable=invalid-name notify_slow_auto_test_run: t.Callable[ [t. Tuple[int], NamedArg(t.Optional[datetime.datetime], 'eta')], t.Any] = _notify_slow_auto_test_run_1.apply_async # pylint: disable=invalid-name check_heartbeat_auto_test_run: t.Callable[ [t.Tuple[str], DefaultNamedArg(t.Optional[datetime.datetime], 'eta')], t.Any] = _check_heartbeat_stop_test_runner_1.apply_async # pylint: disable=invalid-name
import hashlib from typing import Callable, Optional from mypy_extensions import Arg, DefaultNamedArg _UNSET = object() DEFAULT_HASH_NAME = "sha256" DEFAULT_SALT = b"this is not a great salt" DEFAULT_ITERATIONS = 100000 StretcherBackend = Callable[ [Arg(str, "input_key" ), DefaultNamedArg(bytes, "salt")], bytes] def pbkdf2_hmac_stretcher( input_key: str, salt: bytes = DEFAULT_SALT, hash_name: str = DEFAULT_HASH_NAME, iterations: int = DEFAULT_ITERATIONS, ) -> bytes: return hashlib.pbkdf2_hmac( hash_name=hash_name, password=input_key.encode(), salt=salt, iterations=iterations, ) def stretch(
for segment in request.prepath ), request.method, enctype, "utf-8", fieldValues.prevalidationValues, fieldValues.validationErrors, ) return Element(TagLoader(renderable)) _requirerFunctionWithForm = Any _routeCallable = Any _routeDecorator = Callable[ [_routeCallable, DefaultNamedArg(Any, "__session__")], _routeCallable ] _validationFailureHandler = Callable[ [Optional[object], IRequest, "Form", Dict[str, str]], Element ] validationFailureHandlerAttribute = "__kleinFormValidationFailureHandlers__" class IProtoForm(Interface): """ Marker interface for L{ProtoForm}. """ class IForm(Interface):
WhenFilterFn = Callable[..., bool] MetaFilterFn = Callable[..., bool] else: from mypy_extensions import Arg, DefaultNamedArg, KwArg, NamedArg, VarArg # TODO: Try using ParamSpec to support index type checking in callbacks # when PEP 612 is released (https://www.python.org/dev/peps/pep-0612/) ActivityFn = Callable[[ NamedArg(configuration.OperatorSettings, "settings"), NamedArg(ephemera.Index, "*"), NamedArg(int, "retry"), NamedArg(datetime.datetime, "started"), NamedArg(datetime.timedelta, "runtime"), NamedArg(LoggerType, "logger"), NamedArg(ephemera.AnyMemo, "memo"), DefaultNamedArg(Any, "param"), KwArg(Any), ], _SyncOrAsyncResult] ResourceIndexingFn = Callable[[ NamedArg(bodies.Annotations, "annotations"), NamedArg(bodies.Labels, "labels"), NamedArg(bodies.Body, "body"), NamedArg(bodies.Meta, "meta"), NamedArg(bodies.Spec, "spec"), NamedArg(bodies.Status, "status"), NamedArg(references.Resource, "resource"), NamedArg(Optional[str], "uid"), NamedArg(Optional[str], "name"), NamedArg(Optional[str], "namespace"), NamedArg(patches.Patch, "patch"),
assert_equivalent_metrics(A, B) assert len(validation_metrics["A"]) == len(validation_metrics["B"]) for A, B in zip(validation_metrics["A"], validation_metrics["B"]): assert_equivalent_metrics(A, B) return ( (training_metrics["A"], validation_metrics["A"]), (training_metrics["B"], validation_metrics["B"]), ) RestorableMakeControllerFn = Callable[ [ workload.Stream, DefaultNamedArg(Optional[str], "checkpoint_dir"), # noqa: F821 DefaultNamedArg(Optional[str], "latest_checkpoint"), # noqa: F821 DefaultNamedArg(int, "steps_completed"), # noqa: F821 ], det.TrialController, ] def train_and_validate( make_trial_controller_fn: Callable[[workload.Stream], det.TrialController], steps: int = 2, ) -> Tuple[Sequence[Dict[str, Any]], Sequence[Dict[str, Any]]]: metrics: Dict[str, Any] = {"training": [], "validation": []} def make_workloads(steps: int) -> workload.Stream: trainer = TrainAndValidate()
try: import yelp_meteorite except ImportError: # Sorry to any non-yelpers but you won't # get metrics emitted as our metrics lib # is currently not open source yelp_meteorite = None # Marathon REST API: # https://github.com/mesosphere/marathon/blob/master/REST.md#post-v2apps log = logging.getLogger(__name__) LogDeployError = Callable[ [Arg(str, "errormsg"), DefaultNamedArg(str, "level")], None] LogBounceAction = Callable[ [Arg(str, "line"), DefaultNamedArg(str, "level")], None] def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Creates marathon jobs.") parser.add_argument( "service_instance_list", nargs="+", help="The list of marathon service instances to create or update", metavar="SERVICE%sINSTANCE" % SPACER, ) parser.add_argument( "-d",
notify_broker_of_new_job = _notify_broker_of_new_job_1.delay # pylint: disable=invalid-name notify_broker_end_of_job = _notify_broker_end_of_job_1.delay # pylint: disable=invalid-name notify_broker_kill_single_runner = _notify_broker_kill_single_runner_1.delay # pylint: disable=invalid-name adjust_amount_runners = _adjust_amount_runners_1.delay # pylint: disable=invalid-name kill_runners_and_adjust = _kill_runners_and_adjust_1.delay # pylint: disable=invalid-name update_latest_results_in_broker = _update_latest_results_in_broker_1.delay # pylint: disable=invalid-name clone_commit_as_submission = _clone_commit_as_submission_1.delay # pylint: disable=invalid-name delete_mirror_file_at_time = _delete_mirror_file_at_time_1.delay # pylint: disable=invalid-name send_direct_notification_emails = _send_direct_notification_emails_1.delay # pylint: disable=invalid-name send_email_as_user = _send_email_as_user_1.delay # pylint: disable=invalid-name send_login_links_to_users: t.Callable[ [t.Tuple[int, str, str, str, int], NamedArg(DatetimeWithTimezone, 'eta')], object] = _send_login_links_to_users_1.apply_async # pylint: disable=invalid-name send_reminder_mails: t.Callable[ [t.Tuple[int], NamedArg(t.Optional[DatetimeWithTimezone], 'eta')], t.Any] = _send_reminder_mails_1.apply_async # pylint: disable=invalid-name check_heartbeat_auto_test_run: t.Callable[ [t.Tuple[str], DefaultNamedArg(t.Optional[DatetimeWithTimezone], 'eta')], t.Any] = _check_heartbeat_stop_test_runner_1.apply_async # pylint: disable=invalid-name maybe_open_assignment_at: t.Callable[ [t.Tuple[int], DefaultNamedArg(t.Optional[DatetimeWithTimezone], 'eta')], None] = _maybe_open_assignment_at_1.apply_async # pylint: disable=invalid-name