import functools
import json
from itertools import chain
from time import time as utc
from typing import Dict, List, Callable, Tuple, Any, Optional, Union
import requests
from mypy_extensions import DefaultNamedArg
from .config_logging import log
log = log.getChild('TwitchAPI')
RF = Callable[[
str,
DefaultNamedArg(Optional[Dict], 'params'),
DefaultNamedArg(Optional[Dict[str, str]], 'headers')
], requests.Response]
def timed_cache(func: Callable[..., Dict]) -> Callable[..., Dict]:
cache: Dict[str, Tuple[Dict, int]] = {}
@functools.wraps(func)
def wrapper(*args: Any, **kwargs: Any) -> Dict:
_, video_id, *_ = args
if video_id not in cache or cache[video_id][1] < utc():
token = func(*args, **kwargs)
cache[video_id] = token, json.loads(token['token'])['expires']
return cache[video_id][0]
elif byteorder == 'little':
bytes = bytes[::-1]
else:
raise ValueError("byteorder must be either 'little' or 'big'")
hex_str = codecs.encode(bytes, 'hex') # type: ignore
return int(hex_str, 16)
# Make this cast since Python 2 doesn't have syntax for default
# named arguments. Hence, must cast so Mypy thinks it matches the
# original function.
int_from_bytes = cast(
Callable[[
Arg(Sequence[int], 'bytes'),
Arg(str, 'byteorder'),
DefaultNamedArg(bool, 'signed')
], int], _int_from_bytes)
def re_compile_full(pattern, flags=0):
# type: (AnyStr, int) -> Pattern
""" Create compiled regular expression such that it matches the
entire string. Calling re.match on the output of this function
is equivalent to calling re.fullmatch on its input.
This is needed to support Python 2. (On Python 3, we would just
call re.fullmatch.)
Kudos: https://stackoverflow.com/a/30212799
:param pattern: The pattern to compile.
:param flags: Regular expression flags. Refer to Python
from faust.exceptions import KeyDecodeError, ValueDecodeError
from faust.types.app import AppT
from faust.types.codecs import CodecArg
from faust.types.core import K, OpenHeadersArg, V
from faust.types.events import EventT
from faust.types.models import ModelArg
from faust.types.serializers import KT, VT, SchemaT
from faust.types.tuples import Message
__all__ = ["Schema"]
if typing.TYPE_CHECKING: # pragma: no cover
from mypy_extensions import DefaultNamedArg
DecodeFunction = Callable[
[Message, DefaultNamedArg(bool, "propagate")], # noqa: F821
Awaitable[EventT], ]
else:
DecodeFunction = Callable[..., Awaitable[EventT]]
OnKeyDecodeErrorFun = Callable[[Exception, Message], Awaitable[None]]
OnValueDecodeErrorFun = Callable[[Exception, Message], Awaitable[None]]
async def _noop_decode_error(exc: Exception, message: Message) -> None:
...
class Schema(SchemaT):
def __init__(
self,
for A, B in zip(training_metrics["A"], training_metrics["B"]):
assert_equivalent_metrics(A, B)
assert len(validation_metrics["A"]) == len(validation_metrics["B"])
for A, B in zip(validation_metrics["A"], validation_metrics["B"]):
assert_equivalent_metrics(A, B)
return (
(training_metrics["A"], validation_metrics["A"]),
(training_metrics["B"], validation_metrics["B"]),
)
RestorableMakeControllerFn = Callable[
[workload.Stream,
DefaultNamedArg(Optional[pathlib.Path], "load_path")], # noqa: F821
det.TrialController, ]
def checkpointing_and_restoring_test(
make_trial_controller_fn: RestorableMakeControllerFn, tmp_path: Path
) -> Tuple[Sequence[Dict[str, Any]], Sequence[Dict[str, Any]]]:
"""
Tests if a trial controller of any framework can checkpoint and restore from that checkpoint
without state changes.
This test runs two trials.
1) Trial A runs for one steps of 100 batches, checkpoints itself, and restores from
that checkpoint.
2) Trial B runs for two steps of 100 batches.
assert len(training_metrics["A"]) == len(training_metrics["B"])
for A, B in zip(training_metrics["A"], training_metrics["B"]):
assert_equivalent_metrics(A, B)
assert len(validation_metrics["A"]) == len(validation_metrics["B"])
for A, B in zip(validation_metrics["A"], validation_metrics["B"]):
assert_equivalent_metrics(A, B)
return (
(training_metrics["A"], validation_metrics["A"]),
(training_metrics["B"], validation_metrics["B"]),
)
RestorableMakeControllerFn = Callable[
[workload.Stream, DefaultNamedArg(Optional[pathlib.Path], "load_path")], # noqa: F821
det.TrialController,
]
def checkpointing_and_restoring_test(
make_trial_controller_fn: RestorableMakeControllerFn, tmp_path: Path
) -> Tuple[Sequence[Dict[str, Any]], Sequence[Dict[str, Any]]]:
"""
Tests if a trial controller of any framework can checkpoint and restore from that checkpoint
without state changes.
This test runs two trials.
1) Trial A runs for one steps of 100 batches, checkpoints itself, and restores from
that checkpoint.
2) Trial B runs for two steps of 100 batches.
RecordMetadata,
SchemaT,
TP,
V,
)
from .types.core import OpenHeadersArg
from .types.topics import ChannelT, TopicT
from .types.transports import ProducerT
if typing.TYPE_CHECKING: # pragma: no cover
from mypy_extensions import DefaultNamedArg
from .app import App
DecodeFunction = Callable[
[Message, DefaultNamedArg(bool, 'propagate')], Awaitable[EventT]]
else:
class App:
... # noqa
DecodeFunction = Callable[..., Awaitable[EventT]]
__all__ = ['Topic']
logger = get_logger(__name__)
class Topic(Channel, TopicT):
"""Define new topic description.
# pylint: disable=cyclic-import
import bs4 # noqa
import gluetool # noqa
import gluetool.sentry # noqa
# Type definitions
# pylint: disable=invalid-name
ExceptionInfoType = Union[
Tuple[Optional[type], Optional[BaseException], Optional[TracebackType]], # returned by sys.exc_info()
Tuple[None, None, None]
]
LoggingFunctionType = Callable[
[
Arg(str),
DefaultNamedArg(ExceptionInfoType, 'exc_info'),
DefaultNamedArg(Dict[str, Any], 'extra'),
DefaultNamedArg(bool, 'sentry')
],
None
]
ContextInfoType = Tuple[int, Any]
BLOB_HEADER = '---v---v---v---v---v---'
BLOB_FOOTER = '---^---^---^---^---^---'
# Default log level is logging.INFO or logging.DEBUG if GLUETOOL_DEBUG environment variable is set
DEFAULT_LOG_LEVEL = logging.DEBUG if os.getenv('GLUETOOL_DEBUG') else logging.INFO
MessageSentCallback,
ModelArg,
PendingMessage,
RecordMetadata,
TP,
V,
)
from .types.topics import ChannelT, TopicT
from .types.transports import ProducerT
if typing.TYPE_CHECKING: # pragma: no cover
from mypy_extensions import DefaultNamedArg
from .app import App
DecodeFunction = Callable[[Message, DefaultNamedArg(bool, 'propagate')],
Awaitable[EventT]]
else:
class App:
... # noqa
DecodeFunction = Callable[..., Awaitable[EventT]]
__all__ = ['Topic']
logger = get_logger(__name__)
class Topic(Channel, TopicT):
"""Define new topic description.
from paasta_tools.utils import InvalidInstanceConfig
from paasta_tools.utils import InvalidJobNameError
from paasta_tools.utils import load_system_paasta_config
from paasta_tools.utils import NoConfigurationForServiceError
from paasta_tools.utils import NoDeploymentsAvailable
from paasta_tools.utils import NoDockerImageError
from paasta_tools.utils import SPACER
from paasta_tools.utils import SystemPaastaConfig
# Marathon REST API:
# https://github.com/mesosphere/marathon/blob/master/REST.md#post-v2apps
log = logging.getLogger(__name__)
LogDeployError = Callable[
[Arg(str, 'errormsg'), DefaultNamedArg(str, 'level')], None]
LogBounceAction = Callable[
[Arg(str, 'line'), DefaultNamedArg(str, 'level')], None]
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description='Creates marathon jobs.')
parser.add_argument(
'service_instance_list',
nargs='+',
help="The list of marathon service instances to create or update",
metavar="SERVICE%sINSTANCE" % SPACER,
)
parser.add_argument(
'-d',
segment.decode("utf-8", errors='replace')
for segment in request.prepath),
request.method,
enctype,
"utf-8",
fieldValues.prevalidationValues,
fieldValues.validationErrors,
)
return Element(TagLoader(renderable))
_requirerFunctionWithForm = Any
_routeCallable = Any
_routeDecorator = Callable[
[_routeCallable, DefaultNamedArg(Any, '__session__')], _routeCallable]
_validationFailureHandler = Callable[
[Optional[object], IRequest, 'Form', Dict[str, str]], Element]
validationFailureHandlerAttribute = "__kleinFormValidationFailureHandlers__"
class IProtoForm(Interface):
"""
Marker interface for L{ProtoForm}.
"""
class IForm(Interface):
"""
Marker interface for form attached to dependency injection components.
passback_grades = _passback_grades_1.delay # pylint: disable=invalid-name
lint_instances = _lint_instances_1.delay # pylint: disable=invalid-name
add = _add_1.delay # pylint: disable=invalid-name
send_done_mail = _send_done_mail_1.delay # pylint: disable=invalid-name
send_grader_status_mail = _send_grader_status_mail_1.delay # pylint: disable=invalid-name
run_plagiarism_control = _run_plagiarism_control_1.delay # pylint: disable=invalid-name
notify_broker_of_new_job = _notify_broker_of_new_job_1.delay # pylint: disable=invalid-name
notify_broker_end_of_job = _notify_broker_end_of_job_1.delay # pylint: disable=invalid-name
notify_broker_kill_single_runner = _notify_broker_kill_single_runner_1.delay # pylint: disable=invalid-name
adjust_amount_runners = _adjust_amount_runners_1.delay # pylint: disable=invalid-name
send_reminder_mails: t.Callable[
[t.
Tuple[int], NamedArg(t.Optional[datetime.datetime], 'eta')],
t.Any] = _send_reminder_mails_1.apply_async # pylint: disable=invalid-name
stop_auto_test_run: t.Callable[
[t.
Tuple[int], NamedArg(t.Optional[datetime.datetime], 'eta')],
t.Any] = _stop_auto_test_run_1.apply_async # pylint: disable=invalid-name
notify_slow_auto_test_run: t.Callable[
[t.
Tuple[int], NamedArg(t.Optional[datetime.datetime], 'eta')],
t.Any] = _notify_slow_auto_test_run_1.apply_async # pylint: disable=invalid-name
check_heartbeat_auto_test_run: t.Callable[
[t.Tuple[str],
DefaultNamedArg(t.Optional[datetime.datetime], 'eta')],
t.Any] = _check_heartbeat_stop_test_runner_1.apply_async # pylint: disable=invalid-name
import hashlib
from typing import Callable, Optional
from mypy_extensions import Arg, DefaultNamedArg
_UNSET = object()
DEFAULT_HASH_NAME = "sha256"
DEFAULT_SALT = b"this is not a great salt"
DEFAULT_ITERATIONS = 100000
StretcherBackend = Callable[
[Arg(str, "input_key"
), DefaultNamedArg(bytes, "salt")], bytes]
def pbkdf2_hmac_stretcher(
input_key: str,
salt: bytes = DEFAULT_SALT,
hash_name: str = DEFAULT_HASH_NAME,
iterations: int = DEFAULT_ITERATIONS,
) -> bytes:
return hashlib.pbkdf2_hmac(
hash_name=hash_name,
password=input_key.encode(),
salt=salt,
iterations=iterations,
)
def stretch(
for segment in request.prepath
),
request.method,
enctype,
"utf-8",
fieldValues.prevalidationValues,
fieldValues.validationErrors,
)
return Element(TagLoader(renderable))
_requirerFunctionWithForm = Any
_routeCallable = Any
_routeDecorator = Callable[
[_routeCallable, DefaultNamedArg(Any, "__session__")], _routeCallable
]
_validationFailureHandler = Callable[
[Optional[object], IRequest, "Form", Dict[str, str]], Element
]
validationFailureHandlerAttribute = "__kleinFormValidationFailureHandlers__"
class IProtoForm(Interface):
"""
Marker interface for L{ProtoForm}.
"""
class IForm(Interface):
WhenFilterFn = Callable[..., bool]
MetaFilterFn = Callable[..., bool]
else:
from mypy_extensions import Arg, DefaultNamedArg, KwArg, NamedArg, VarArg
# TODO: Try using ParamSpec to support index type checking in callbacks
# when PEP 612 is released (https://www.python.org/dev/peps/pep-0612/)
ActivityFn = Callable[[
NamedArg(configuration.OperatorSettings, "settings"),
NamedArg(ephemera.Index, "*"),
NamedArg(int, "retry"),
NamedArg(datetime.datetime, "started"),
NamedArg(datetime.timedelta, "runtime"),
NamedArg(LoggerType, "logger"),
NamedArg(ephemera.AnyMemo, "memo"),
DefaultNamedArg(Any, "param"),
KwArg(Any),
], _SyncOrAsyncResult]
ResourceIndexingFn = Callable[[
NamedArg(bodies.Annotations, "annotations"),
NamedArg(bodies.Labels, "labels"),
NamedArg(bodies.Body, "body"),
NamedArg(bodies.Meta, "meta"),
NamedArg(bodies.Spec, "spec"),
NamedArg(bodies.Status, "status"),
NamedArg(references.Resource, "resource"),
NamedArg(Optional[str], "uid"),
NamedArg(Optional[str], "name"),
NamedArg(Optional[str], "namespace"),
NamedArg(patches.Patch, "patch"),
assert_equivalent_metrics(A, B)
assert len(validation_metrics["A"]) == len(validation_metrics["B"])
for A, B in zip(validation_metrics["A"], validation_metrics["B"]):
assert_equivalent_metrics(A, B)
return (
(training_metrics["A"], validation_metrics["A"]),
(training_metrics["B"], validation_metrics["B"]),
)
RestorableMakeControllerFn = Callable[
[
workload.Stream,
DefaultNamedArg(Optional[str], "checkpoint_dir"), # noqa: F821
DefaultNamedArg(Optional[str], "latest_checkpoint"), # noqa: F821
DefaultNamedArg(int, "steps_completed"), # noqa: F821
],
det.TrialController,
]
def train_and_validate(
make_trial_controller_fn: Callable[[workload.Stream], det.TrialController],
steps: int = 2,
) -> Tuple[Sequence[Dict[str, Any]], Sequence[Dict[str, Any]]]:
metrics: Dict[str, Any] = {"training": [], "validation": []}
def make_workloads(steps: int) -> workload.Stream:
trainer = TrainAndValidate()
try:
import yelp_meteorite
except ImportError:
# Sorry to any non-yelpers but you won't
# get metrics emitted as our metrics lib
# is currently not open source
yelp_meteorite = None
# Marathon REST API:
# https://github.com/mesosphere/marathon/blob/master/REST.md#post-v2apps
log = logging.getLogger(__name__)
LogDeployError = Callable[
[Arg(str, "errormsg"), DefaultNamedArg(str, "level")], None]
LogBounceAction = Callable[
[Arg(str, "line"), DefaultNamedArg(str, "level")], None]
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Creates marathon jobs.")
parser.add_argument(
"service_instance_list",
nargs="+",
help="The list of marathon service instances to create or update",
metavar="SERVICE%sINSTANCE" % SPACER,
)
parser.add_argument(
"-d",
notify_broker_of_new_job = _notify_broker_of_new_job_1.delay # pylint: disable=invalid-name
notify_broker_end_of_job = _notify_broker_end_of_job_1.delay # pylint: disable=invalid-name
notify_broker_kill_single_runner = _notify_broker_kill_single_runner_1.delay # pylint: disable=invalid-name
adjust_amount_runners = _adjust_amount_runners_1.delay # pylint: disable=invalid-name
kill_runners_and_adjust = _kill_runners_and_adjust_1.delay # pylint: disable=invalid-name
update_latest_results_in_broker = _update_latest_results_in_broker_1.delay # pylint: disable=invalid-name
clone_commit_as_submission = _clone_commit_as_submission_1.delay # pylint: disable=invalid-name
delete_mirror_file_at_time = _delete_mirror_file_at_time_1.delay # pylint: disable=invalid-name
send_direct_notification_emails = _send_direct_notification_emails_1.delay # pylint: disable=invalid-name
send_email_as_user = _send_email_as_user_1.delay # pylint: disable=invalid-name
send_login_links_to_users: t.Callable[
[t.Tuple[int, str, str, str, int],
NamedArg(DatetimeWithTimezone, 'eta')],
object] = _send_login_links_to_users_1.apply_async # pylint: disable=invalid-name
send_reminder_mails: t.Callable[
[t.Tuple[int],
NamedArg(t.Optional[DatetimeWithTimezone], 'eta')],
t.Any] = _send_reminder_mails_1.apply_async # pylint: disable=invalid-name
check_heartbeat_auto_test_run: t.Callable[
[t.Tuple[str],
DefaultNamedArg(t.Optional[DatetimeWithTimezone], 'eta')],
t.Any] = _check_heartbeat_stop_test_runner_1.apply_async # pylint: disable=invalid-name
maybe_open_assignment_at: t.Callable[
[t.Tuple[int],
DefaultNamedArg(t.Optional[DatetimeWithTimezone], 'eta')],
None] = _maybe_open_assignment_at_1.apply_async # pylint: disable=invalid-name
