def __init__(
self,
population_size: int,
generations: Optional[int],
*,
individual_structure: IndividualStructure[IndividualType, GeneType],
elite_size: int = 0,
):
self.population_size = population_size
self.generations = generations
self._multiple_individual_operators: List[
Tuple[MultipleIndividualOperatorProtocol[IndividualType], float]
] = []
self._single_individual_operators: List[
Tuple[SingleIndividualOperatorProtocol[IndividualType], float]
] = []
self._individual_structure: Final = individual_structure
self.elite_size = elite_size
self._callbacks: List[Callable[[Evolution], None]] = []
self._executor: Optional[Executor] = None
self._selector: Optional[Selector] = None
self._ranker: Callable[
[
Callable[[IndividualType], Q],
DefaultNamedArg(Optional[Executor], "executor"),
],
Ranker,
] = Ranker
def gen_make_request(
service_name: str, api_error_cls: Type[BaseApiError]
) -> Callable[[
str,
str,
Type[T],
DefaultNamedArg(str, "access_token"),
DefaultNamedArg(httpx.models.RequestData, "data"),
], Awaitable[T], ]:
"""
Make a request handler for a specific service
"""
return partial( # type:ignore
_make_request_base,
service_name=service_name,
api_error_cls=api_error_cls,
)
def ranker_type(
self,
r: Callable[
[
Callable[[IndividualType], Q],
DefaultNamedArg(Optional[Executor], "executor"),
],
Ranker,
],
):
self._ranker = r
return self
def _oas_handler_msg(
msg_fn: Callable[[
str,
Any,
DefaultNamedArg(Optional[ErrorCode], 'code'),
], None],
f_ctx: FunctionContext,
msg: Tuple[Optional[ErrorCode], str],
line_number: Optional[int] = None,
) -> Type:
ctx = f_ctx.context
ctx.line = line_number or ctx.line
msg_fn(msg[1], ctx, msg[0]) # type: ignore
return f_ctx.default_return_type
task_result.as_task(__task)
p.models.db.session.commit()
lint_instances = _lint_instances_1.delay # pylint: disable=invalid-name
add = _add_1.delay # pylint: disable=invalid-name
send_done_mail = _send_done_mail_1.delay # pylint: disable=invalid-name
send_grader_status_mail = _send_grader_status_mail_1.delay # pylint: disable=invalid-name
run_plagiarism_control = _run_plagiarism_control_1.delay # pylint: disable=invalid-name
notify_broker_of_new_job = _notify_broker_of_new_job_1.delay # pylint: disable=invalid-name
notify_broker_end_of_job = _notify_broker_end_of_job_1.delay # pylint: disable=invalid-name
notify_broker_kill_single_runner = _notify_broker_kill_single_runner_1.delay # pylint: disable=invalid-name
adjust_amount_runners = _adjust_amount_runners_1.delay # pylint: disable=invalid-name
kill_runners_and_adjust = _kill_runners_and_adjust_1.delay # pylint: disable=invalid-name
update_latest_results_in_broker = _update_latest_results_in_broker_1.delay # pylint: disable=invalid-name
clone_commit_as_submission = _clone_commit_as_submission_1.delay # pylint: disable=invalid-name
delete_file_at_time = _delete_file_at_time_1.delay # pylint: disable=invalid-name
send_direct_notification_emails = _send_direct_notification_emails_1.delay # pylint: disable=invalid-name
send_email_as_user = _send_email_as_user_1.delay # pylint: disable=invalid-name
send_reminder_mails: t.Callable[
[t.Tuple[int],
NamedArg(t.Optional[DatetimeWithTimezone], 'eta')], t.
Any] = _send_reminder_mails_1.apply_async # pylint: disable=invalid-name
check_heartbeat_auto_test_run: t.Callable[
[t.Tuple[str],
DefaultNamedArg(t.Optional[DatetimeWithTimezone], 'eta')], t.
Any] = _check_heartbeat_stop_test_runner_1.apply_async # pylint: disable=invalid-name
MessageSentCallback,
ModelArg,
PendingMessage,
RecordMetadata,
TP,
V,
)
from .types.topics import ChannelT, TopicT
from .types.transports import ProducerT
if typing.TYPE_CHECKING: # pragma: no cover
from mypy_extensions import DefaultNamedArg
from .app import App
DecodeFunction = Callable[[Message, DefaultNamedArg(bool, 'propagate')],
Awaitable[EventT]]
else:
class App:
... # noqa
DecodeFunction = Callable[..., Awaitable[EventT]]
__all__ = ['Topic']
logger = get_logger(__name__)
class Topic(Channel, TopicT):
"""Define new topic description.
for A, B in zip(training_metrics["A"], training_metrics["B"]):
assert_equivalent_metrics(A, B)
assert len(validation_metrics["A"]) == len(validation_metrics["B"])
for A, B in zip(validation_metrics["A"], validation_metrics["B"]):
assert_equivalent_metrics(A, B)
return (
(training_metrics["A"], validation_metrics["A"]),
(training_metrics["B"], validation_metrics["B"]),
)
RestorableMakeControllerFn = Callable[
[workload.Stream,
DefaultNamedArg(Optional[pathlib.Path], "load_path")], # noqa: F821
det.TrialController, ]
def checkpointing_and_restoring_test(
make_trial_controller_fn: RestorableMakeControllerFn, tmp_path: Path
) -> Tuple[Sequence[Dict[str, Any]], Sequence[Dict[str, Any]]]:
"""
Tests if a trial controller of any framework can checkpoint and restore from that checkpoint
without state changes.
This test runs two trials.
1) Trial A runs for one steps of 100 batches, checkpoints itself, and restores from
that checkpoint.
2) Trial B runs for two steps of 100 batches.
for A, B in zip(training_metrics["A"], training_metrics["B"]):
assert_equivalent_metrics(A, B)
assert len(validation_metrics["A"]) == len(validation_metrics["B"])
for A, B in zip(validation_metrics["A"], validation_metrics["B"]):
assert_equivalent_metrics(A, B)
return (
(training_metrics["A"], validation_metrics["A"]),
(training_metrics["B"], validation_metrics["B"]),
)
RestorableMakeControllerFn = Callable[[
workload.Stream,
DefaultNamedArg(Optional[str], "checkpoint_dir"), # noqa: F821
DefaultNamedArg(Optional[str], "latest_checkpoint"), # noqa: F821
DefaultNamedArg(int, "steps_completed"), # noqa: F821
], det.TrialController, ]
def train_and_validate(
make_trial_controller_fn: Callable[[workload.Stream], det.TrialController],
steps: int = 2,
) -> Tuple[Sequence[Dict[str, Any]], Sequence[Dict[str, Any]]]:
metrics: Dict[str, Any] = {"training": [], "validation": []}
def make_workloads(steps: int) -> workload.Stream:
trainer = TrainAndValidate()
yield from trainer.send(steps, validation_freq=1, scheduling_unit=10)
MetaFilterFn = Callable[..., bool] # strictly sync, no async!
else:
from mypy_extensions import Arg, DefaultNamedArg, KwArg, NamedArg
# TODO:1: Split to specialised LoginFn, ProbeFn, StartupFn, etc. -- with different result types.
# TODO:2: Try using ParamSpec to support index type checking in callbacks
# when PEP 612 is released (https://www.python.org/dev/peps/pep-0612/)
ActivityFn = Callable[[
NamedArg(configuration.OperatorSettings, "settings"),
NamedArg(ephemera.Index, "*"),
NamedArg(int, "retry"),
NamedArg(datetime.datetime, "started"),
NamedArg(datetime.timedelta, "runtime"),
NamedArg(typedefs.Logger, "logger"),
NamedArg(Any, "memo"),
DefaultNamedArg(Any, "param"),
KwArg(Any),
], invocation.SyncOrAsync[Optional[object]]]
IndexingFn = Callable[[
NamedArg(bodies.Annotations, "annotations"),
NamedArg(bodies.Labels, "labels"),
NamedArg(bodies.Body, "body"),
NamedArg(bodies.Meta, "meta"),
NamedArg(bodies.Spec, "spec"),
NamedArg(bodies.Status, "status"),
NamedArg(references.Resource, "resource"),
NamedArg(Optional[str], "uid"),
NamedArg(Optional[str], "name"),
NamedArg(Optional[str], "namespace"),
NamedArg(patches.Patch, "patch"),
try:
import yelp_meteorite
except ImportError:
# Sorry to any non-yelpers but you won't
# get metrics emitted as our metrics lib
# is currently not open source
yelp_meteorite = None
# Marathon REST API:
# https://github.com/mesosphere/marathon/blob/master/REST.md#post-v2apps
log = logging.getLogger(__name__)
LogDeployError = Callable[[Arg(str, "errormsg"), DefaultNamedArg(str, "level")], None]
LogBounceAction = Callable[[Arg(str, "line"), DefaultNamedArg(str, "level")], None]
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Creates marathon jobs.")
parser.add_argument(
"service_instance_list",
nargs="+",
help="The list of marathon service instances to create or update",
metavar="SERVICE%sINSTANCE" % SPACER,
)
parser.add_argument(
"-d",
segment.decode("utf-8", errors='replace')
for segment in request.prepath),
request.method,
enctype,
"utf-8",
fieldValues.prevalidationValues,
fieldValues.validationErrors,
)
return Element(TagLoader(renderable))
_requirerFunctionWithForm = Any
_routeCallable = Any
_routeDecorator = Callable[
[_routeCallable, DefaultNamedArg(Any, '__session__')], _routeCallable]
_validationFailureHandler = Callable[
[Optional[object], IRequest, 'Form', Dict[str, str]], Element]
validationFailureHandlerAttribute = "__kleinFormValidationFailureHandlers__"
class IProtoForm(Interface):
"""
Marker interface for L{ProtoForm}.
"""
class IForm(Interface):
"""
Marker interface for form attached to dependency injection components.
passback_grades = _passback_grades_1.delay # pylint: disable=invalid-name
lint_instances = _lint_instances_1.delay # pylint: disable=invalid-name
add = _add_1.delay # pylint: disable=invalid-name
send_done_mail = _send_done_mail_1.delay # pylint: disable=invalid-name
send_grader_status_mail = _send_grader_status_mail_1.delay # pylint: disable=invalid-name
run_plagiarism_control = _run_plagiarism_control_1.delay # pylint: disable=invalid-name
notify_broker_of_new_job = _notify_broker_of_new_job_1.delay # pylint: disable=invalid-name
notify_broker_end_of_job = _notify_broker_end_of_job_1.delay # pylint: disable=invalid-name
notify_broker_kill_single_runner = _notify_broker_kill_single_runner_1.delay # pylint: disable=invalid-name
adjust_amount_runners = _adjust_amount_runners_1.delay # pylint: disable=invalid-name
send_reminder_mails: t.Callable[
[t.
Tuple[int], NamedArg(t.Optional[datetime.datetime], 'eta')],
t.Any] = _send_reminder_mails_1.apply_async # pylint: disable=invalid-name
stop_auto_test_run: t.Callable[
[t.
Tuple[int], NamedArg(t.Optional[datetime.datetime], 'eta')],
t.Any] = _stop_auto_test_run_1.apply_async # pylint: disable=invalid-name
notify_slow_auto_test_run: t.Callable[
[t.
Tuple[int], NamedArg(t.Optional[datetime.datetime], 'eta')],
t.Any] = _notify_slow_auto_test_run_1.apply_async # pylint: disable=invalid-name
check_heartbeat_auto_test_run: t.Callable[
[t.Tuple[str],
DefaultNamedArg(t.Optional[datetime.datetime], 'eta')],
t.Any] = _check_heartbeat_stop_test_runner_1.apply_async # pylint: disable=invalid-name
import hashlib
from typing import Callable, Optional
from mypy_extensions import Arg, DefaultNamedArg
_UNSET = object()
DEFAULT_HASH_NAME = "sha256"
DEFAULT_SALT = b"this is not a great salt"
DEFAULT_ITERATIONS = 100000
StretcherBackend = Callable[
[Arg(str, "input_key"
), DefaultNamedArg(bytes, "salt")], bytes]
def pbkdf2_hmac_stretcher(
input_key: str,
salt: bytes = DEFAULT_SALT,
hash_name: str = DEFAULT_HASH_NAME,
iterations: int = DEFAULT_ITERATIONS,
) -> bytes:
return hashlib.pbkdf2_hmac(
hash_name=hash_name,
password=input_key.encode(),
salt=salt,
iterations=iterations,
)
def stretch(
for segment in request.prepath
),
request.method,
enctype,
"utf-8",
fieldValues.prevalidationValues,
fieldValues.validationErrors,
)
return Element(TagLoader(renderable))
_requirerFunctionWithForm = Any
_routeCallable = Any
_routeDecorator = Callable[
[_routeCallable, DefaultNamedArg(Any, "__session__")], _routeCallable
]
_validationFailureHandler = Callable[
[Optional[object], IRequest, "Form", Dict[str, str]], Element
]
validationFailureHandlerAttribute = "__kleinFormValidationFailureHandlers__"
class IProtoForm(Interface):
"""
Marker interface for L{ProtoForm}.
"""
class IForm(Interface):
try:
import yelp_meteorite
except ImportError:
# Sorry to any non-yelpers but you won't
# get metrics emitted as our metrics lib
# is currently not open source
yelp_meteorite = None
# Marathon REST API:
# https://github.com/mesosphere/marathon/blob/master/REST.md#post-v2apps
log = logging.getLogger(__name__)
LogDeployError = Callable[
[Arg(str, "errormsg"), DefaultNamedArg(str, "level")], None]
LogBounceAction = Callable[
[Arg(str, "line"), DefaultNamedArg(str, "level")], None]
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Creates marathon jobs.")
parser.add_argument(
"service_instance_list",
nargs="+",
help="The list of marathon service instances to create or update",
metavar="SERVICE%sINSTANCE" % SPACER,
)
parser.add_argument(
"-d",
*,
logger: Optional[logging.Logger]) -> 'MailT': ...
class MarkdownRow(NamedTuple):
name: str
price: int
class MarkdownRecord(NamedTuple):
description: str
row_list: Tuple[MarkdownRow, ...]
ToMarkdown = Callable[
[ReceiptT, DefaultNamedArg(Optional[logging.Logger], 'logger')],
MarkdownRecord]
def write_markdown(
path: pathlib.Path,
receipt_list: List[ReceiptBase],
to_markdown: ToMarkdown,
logger: Optional[logging.Logger] = None,
timezone: Optional[datetime.tzinfo] = None) -> None:
with path.open(mode='w') as f:
last_date: Optional[datetime.date] = None
for receipt in receipt_list:
data = to_markdown(receipt, logger=logger)
time = receipt.purchased_date.astimezone(tz=timezone)
if last_date is None or last_date != time.date():
# pylint: disable=cyclic-import
import bs4 # noqa
import gluetool # noqa
import gluetool.sentry # noqa
# Type definitions
# pylint: disable=invalid-name
ExceptionInfoType = Union[
Tuple[Optional[type], Optional[BaseException], Optional[TracebackType]], # returned by sys.exc_info()
Tuple[None, None, None]
]
LoggingFunctionType = Callable[
[
Arg(str),
DefaultNamedArg(ExceptionInfoType, 'exc_info'),
DefaultNamedArg(Dict[str, Any], 'extra'),
DefaultNamedArg(bool, 'sentry')
],
None
]
ContextInfoType = Tuple[int, Any]
BLOB_HEADER = '---v---v---v---v---v---'
BLOB_FOOTER = '---^---^---^---^---^---'
# Default log level is logging.INFO or logging.DEBUG if GLUETOOL_DEBUG environment variable is set
DEFAULT_LOG_LEVEL = logging.DEBUG if os.getenv('GLUETOOL_DEBUG') else logging.INFO
from faust.exceptions import KeyDecodeError, ValueDecodeError
from faust.types.app import AppT
from faust.types.core import K, OpenHeadersArg, V
from faust.types.codecs import CodecArg
from faust.types.events import EventT
from faust.types.models import ModelArg
from faust.types.serializers import KT, SchemaT, VT
from faust.types.tuples import Message
__all__ = ['Schema']
if typing.TYPE_CHECKING: # pragma: no cover
from mypy_extensions import DefaultNamedArg
DecodeFunction = Callable[
[Message, DefaultNamedArg(bool, 'propagate')], # noqa: F821
Awaitable[EventT], ]
else:
DecodeFunction = Callable[..., Awaitable[EventT]]
OnKeyDecodeErrorFun = Callable[[Exception, Message], Awaitable[None]]
OnValueDecodeErrorFun = Callable[[Exception, Message], Awaitable[None]]
async def _noop_decode_error(exc: Exception, message: Message) -> None:
...
class Schema(SchemaT):
def __init__(self,
*,
from faust.exceptions import KeyDecodeError, ValueDecodeError
from faust.types.app import AppT
from faust.types.codecs import CodecArg
from faust.types.core import K, OpenHeadersArg, V
from faust.types.events import EventT
from faust.types.models import ModelArg
from faust.types.serializers import KT, VT, SchemaT
from faust.types.tuples import Message
__all__ = ["Schema"]
if typing.TYPE_CHECKING: # pragma: no cover
from mypy_extensions import DefaultNamedArg
DecodeFunction = Callable[
[Message, DefaultNamedArg(bool, "propagate")], # noqa: F821
Awaitable[EventT], ]
else:
DecodeFunction = Callable[..., Awaitable[EventT]]
OnKeyDecodeErrorFun = Callable[[Exception, Message], Awaitable[None]]
OnValueDecodeErrorFun = Callable[[Exception, Message], Awaitable[None]]
async def _noop_decode_error(exc: Exception, message: Message) -> None:
...
class Schema(SchemaT):
def __init__(
self,
ModelArg,
PendingMessage,
RecordMetadata,
TP,
V,
)
from .types.topics import ChannelT, TopicT
from .types.transports import ProducerT
if typing.TYPE_CHECKING: # pragma: no cover
from mypy_extensions import DefaultNamedArg
from .app import App
DecodeFunction = Callable[
[Message, DefaultNamedArg(bool, 'propagate')], Awaitable[EventT]]
else:
class App:
... # noqa
DecodeFunction = Callable[..., Awaitable[EventT]]
__all__ = ['Topic']
logger = get_logger(__name__)
class Topic(Channel, TopicT):
"""Define new topic description.
import functools
import json
from itertools import chain
from time import time as utc
from typing import Dict, List, Callable, Tuple, Any, Optional, Union
import requests
from mypy_extensions import DefaultNamedArg
from .config_logging import log
log = log.getChild('TwitchAPI')
RF = Callable[[
str,
DefaultNamedArg(Optional[Dict], 'params'),
DefaultNamedArg(Optional[Dict[str, str]], 'headers')
], requests.Response]
def timed_cache(func: Callable[..., Dict]) -> Callable[..., Dict]:
cache: Dict[str, Tuple[Dict, int]] = {}
@functools.wraps(func)
def wrapper(*args: Any, **kwargs: Any) -> Dict:
_, video_id, *_ = args
if video_id not in cache or cache[video_id][1] < utc():
token = func(*args, **kwargs)
cache[video_id] = token, json.loads(token['token'])['expires']
return cache[video_id][0]
assert len(training_metrics["A"]) == len(training_metrics["B"])
for A, B in zip(training_metrics["A"], training_metrics["B"]):
assert_equivalent_metrics(A, B)
assert len(validation_metrics["A"]) == len(validation_metrics["B"])
for A, B in zip(validation_metrics["A"], validation_metrics["B"]):
assert_equivalent_metrics(A, B)
return (
(training_metrics["A"], validation_metrics["A"]),
(training_metrics["B"], validation_metrics["B"]),
)
OptimizerStateMakeControllerFn = Callable[
[workload.Stream, DefaultNamedArg(Optional[pathlib.Path], "load_path")], # noqa: F821
det.TrialController,
]
def optimizer_state_test(
make_trial_controller_fn: OptimizerStateMakeControllerFn, tmp_path: Path
) -> Tuple[Sequence[Dict[str, Any]], Sequence[Dict[str, Any]]]:
# To test checkpointing optimizer state, run a set of two trials.
#
# 1) Trial A is run for two steps of 100 batches each, between two
# separate trial instances.
# 2) Trial B is run for two steps of 100 batches each with the same
# trial instance.
#
# Thus the training procedure is equivalent, but in the case of Trial
elif byteorder == 'little':
bytes = bytes[::-1]
else:
raise ValueError("byteorder must be either 'little' or 'big'")
hex_str = codecs.encode(bytes, 'hex') # type: ignore
return int(hex_str, 16)
# Make this cast since Python 2 doesn't have syntax for default
# named arguments. Hence, must cast so Mypy thinks it matches the
# original function.
int_from_bytes = cast(
Callable[[
Arg(Sequence[int], 'bytes'),
Arg(str, 'byteorder'),
DefaultNamedArg(bool, 'signed')
], int], _int_from_bytes)
def re_compile_full(pattern, flags=0):
# type: (AnyStr, int) -> Pattern
""" Create compiled regular expression such that it matches the
entire string. Calling re.match on the output of this function
is equivalent to calling re.fullmatch on its input.
This is needed to support Python 2. (On Python 3, we would just
call re.fullmatch.)
Kudos: https://stackoverflow.com/a/30212799
:param pattern: The pattern to compile.
:param flags: Regular expression flags. Refer to Python
from paasta_tools.utils import InvalidInstanceConfig
from paasta_tools.utils import InvalidJobNameError
from paasta_tools.utils import load_system_paasta_config
from paasta_tools.utils import NoConfigurationForServiceError
from paasta_tools.utils import NoDeploymentsAvailable
from paasta_tools.utils import NoDockerImageError
from paasta_tools.utils import SPACER
from paasta_tools.utils import SystemPaastaConfig
# Marathon REST API:
# https://github.com/mesosphere/marathon/blob/master/REST.md#post-v2apps
log = logging.getLogger(__name__)
LogDeployError = Callable[
[Arg(str, 'errormsg'), DefaultNamedArg(str, 'level')], None]
LogBounceAction = Callable[
[Arg(str, 'line'), DefaultNamedArg(str, 'level')], None]
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description='Creates marathon jobs.')
parser.add_argument(
'service_instance_list',
nargs='+',
help="The list of marathon service instances to create or update",
metavar="SERVICE%sINSTANCE" % SPACER,
)
parser.add_argument(
'-d',
assert len(training_metrics["A"]) == len(training_metrics["B"])
for A, B in zip(training_metrics["A"], training_metrics["B"]):
assert_equivalent_metrics(A, B)
assert len(validation_metrics["A"]) == len(validation_metrics["B"])
for A, B in zip(validation_metrics["A"], validation_metrics["B"]):
assert_equivalent_metrics(A, B)
return (
(training_metrics["A"], validation_metrics["A"]),
(training_metrics["B"], validation_metrics["B"]),
)
OptimizerStateMakeControllerFn = Callable[
[workload.Stream, DefaultNamedArg(Optional[pathlib.Path], "load_path")], det.TrialController
]
def optimizer_state_test(
make_trial_controller_fn: OptimizerStateMakeControllerFn, tmp_path: Path
) -> Tuple[Sequence[Dict[str, Any]], Sequence[Dict[str, Any]]]:
# To test checkpointing optimizer state, run a set of two trials.
#
# 1) Trial A is run for two steps of 100 batches each, between two
# separate trial instances.
# 2) Trial B is run for two steps of 100 batches each with the same
# trial instance.
#
# Thus the training procedure is equivalent, but in the case of Trial
# A, the full training state must be restored from checkpoint. At the
