def _is_receiving_messages(self):
# TODO: this is a hack to handle the fact that we sometimes seem to loose contact with kafka
self._logger.debug(
"No message for %r seconds",
int(time_monotonic()) - self._last_message_timestamp)
return self._last_message_timestamp > int(
time_monotonic()) - ALLOW_WITHOUT_MESSAGES_S
def wait_until(action,
description=None,
exception_class=AssertionError,
patience=60):
"""Attempt to call 'action' every 2 seconds, until it completes without exception or patience runs out"""
__tracebackhide__ = True
start = time_monotonic()
cause = []
if not description:
description = action.__doc__ or action.__name__
message = []
while time_monotonic() < (start + patience):
try:
action()
return
except BaseException:
cause = traceback.format_exception(*sys.exc_info())
time.sleep(2)
if cause:
message.append("\nThe last exception was:\n")
message.extend(cause)
header = "Gave up waiting for {} after {} seconds at {}".format(
description, patience,
datetime.now().isoformat(" "))
message.insert(0, header)
raise exception_class("".join(message))
def start(self):
extra_params = ["--keep-context"]
if self._k8s_version:
extra_params.extend(("--kubernetes-version", self._k8s_version))
extra_params.extend(self._driver.arguments)
running = self._attempt_start(extra_params)
start = time_monotonic()
while not running and time_monotonic() < (start + PATIENCE):
running = self._attempt_start(extra_params)
if not running:
raise MinikubeError("Gave up starting minikube after %d seconds" %
PATIENCE)
def __call__(self):
repository = _repository(self._app_spec)
if self._ready():
self._lifecycle.success(
app_name=self._app_spec.name,
namespace=self._app_spec.namespace,
deployment_id=self._app_spec.deployment_id,
repository=repository,
labels=self._app_spec.labels.status,
annotations=self._app_spec.annotations.status)
self._bookkeeper.success(self._app_spec)
return False
if time_monotonic() >= self._fail_after:
LOG.error(
"Timed out after %d seconds waiting for %s to become ready",
self._fail_after_seconds, self._app_spec.name)
self._lifecycle.failed(
app_name=self._app_spec.name,
namespace=self._app_spec.namespace,
deployment_id=self._app_spec.deployment_id,
repository=repository,
labels=self._app_spec.labels.status,
annotations=self._app_spec.annotations.status)
self._bookkeeper.failed(self._app_spec)
return False
return True
def __init__(self, app_spec, bookkeeper, lifecycle, lifecycle_subject,
config):
self._app_spec = app_spec
self._bookkeeper = bookkeeper
self._lifecycle = lifecycle
self._lifecycle_subject = lifecycle_subject
self._fail_after_seconds = _calculate_fail_time(
config.ready_check_timeout_multiplier, app_spec.replicas,
app_spec.health_checks.readiness.initial_delay_seconds)
self._fail_after = time_monotonic() + self._fail_after_seconds
def _wait_for_readiness(self, wait_time_seconds, timeout_seconds):
start = time_monotonic()
while time_monotonic() < (start + timeout_seconds):
if all(status == STATUS_SUCCESS
for status in self._status_collector.values()):
LOG.info("Bootstrapped {} applications".format(
len(self._status_collector.values())))
return True
else:
time.sleep(wait_time_seconds)
message = "Timed out after waiting {}s for applications to become ready.\n".format(
timeout_seconds)
message += "Applications which failed to become ready:\n"
for name, namespace, status in self._status_collector.items():
message += "{} in namespace {} had final state {}\n".format(
name, namespace, status)
LOG.error(message)
return False
def __call__(self):
if self._ready():
self._lifecycle.success(self._lifecycle_subject)
self._bookkeeper.success(self._app_spec)
return False
if time_monotonic() >= self._fail_after:
LOG.error("Timed out after %d seconds waiting for %s to become ready",
self._fail_after_seconds, self._app_spec.name)
self._lifecycle.failed(self._lifecycle_subject)
self._bookkeeper.failed(self._app_spec)
return False
return True
def __init__(self, deploy_queue, config, reporter, spec_factory,
app_config_downloader, lifecycle):
super(Consumer, self).__init__()
self._logger = logging.getLogger(__name__)
self._deploy_queue = deploy_queue
self._consumer = None
self._config = config
self._environment = config.environment
self._reporter = reporter
self._spec_factory = spec_factory
self._app_config_downloader = app_config_downloader
self._lifecycle = lifecycle
self._last_message_timestamp = int(time_monotonic())
def test_deployment_failed(self, get, app_spec, bookkeeper, requested, replicas, available, updated,
lifecycle, annotations, repository):
if annotations:
app_spec = app_spec._replace(annotations=LabelAndAnnotationSpec(*[annotations] * 5))
self._create_response(get, requested, replicas, available, updated)
ready = ReadyCheck(app_spec, bookkeeper, lifecycle)
ready._fail_after = time_monotonic()
assert ready() is False
bookkeeper.success.assert_not_called()
bookkeeper.failed.assert_called_with(app_spec)
lifecycle.success.assert_not_called()
lifecycle.failed.assert_called_with(app_name=app_spec.name, namespace=app_spec.namespace,
deployment_id=app_spec.deployment_id, repository=repository)
def __call__(self):
repository = _repository(self._app_spec)
if self._ready():
self._lifecycle.success(app_name=self._app_spec.name,
namespace=self._app_spec.namespace,
deployment_id=self._app_spec.deployment_id,
repository=repository)
self._bookkeeper.success(self._app_spec)
return False
if time_monotonic() >= self._fail_after:
self._lifecycle.failed(app_name=self._app_spec.name,
namespace=self._app_spec.namespace,
deployment_id=self._app_spec.deployment_id,
repository=repository)
self._bookkeeper.failed(self._app_spec)
return False
return True
def _handle_message(self, deploy_counter, message, message_counter):
message_counter.inc()
self._last_message_timestamp = int(time_monotonic())
event = self._deserialize(message)
self._logger.debug("Got event: %r", event)
if event[u"environment"] == self._environment:
try:
self._lifecycle.initiate(
app_name=event[u"project_name"],
namespace=DEFAULT_NAMESPACE,
deployment_id=self._deployment_id(event))
app_spec = self._create_spec(event)
set_extras(app_spec)
self._check_app_acceptable(app_spec)
self._add_deployment_label(app_spec)
self._deploy_queue.put(DeployerEvent("UPDATE", app_spec))
self._reporter.register(app_spec, event[u"callback_url"])
deploy_counter.inc()
except (NoDockerArtifactException, NoFiaasArtifactException):
self._logger.debug("Ignoring event %r with missing artifacts",
event)
except YAMLError:
self._logger.exception("Failure when parsing FIAAS-config")
self._lifecycle.failed(
app_name=event[u"project_name"],
namespace=DEFAULT_NAMESPACE,
deployment_id=self._deployment_id(event))
except InvalidConfiguration:
self._logger.exception(
"Invalid configuration for application %s",
event.get("project_name"))
self._lifecycle.failed(
app_name=event[u"project_name"],
namespace=DEFAULT_NAMESPACE,
deployment_id=self._deployment_id(event))
except HTTPError:
self._logger.exception("Failure when downloading FIAAS-config")
self._lifecycle.failed(
app_name=event[u"project_name"],
namespace=DEFAULT_NAMESPACE,
deployment_id=self._deployment_id(event))
except (NotWhiteListedApplicationException,
BlackListedApplicationException) as e:
self._logger.warn("App not deployed. %s", str(e))
def __init__(self, app_spec, bookkeeper, lifecycle):
self._app_spec = app_spec
self._bookkeeper = bookkeeper
self._lifecycle = lifecycle
self._fail_after_seconds = FAIL_LIMIT_MULTIPLIER * app_spec.replicas * app_spec.health_checks.readiness.initial_delay_seconds
self._fail_after = time_monotonic() + self._fail_after_seconds
