def _is_receiving_messages(self): # TODO: this is a hack to handle the fact that we sometimes seem to loose contact with kafka self._logger.debug( "No message for %r seconds", int(time_monotonic()) - self._last_message_timestamp) return self._last_message_timestamp > int( time_monotonic()) - ALLOW_WITHOUT_MESSAGES_S
def wait_until(action, description=None, exception_class=AssertionError, patience=60): """Attempt to call 'action' every 2 seconds, until it completes without exception or patience runs out""" __tracebackhide__ = True start = time_monotonic() cause = [] if not description: description = action.__doc__ or action.__name__ message = [] while time_monotonic() < (start + patience): try: action() return except BaseException: cause = traceback.format_exception(*sys.exc_info()) time.sleep(2) if cause: message.append("\nThe last exception was:\n") message.extend(cause) header = "Gave up waiting for {} after {} seconds at {}".format( description, patience, datetime.now().isoformat(" ")) message.insert(0, header) raise exception_class("".join(message))
def start(self): extra_params = ["--keep-context"] if self._k8s_version: extra_params.extend(("--kubernetes-version", self._k8s_version)) extra_params.extend(self._driver.arguments) running = self._attempt_start(extra_params) start = time_monotonic() while not running and time_monotonic() < (start + PATIENCE): running = self._attempt_start(extra_params) if not running: raise MinikubeError("Gave up starting minikube after %d seconds" % PATIENCE)
def __call__(self): repository = _repository(self._app_spec) if self._ready(): self._lifecycle.success( app_name=self._app_spec.name, namespace=self._app_spec.namespace, deployment_id=self._app_spec.deployment_id, repository=repository, labels=self._app_spec.labels.status, annotations=self._app_spec.annotations.status) self._bookkeeper.success(self._app_spec) return False if time_monotonic() >= self._fail_after: LOG.error( "Timed out after %d seconds waiting for %s to become ready", self._fail_after_seconds, self._app_spec.name) self._lifecycle.failed( app_name=self._app_spec.name, namespace=self._app_spec.namespace, deployment_id=self._app_spec.deployment_id, repository=repository, labels=self._app_spec.labels.status, annotations=self._app_spec.annotations.status) self._bookkeeper.failed(self._app_spec) return False return True
def __init__(self, app_spec, bookkeeper, lifecycle, lifecycle_subject, config): self._app_spec = app_spec self._bookkeeper = bookkeeper self._lifecycle = lifecycle self._lifecycle_subject = lifecycle_subject self._fail_after_seconds = _calculate_fail_time( config.ready_check_timeout_multiplier, app_spec.replicas, app_spec.health_checks.readiness.initial_delay_seconds) self._fail_after = time_monotonic() + self._fail_after_seconds
def _wait_for_readiness(self, wait_time_seconds, timeout_seconds): start = time_monotonic() while time_monotonic() < (start + timeout_seconds): if all(status == STATUS_SUCCESS for status in self._status_collector.values()): LOG.info("Bootstrapped {} applications".format( len(self._status_collector.values()))) return True else: time.sleep(wait_time_seconds) message = "Timed out after waiting {}s for applications to become ready.\n".format( timeout_seconds) message += "Applications which failed to become ready:\n" for name, namespace, status in self._status_collector.items(): message += "{} in namespace {} had final state {}\n".format( name, namespace, status) LOG.error(message) return False
def __call__(self): if self._ready(): self._lifecycle.success(self._lifecycle_subject) self._bookkeeper.success(self._app_spec) return False if time_monotonic() >= self._fail_after: LOG.error("Timed out after %d seconds waiting for %s to become ready", self._fail_after_seconds, self._app_spec.name) self._lifecycle.failed(self._lifecycle_subject) self._bookkeeper.failed(self._app_spec) return False return True
def __init__(self, deploy_queue, config, reporter, spec_factory, app_config_downloader, lifecycle): super(Consumer, self).__init__() self._logger = logging.getLogger(__name__) self._deploy_queue = deploy_queue self._consumer = None self._config = config self._environment = config.environment self._reporter = reporter self._spec_factory = spec_factory self._app_config_downloader = app_config_downloader self._lifecycle = lifecycle self._last_message_timestamp = int(time_monotonic())
def test_deployment_failed(self, get, app_spec, bookkeeper, requested, replicas, available, updated, lifecycle, annotations, repository): if annotations: app_spec = app_spec._replace(annotations=LabelAndAnnotationSpec(*[annotations] * 5)) self._create_response(get, requested, replicas, available, updated) ready = ReadyCheck(app_spec, bookkeeper, lifecycle) ready._fail_after = time_monotonic() assert ready() is False bookkeeper.success.assert_not_called() bookkeeper.failed.assert_called_with(app_spec) lifecycle.success.assert_not_called() lifecycle.failed.assert_called_with(app_name=app_spec.name, namespace=app_spec.namespace, deployment_id=app_spec.deployment_id, repository=repository)
def __call__(self): repository = _repository(self._app_spec) if self._ready(): self._lifecycle.success(app_name=self._app_spec.name, namespace=self._app_spec.namespace, deployment_id=self._app_spec.deployment_id, repository=repository) self._bookkeeper.success(self._app_spec) return False if time_monotonic() >= self._fail_after: self._lifecycle.failed(app_name=self._app_spec.name, namespace=self._app_spec.namespace, deployment_id=self._app_spec.deployment_id, repository=repository) self._bookkeeper.failed(self._app_spec) return False return True
def _handle_message(self, deploy_counter, message, message_counter): message_counter.inc() self._last_message_timestamp = int(time_monotonic()) event = self._deserialize(message) self._logger.debug("Got event: %r", event) if event[u"environment"] == self._environment: try: self._lifecycle.initiate( app_name=event[u"project_name"], namespace=DEFAULT_NAMESPACE, deployment_id=self._deployment_id(event)) app_spec = self._create_spec(event) set_extras(app_spec) self._check_app_acceptable(app_spec) self._add_deployment_label(app_spec) self._deploy_queue.put(DeployerEvent("UPDATE", app_spec)) self._reporter.register(app_spec, event[u"callback_url"]) deploy_counter.inc() except (NoDockerArtifactException, NoFiaasArtifactException): self._logger.debug("Ignoring event %r with missing artifacts", event) except YAMLError: self._logger.exception("Failure when parsing FIAAS-config") self._lifecycle.failed( app_name=event[u"project_name"], namespace=DEFAULT_NAMESPACE, deployment_id=self._deployment_id(event)) except InvalidConfiguration: self._logger.exception( "Invalid configuration for application %s", event.get("project_name")) self._lifecycle.failed( app_name=event[u"project_name"], namespace=DEFAULT_NAMESPACE, deployment_id=self._deployment_id(event)) except HTTPError: self._logger.exception("Failure when downloading FIAAS-config") self._lifecycle.failed( app_name=event[u"project_name"], namespace=DEFAULT_NAMESPACE, deployment_id=self._deployment_id(event)) except (NotWhiteListedApplicationException, BlackListedApplicationException) as e: self._logger.warn("App not deployed. %s", str(e))
def __init__(self, app_spec, bookkeeper, lifecycle): self._app_spec = app_spec self._bookkeeper = bookkeeper self._lifecycle = lifecycle self._fail_after_seconds = FAIL_LIMIT_MULTIPLIER * app_spec.replicas * app_spec.health_checks.readiness.initial_delay_seconds self._fail_after = time_monotonic() + self._fail_after_seconds