def autoscale(self, proc_type, autoscale): """ Set autoscale rules for the application """ name = '{}-{}'.format(self.id, proc_type) # basically fake out a Deployment object (only thing we use) to assign to the HPA target = {'apiVersion': 'apps/v1', 'kind': 'Deployment', 'metadata': {'name': name}} try: # get the target for autoscaler, in this case Deployment self._scheduler.hpa.get(self.id, name) if autoscale is None: self._scheduler.hpa.delete(self.id, name) else: self._scheduler.hpa.update( self.id, name, proc_type, target, **autoscale ) except KubeHTTPException as e: if e.response.status_code == 404: self._scheduler.hpa.create( self.id, name, proc_type, target, **autoscale ) else: # let the user know about any other errors raise ServiceUnavailable(str(e)) from e
def run(self, user, command): def pod_name(size=5, chars=string.ascii_lowercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) """Run a one-off command in an ephemeral app container.""" release = self.release_set.latest() if release.build is None: raise DeisException( 'No build associated with this release to run this command') # TODO: add support for interactive shell entrypoint, command = self._get_command_run(command) name = self._get_job_id('run') + '-' + pod_name() self.log("{} on {} runs '{}'".format(user.username, name, command)) kwargs = { 'memory': release.config.memory, 'cpu': release.config.cpu, 'tags': release.config.tags, 'envs': release.config.values, 'registry': release.config.registry, 'version': "v{}".format(release.version), 'build_type': release.build.type, } try: exit_code, output = self._scheduler.run(self.id, name, release.image, entrypoint, command, **kwargs) return exit_code, output except Exception as e: err = '{} (run): {}'.format(name, e) raise ServiceUnavailable(err) from e
def run(self, user, command): def pod_name(size=5, chars=string.ascii_lowercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) """Run a one-off command in an ephemeral app container.""" release = self.release_set.filter(failed=False).latest() if release.build is None: raise DeisException( 'No build associated with this release to run this command') app_settings = self.appsettings_set.latest() # use slugrunner image for app if buildpack app otherwise use normal image image = settings.SLUGRUNNER_IMAGE if release.build.type == 'buildpack' else release.image data = self._gather_app_settings(release, app_settings, process_type='run', replicas=1) # create application config and build the pod manifest self.set_application_config(release) scale_type = 'run' name = self._get_job_id(scale_type) + '-' + pod_name() self.log("{} on {} runs '{}'".format(user.username, name, command)) try: exit_code, output = self._scheduler.run( self.id, name, image, self._get_entrypoint(scale_type), [command], **data) return exit_code, output except Exception as e: err = '{} (run): {}'.format(name, e) raise ServiceUnavailable(err) from e
def _update_application_service(self, namespace, app_type, port, routable=False, annotations={}): # noqa """Update application service with all the various required information""" service = self._fetch_service_config(namespace) old_service = service.copy() # in case anything fails for rollback try: # Update service information for key, value in annotations.items(): if value is not None: service['metadata']['annotations']['router.deis.io/%s' % key] = str(value) else: service['metadata']['annotations'].pop('router.deis.io/%s' % key, None) if routable: service['metadata']['labels']['router.deis.io/routable'] = 'true' else: # delete the annotation service['metadata']['labels'].pop('router.deis.io/routable', None) # Set app type selector service['spec']['selector']['type'] = app_type # Find if target port exists already, update / create as required if routable: for pos, item in enumerate(service['spec']['ports']): if item['port'] == 80 and port != item['targetPort']: # port 80 is the only one we care about right now service['spec']['ports'][pos]['targetPort'] = int(port) self._scheduler.svc.update(namespace, namespace, data=service) except Exception as e: # Fix service to old port and app type self._scheduler.svc.update(namespace, namespace, data=old_service) raise ServiceUnavailable(str(e)) from e
def detach(self, *args, **kwargs): # remove the certificate from the domain domain = get_object_or_404(Domain, domain=kwargs['domain']) domain.certificate = None domain.save() name = '%s-cert' % self.name app = domain.app # only delete if it exists and if no other domains depend on secret if len(self.domains) == 0: try: # We raise an exception when a secret doesn't exist self._scheduler.get_secret(app, name) self._scheduler.delete_secret(app, name) except KubeException as e: raise ServiceUnavailable( "Could not delete certificate secret {} for application {}" .format(name, app)) from e # noqa # get config for the service config = self._load_service_config(app, 'router') # See if certificates are available if 'certificates' not in config: config['certificates'] = '' # convert from string to list to work with and filter out empty strings cert = '{}:{}'.format(domain.domain, self.name) certificates = [_f for _f in config['certificates'].split(',') if _f] if cert in certificates: certificates.remove(cert) config['certificates'] = ','.join(certificates) self._save_service_config(app, 'router', config)
def delete(self, *args, **kwargs): """Delete this application including all containers""" self.log("deleting environment") try: # check if namespace exists self._scheduler.ns.get(self.id) try: self._scheduler.ns.delete(self.id) # wait 30 seconds for termination for _ in range(30): try: self._scheduler.ns.get(self.id) except KubeHTTPException as e: # only break out on a 404 if e.response.status_code == 404: break except KubeException as e: raise ServiceUnavailable( 'Could not delete Kubernetes Namespace {} within 30 seconds' .format(self.id)) from e # noqa except KubeHTTPException: # it's fine if the namespace does not exist - delete app from the DB pass self._clean_app_logs() return super(App, self).delete(*args, **kwargs)
def _scale_pods(self, scale_types): release = self.release_set.filter(failed=False).latest() app_settings = self.appsettings_set.latest() # use slugrunner image for app if buildpack app otherwise use normal image image = settings.SLUGRUNNER_IMAGE if release.build.type == 'buildpack' else release.image tasks = [] for scale_type, replicas in scale_types.items(): data = self._gather_app_settings(release, app_settings, scale_type, replicas) # noqa # gather all proc types to be deployed tasks.append( functools.partial(self._scheduler.scale, namespace=self.id, name=self._get_job_id(scale_type), image=image, entrypoint=self._get_entrypoint(scale_type), command=self._get_command(scale_type), **data)) try: # create the application config in k8s (secret in this case) for all deploy objects self.set_application_config(release) async_run(tasks) except Exception as e: err = '(scale): {}'.format(e) self.log(err, logging.ERROR) raise ServiceUnavailable(err) from e
def get(self, request): try: import django.db with django.db.connection.cursor() as c: c.execute("SELECT 0") except django.db.Error as e: raise ServiceUnavailable("Database health check failed") from e return HttpResponse("OK")
def run(self, user, command): def pod_name(size=5, chars=string.ascii_lowercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) """Run a one-off command in an ephemeral app container.""" scale_type = 'run' release = self.release_set.latest() if release.build is None: raise DeisException( 'No build associated with this release to run this command') image = release.image registry = release.config.registry version = "v{}".format(release.version) envs = self._build_env_vars(release.build.type, version, image, release.config.values) # see if the app config has deploy timeout preference, otherwise use global deploy_timeout = release.config.values.get( 'DEIS_DEPLOY_TIMEOUT', settings.DEIS_DEPLOY_TIMEOUT) # noqa # get application level pod termination grace period pod_termination_grace_period_seconds = release.config.values.get( 'KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS', settings.KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS) # noqa # create image pull secret if needed image_pull_secret_name = self.image_pull_secret( self.id, registry, image) name = self._get_job_id(scale_type) + '-' + pod_name() self.log("{} on {} runs '{}'".format(user.username, name, command)) kwargs = { 'memory': release.config.memory, 'cpu': release.config.cpu, 'tags': release.config.tags, 'envs': envs, 'registry': registry, 'version': version, 'build_type': release.build.type, 'deploy_timeout': deploy_timeout, 'pod_termination_grace_period_seconds': pod_termination_grace_period_seconds, 'image_pull_secret_name': image_pull_secret_name, } try: exit_code, output = self._scheduler.run( self.id, name, image, self._get_entrypoint(scale_type), [command], **kwargs) return exit_code, output except Exception as e: err = '{} (run): {}'.format(name, e) raise ServiceUnavailable(err) from e
def list_pods(self, *args, **kwargs): """Used to list basic information about pods running for a given application""" try: labels = self._scheduler_filter(**kwargs) # in case a singular pod is requested if 'name' in kwargs: pods = [ self._scheduler.pod.get(self.id, kwargs['name']).json() ] else: pods = self._scheduler.pod.get(self.id, labels=labels).json()['items'] if not pods: pods = [] data = [] for p in pods: labels = p['metadata']['labels'] # specifically ignore run pods if labels['type'] == 'run': continue state = str(self._scheduler.pod.state(p)) # follows kubelete convention - these are hidden unless show-all is set if state in ['down', 'crashed']: continue # hide pod if it is passed the graceful termination period if self._scheduler.pod.deleted(p): continue item = Pod() item['name'] = p['metadata']['name'] item['state'] = state item['release'] = labels['version'] item['type'] = labels['type'] if 'startTime' in p['status']: started = p['status']['startTime'] else: started = str(datetime.utcnow().strftime( settings.DEIS_DATETIME_FORMAT)) item['started'] = started data.append(item) # sorting so latest start date is first data.sort(key=lambda x: x['started'], reverse=True) return data except KubeHTTPException as e: pass except Exception as e: err = '(list pods): {}'.format(e) self.log(err, logging.ERROR) raise ServiceUnavailable(err) from e
def _scale_pods(self, scale_types): release = self.release_set.latest() envs = release.config.values # see if the app config has deploy batch preference, otherwise use global batches = release.config.values.get('DEIS_DEPLOY_BATCHES', settings.DEIS_DEPLOY_BATCHES) # see if the app config has deploy timeout preference, otherwise use global deploy_timeout = release.config.values.get( 'DEIS_DEPLOY_TIMEOUT', settings.DEIS_DEPLOY_TIMEOUT) # noqa tasks = [] for scale_type, replicas in scale_types.items(): # only web / cmd are routable # http://docs.deis.io/en/latest/using_deis/process-types/#web-vs-cmd-process-types routable = True if scale_type in ['web', 'cmd'] else False # fetch application port and inject into ENV Vars as needed port = release.get_port() if port: envs['PORT'] = port kwargs = { 'memory': release.config.memory, 'cpu': release.config.cpu, 'tags': release.config.tags, 'envs': envs, 'registry': release.config.registry, 'version': "v{}".format(release.version), 'replicas': replicas, 'app_type': scale_type, 'build_type': release.build.type, 'healthcheck': release.config.healthcheck, 'routable': routable, 'deploy_batches': batches, 'deploy_timeout': deploy_timeout, } # gather all proc types to be deployed tasks.append( functools.partial(self._scheduler.scale, namespace=self.id, name=self._get_job_id(scale_type), image=release.image, entrypoint=self._get_entrypoint(scale_type), command=self._get_command(scale_type), **kwargs)) try: async_run(tasks) except Exception as e: err = '(scale): {}'.format(e) self.log(err, logging.ERROR) raise ServiceUnavailable(err) from e
def create(self, *args, **kwargs): # noqa # create required minimum service in k8s for the application namespace = self._namespace() svc_name = self._svc_name() self.log('creating Service: {}'.format(svc_name), level=logging.DEBUG) try: try: self._scheduler.svc.get(namespace, svc_name) except KubeException: self._scheduler.svc.create(namespace, svc_name) except KubeException as e: raise ServiceUnavailable( 'Kubernetes service could not be created') from e
def routable(self, routable): """ Turn on/off if an application is publically routable """ service = self._fetch_service_config(self.id) old_service = service.copy() # in case anything fails for rollback try: service['metadata']['labels']['router.deis.io/routable'] = str(routable).lower() self._scheduler.svc.update(self.id, self.id, data=service) except KubeException as e: self._scheduler.svc.update(self.id, self.id, data=old_service) raise ServiceUnavailable(str(e)) from e
def maintenance_mode(self, mode): """ Turn application maintenance mode on/off """ service = self._fetch_service_config(self.id) old_service = service.copy() # in case anything fails for rollback try: service['metadata']['annotations'][ 'router.deis.io/maintenance'] = str(mode).lower() self._scheduler.svc.update(self.id, self.id, data=service) except KubeException as e: self._scheduler.svc.update(self.id, self.id, data=old_service) raise ServiceUnavailable(str(e)) from e
def create(self, *args, **kwargs): # noqa # create required minimum service in k8s for the application namespace = self._namespace() svc_name = self._svc_name() self.log('creating Service: {}'.format(svc_name), level=logging.DEBUG) try: try: self._scheduler.svc.get(namespace, svc_name) except KubeException: self._scheduler.svc.create(namespace, svc_name) except KubeException as e: raise ServiceUnavailable('Kubernetes service could not be created') from e # config service annotations = self._gather_settings() routable = annotations.pop('routable') self._update_service(namespace, self.procfile_type, routable, annotations)
def whitelist(self, whitelist): """ Add/ Delete addresses to application whitelist """ service = self._fetch_service_config(self.id) try: if whitelist: addresses = ",".join(address for address in whitelist) service['metadata']['annotations']['router.deis.io/whitelist'] = addresses elif 'router.deis.io/whitelist' in service['metadata']['annotations']: service['metadata']['annotations'].pop('router.deis.io/whitelist', None) else: return self._scheduler.svc.update(self.id, self.id, data=service) except KubeException as e: raise ServiceUnavailable(str(e)) from e
def delete(self, *args, **kwargs): """Delete this application including all containers""" self.log("deleting environment") try: self._scheduler.delete_namespace(self.id) # wait 30 seconds for termination for _ in range(30): try: self._scheduler.get_namespace(self.id) except KubeException: break except KubeException as e: raise ServiceUnavailable( 'Could not delete Kubernetes Namespace {}'.format( self.id)) from e # noqa self._clean_app_logs() return super(App, self).delete(*args, **kwargs)
def detach(self, *args, **kwargs): # remove the certificate from the domain domain = get_object_or_404(Domain, domain=kwargs['domain']) domain.certificate = None domain.save() name = '%s-certificate' % self.name namespace = domain.app.id # only delete if it exists and if no other domains depend on secret if len(self.domains) == 0: try: # We raise an exception when a secret doesn't exist self._scheduler.secret.get(namespace, name) self._scheduler.secret.delete(namespace, name) except KubeException as e: raise ServiceUnavailable( "Could not delete certificate secret {} for application {}" .format(name, namespace)) from e # noqa
def attach_in_kubernetes(self, domain): """Creates the certificate as a kubernetes secret""" # only create if it exists - We raise an exception when a secret doesn't exist try: name = '%s-certificate' % self.name namespace = domain.app.id data = {'tls.crt': self.certificate, 'tls.key': self.key} secret = self._scheduler.secret.get(namespace, name).json()['data'] except KubeException: self._scheduler.secret.create(namespace, name, data) else: # update cert secret to the TLS Ingress format if required if secret != data: try: self._scheduler.secret.update(namespace, name, data) except KubeException as e: msg = 'There was a problem updating the certificate secret ' \ '{} for {}'.format(name, namespace) raise ServiceUnavailable(msg) from e
def maintenance_mode(self, mode): """ Turn service maintenance mode on/off """ namespace = self._namespace() svc_name = self._svc_name() try: service = self._fetch_service_config(namespace, svc_name) except (ServiceUnavailable, KubeException) as e: # ignore non-existing services return old_service = service.copy() # in case anything fails for rollback try: service['metadata']['annotations']['router.deis.io/maintenance'] = str(mode).lower() self._scheduler.svc.update(namespace, svc_name, data=service) except KubeException as e: self._scheduler.svc.update(namespace, svc_name, data=old_service) raise ServiceUnavailable(str(e)) from e
def attach_in_kubernetes(self, domain): """Creates the certificate as a kubernetes secret""" # only create if it exists - We raise an exception when a secret doesn't exist try: name = '%s-cert' % self.name namespace = domain.app.id data = { 'tls.crt': self.certificate, 'tls.key': self.key } secret = self._scheduler.secret.get(namespace, name).json()['data'] except KubeException: self._scheduler.secret.create(namespace, name, data) else: # update cert secret to the TLS Ingress format if required if secret != data: try: self._scheduler.secret.update(namespace, name, data) except KubeException as e: msg = 'There was a problem updating the certificate secret ' \ '{} for {}'.format(name, namespace) raise ServiceUnavailable(msg) from e # get config for the service config = self._load_service_config(namespace, 'router') # See if certificates are available if 'certificates' not in config: config['certificates'] = '' # convert from string to list to work with and filter out empty strings cert = '{}:{}'.format(domain.domain, self.name) certificates = [_f for _f in config['certificates'].split(',') if _f] if cert not in certificates: certificates.append(cert) config['certificates'] = ','.join(certificates) self._save_service_config(namespace, 'router', config)
def run(self, user, command): def pod_name(size=5, chars=string.ascii_lowercase + string.digits): return ''.join(random.choice(chars) for _ in range(size)) """Run a one-off command in an ephemeral app container.""" scale_type = 'run' release = self.release_set.latest() if release.build is None: raise DeisException( 'No build associated with this release to run this command') # see if the app config has deploy timeout preference, otherwise use global deploy_timeout = release.config.values.get( 'DEIS_DEPLOY_TIMEOUT', settings.DEIS_DEPLOY_TIMEOUT) # noqa name = self._get_job_id(scale_type) + '-' + pod_name() self.log("{} on {} runs '{}'".format(user.username, name, command)) kwargs = { 'memory': release.config.memory, 'cpu': release.config.cpu, 'tags': release.config.tags, 'envs': release.config.values, 'registry': release.config.registry, 'version': "v{}".format(release.version), 'build_type': release.build.type, 'deploy_timeout': deploy_timeout } try: exit_code, output = self._scheduler.run( self.id, name, release.image, self._get_entrypoint(scale_type), [command], **kwargs) return exit_code, output except Exception as e: err = '{} (run): {}'.format(name, e) raise ServiceUnavailable(err) from e
def _scale_pods(self, scale_types): release = self.release_set.latest() envs = release.config.values for scale_type, replicas in scale_types.items(): # only web / cmd are routable # http://docs.deis.io/en/latest/using_deis/process-types/#web-vs-cmd-process-types routable = True if scale_type in ['web', 'cmd'] else False # fetch application port and inject into ENV Vars as needed port = release.get_port(routable) if port: envs['PORT'] = port kwargs = { 'memory': release.config.memory, 'cpu': release.config.cpu, 'tags': release.config.tags, 'envs': envs, 'registry': release.config.registry, 'version': "v{}".format(release.version), 'replicas': replicas, 'app_type': scale_type, 'build_type': release.build.type, 'healthcheck': release.config.healthcheck, 'routable': routable } command = self._get_command(scale_type) try: self._scheduler.scale(namespace=self.id, name=self._get_job_id(scale_type), image=release.image, command=command, **kwargs) except Exception as e: err = '{} (scale): {}'.format(self._get_job_id(scale_type), e) self.log(err, logging.ERROR) raise ServiceUnavailable(err) from e
def _scale_pods(self, scale_types): release = self.release_set.latest() app_settings = self.appsettings_set.latest() version = "v{}".format(release.version) image = release.image envs = self._build_env_vars(release.build.type, version, image, release.config.values) registry = release.config.registry # see if the app config has deploy batch preference, otherwise use global batches = release.config.values.get('DEIS_DEPLOY_BATCHES', settings.DEIS_DEPLOY_BATCHES) # see if the app config has deploy timeout preference, otherwise use global deploy_timeout = release.config.values.get( 'DEIS_DEPLOY_TIMEOUT', settings.DEIS_DEPLOY_TIMEOUT) # noqa # get application level pod termination grace period pod_termination_grace_period_seconds = release.config.values.get( 'KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS', settings.KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS) # noqa # create image pull secret if needed image_pull_secret_name = self.image_pull_secret( self.id, registry, image) tasks = [] for scale_type, replicas in scale_types.items(): # only web / cmd are routable # http://docs.deis.io/en/latest/using_deis/process-types/#web-vs-cmd-process-types routable = True if scale_type in [ 'web', 'cmd' ] and app_settings.routable else False # fetch application port and inject into ENV Vars as needed port = release.get_port() if port: envs['PORT'] = port healthcheck = release.config.get_healthcheck().get(scale_type, {}) if not healthcheck and scale_type in ['web', 'cmd']: healthcheck = release.config.get_healthcheck().get( 'web/cmd', {}) kwargs = { 'memory': release.config.memory, 'cpu': release.config.cpu, 'tags': release.config.tags, 'envs': envs, 'registry': registry, 'version': version, 'replicas': replicas, 'app_type': scale_type, 'build_type': release.build.type, 'healthcheck': healthcheck, 'routable': routable, 'deploy_batches': batches, 'deploy_timeout': deploy_timeout, 'pod_termination_grace_period_seconds': pod_termination_grace_period_seconds, 'image_pull_secret_name': image_pull_secret_name, } # gather all proc types to be deployed tasks.append( functools.partial(self._scheduler.scale, namespace=self.id, name=self._get_job_id(scale_type), image=image, entrypoint=self._get_entrypoint(scale_type), command=self._get_command(scale_type), **kwargs)) try: # create the application config in k8s (secret in this case) for all deploy objects self._scheduler.set_application_config(self.id, envs, version) async_run(tasks) except Exception as e: err = '(scale): {}'.format(e) self.log(err, logging.ERROR) raise ServiceUnavailable(err) from e
def deploy(self, release, force_deploy=False): """ Deploy a new release to this application force_deploy can be used when a deployment is broken, such as for Rollback """ if release.build is None: raise DeisException('No build associated with this release') # use create to make sure minimum resources are created self.create() if self.structure == {}: self.structure = self._default_structure(release) self.save() # see if the app config has deploy batch preference, otherwise use global batches = release.config.values.get('DEIS_DEPLOY_BATCHES', settings.DEIS_DEPLOY_BATCHES) # see if the app config has deploy timeout preference, otherwise use global deploy_timeout = release.config.values.get( 'DEIS_DEPLOY_TIMEOUT', settings.DEIS_DEPLOY_TIMEOUT) # noqa deployment_history = release.config.values.get( 'KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT', settings.KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT) # noqa # deploy application to k8s. Also handles initial scaling deploys = {} envs = release.config.values for scale_type, replicas in self.structure.items(): # only web / cmd are routable # http://docs.deis.io/en/latest/using_deis/process-types/#web-vs-cmd-process-types routable = True if scale_type in ['web', 'cmd'] else False # fetch application port and inject into ENV vars as needed port = release.get_port() if port: envs['PORT'] = port deploys[scale_type] = { 'memory': release.config.memory, 'cpu': release.config.cpu, 'tags': release.config.tags, 'envs': envs, 'registry': release.config.registry, # only used if there is no previous RC 'replicas': replicas, 'version': "v{}".format(release.version), 'app_type': scale_type, 'build_type': release.build.type, 'healthcheck': release.config.healthcheck, 'routable': routable, 'deploy_batches': batches, 'deploy_timeout': deploy_timeout, 'deployment_history_limit': deployment_history, 'release_summary': release.summary } # Sort deploys so routable comes first deploys = OrderedDict( sorted(deploys.items(), key=lambda d: d[1].get('routable'))) # Check if any proc type has a Deployment in progress for scale_type, kwargs in deploys.items(): # Is there an existing deployment in progress? name = self._get_job_id(scale_type) if not force_deploy and release.deployment_in_progress( self.id, name): raise AlreadyExists( 'Deployment for {} is already in progress'.format(name)) try: # gather all proc types to be deployed tasks = [ functools.partial(self._scheduler.deploy, namespace=self.id, name=self._get_job_id(scale_type), image=release.image, entrypoint=self._get_entrypoint(scale_type), command=self._get_command(scale_type), **kwargs) for scale_type, kwargs in deploys.items() ] async_run(tasks) except Exception as e: err = '(app::deploy): {}'.format(e) self.log(err, logging.ERROR) raise ServiceUnavailable(err) from e # Wait until application is available in the router # Only run when there is no previous build / release old = release.previous() if old is None or old.build is None: self.verify_application_health(**kwargs) # cleanup old release objects from kubernetes release.cleanup_old()
def create(self, *args, **kwargs): # noqa """ Create a application with an initial config, settings, release, domain and k8s resource if needed """ try: cfg = self.config_set.latest() except Config.DoesNotExist: cfg = Config.objects.create(owner=self.owner, app=self) # Only create if no release can be found try: rel = self.release_set.latest() except Release.DoesNotExist: rel = Release.objects.create(version=1, owner=self.owner, app=self, config=cfg, build=None) # create required minimum resources in k8s for the application namespace = self.id service = self.id try: self.log('creating Namespace {} and services'.format(namespace), level=logging.DEBUG) # Create essential resources try: self._scheduler.ns.get(namespace) except KubeException: self._scheduler.ns.create(namespace) try: self._scheduler.svc.get(namespace, service) except KubeException: self._scheduler.svc.create(namespace, service) except KubeException as e: # Blow it all away only if something horrible happens try: self._scheduler.ns.delete(namespace) except KubeException as e: # Just feed into the item below raise ServiceUnavailable( 'Could not delete the Namespace in Kubernetes') from e raise ServiceUnavailable( 'Kubernetes resources could not be created') from e try: self.appsettings_set.latest() except AppSettings.DoesNotExist: AppSettings.objects.create(owner=self.owner, app=self) try: self.tls_set.latest() except TLS.DoesNotExist: TLS.objects.create(owner=self.owner, app=self) # Attach the platform specific application sub domain to the k8s service # Only attach it on first release in case a customer has remove the app domain if rel.version == 1 and not Domain.objects.filter( domain=self.id).exists(): Domain(owner=self.owner, app=self, domain=self.id).save()
def deploy(self, release, force_deploy=False, rollback_on_failure=True): # noqa """ Deploy a new release to this application force_deploy can be used when a deployment is broken, such as for Rollback """ if release.build is None: raise DeisException('No build associated with this release') app_settings = self.appsettings_set.latest() addresses = ",".join(address for address in app_settings.whitelist) service_annotations = { 'maintenance': app_settings.maintenance, 'whitelist': addresses } # use create to make sure minimum resources are created self.create() if self.structure == {}: self.structure = self._default_structure(release) self.save() image = release.image registry = release.config.registry version = "v{}".format(release.version) envs = self._build_env_vars(release.build.type, version, image, release.config.values) tags = release.config.tags # see if the app config has deploy batch preference, otherwise use global batches = release.config.values.get('DEIS_DEPLOY_BATCHES', settings.DEIS_DEPLOY_BATCHES) # see if the app config has deploy timeout preference, otherwise use global deploy_timeout = release.config.values.get( 'DEIS_DEPLOY_TIMEOUT', settings.DEIS_DEPLOY_TIMEOUT) # noqa deployment_history = release.config.values.get( 'KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT', settings.KUBERNETES_DEPLOYMENTS_REVISION_HISTORY_LIMIT) # noqa # get application level pod termination grace period pod_termination_grace_period_seconds = release.config.values.get( 'KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS', settings.KUBERNETES_POD_TERMINATION_GRACE_PERIOD_SECONDS) # noqa # create image pull secret if needed image_pull_secret_name = self.image_pull_secret( self.id, registry, image) # deploy application to k8s. Also handles initial scaling deploys = {} for scale_type, replicas in self.structure.items(): # only web / cmd are routable # http://docs.deis.io/en/latest/using_deis/process-types/#web-vs-cmd-process-types routable = True if scale_type in [ 'web', 'cmd' ] and app_settings.routable else False # fetch application port and inject into ENV vars as needed port = release.get_port() if port: envs['PORT'] = port healthcheck = release.config.get_healthcheck().get(scale_type, {}) if not healthcheck and scale_type in ['web', 'cmd']: healthcheck = release.config.get_healthcheck().get( 'web/cmd', {}) deploys[scale_type] = { 'memory': release.config.memory, 'cpu': release.config.cpu, 'tags': tags, 'envs': envs, 'registry': registry, 'replicas': replicas, 'version': version, 'app_type': scale_type, 'build_type': release.build.type, 'healthcheck': healthcheck, 'routable': routable, 'deploy_batches': batches, 'deploy_timeout': deploy_timeout, 'deployment_revision_history_limit': deployment_history, 'release_summary': release.summary, 'pod_termination_grace_period_seconds': pod_termination_grace_period_seconds, 'image_pull_secret_name': image_pull_secret_name, } # Sort deploys so routable comes first deploys = OrderedDict( sorted(deploys.items(), key=lambda d: d[1].get('routable'))) # Check if any proc type has a Deployment in progress self._check_deployment_in_progress(deploys, force_deploy) try: # create the application config in k8s (secret in this case) for all deploy objects self._scheduler.set_application_config(self.id, envs, version) # gather all proc types to be deployed tasks = [ functools.partial(self._scheduler.deploy, namespace=self.id, name=self._get_job_id(scale_type), image=image, entrypoint=self._get_entrypoint(scale_type), command=self._get_command(scale_type), **kwargs) for scale_type, kwargs in deploys.items() ] try: async_run(tasks) except KubeException as e: if rollback_on_failure: err = 'There was a problem deploying {}. Rolling back process types to release {}.'.format( version, "v{}".format(release.previous().version)) # noqa # This goes in the log before the rollback starts self.log(err, logging.ERROR) # revert all process types to old release self.deploy(release.previous(), force_deploy=True, rollback_on_failure=False) # let it bubble up raise DeisException('{}\n{}'.format(err, str(e))) from e # otherwise just re-raise raise except Exception as e: # This gets shown to the end user err = '(app::deploy): {}'.format(e) self.log(err, logging.ERROR) raise ServiceUnavailable(err) from e app_type = 'web' if 'web' in deploys else 'cmd' if 'cmd' in deploys else None # Make sure the application is routable and uses the correct port Done after the fact to # let initial deploy settle before routing traffic to the application if deploys and app_type: routable = deploys[app_type].get('routable') port = deploys[app_type].get('envs', {}).get('PORT', None) self._update_application_service(self.id, app_type, port, routable, service_annotations) # noqa # Wait until application is available in the router # Only run when there is no previous build / release old = release.previous() if old is None or old.build is None: self.verify_application_health(**deploys[app_type]) # cleanup old release objects from kubernetes release.cleanup_old()
def deploy(self, release): """Deploy a new release to this application""" if release.build is None: raise DeisException('No build associated with this release') # use create to make sure minimum resources are created self.create() if self.structure == {}: self.structure = self._default_structure(release) self.save() # see if the app config has deploy batch preference, otherwise use global batches = release.config.values.get('DEIS_DEPLOY_BATCHES', settings.DEIS_DEPLOY_BATCHES) # deploy application to k8s. Also handles initial scaling deploys = {} envs = release.config.values for scale_type, replicas in self.structure.items(): # only web / cmd are routable # http://docs.deis.io/en/latest/using_deis/process-types/#web-vs-cmd-process-types routable = True if scale_type in ['web', 'cmd'] else False # fetch application port and inject into ENV vars as needed port = release.get_port(routable) if port: envs['PORT'] = port deploys[scale_type] = { 'memory': release.config.memory, 'cpu': release.config.cpu, 'tags': release.config.tags, 'envs': envs, 'registry': release.config.registry, # only used if there is no previous RC 'replicas': replicas, 'version': "v{}".format(release.version), 'app_type': scale_type, 'build_type': release.build.type, 'healthcheck': release.config.healthcheck, 'routable': routable, 'batches': batches } # Sort deploys so routable comes first deploys = OrderedDict( sorted(deploys.items(), key=lambda d: d[1].get('routable'))) for scale_type, kwargs in deploys.items(): try: self._scheduler.deploy(namespace=self.id, name=self._get_job_id(scale_type), image=release.image, command=self._get_command(scale_type), **kwargs) # Wait until application is available in the router # Only run when there is no previous build / release old = release.previous() if old is None or old.build is None: self.verify_application_health(**kwargs) except Exception as e: err = '{} (app::deploy): {}'.format( self._get_job_id(scale_type), e) self.log(err, logging.ERROR) raise ServiceUnavailable(err) from e # cleanup old releases from kubernetes release.cleanup_old()
def create(self, *args, **kwargs): # noqa """ Create a application with an initial config, settings, release, domain and k8s resource if needed """ try: cfg = self.config_set.latest() except Config.DoesNotExist: cfg = Config.objects.create(owner=self.owner, app=self) # Only create if no release can be found try: rel = self.release_set.latest() except Release.DoesNotExist: rel = Release.objects.create( version=1, owner=self.owner, app=self, config=cfg, build=None ) # create required minimum resources in k8s for the application namespace = self.id ingress = self.id service = self.id quota_name = '{}-quota'.format(self.id) try: self.log('creating Namespace {} and services'.format(namespace), level=logging.DEBUG) # Create essential resources try: self._scheduler.ns.get(namespace) except KubeException: try: self._scheduler.ns.create(namespace, settings.ENABLE_ISTIO_INJECTION) except KubeException as e: raise ServiceUnavailable('Could not create the Namespace in Kubernetes') from e if settings.KUBERNETES_NAMESPACE_DEFAULT_QUOTA_SPEC != '': quota_spec = json.loads(settings.KUBERNETES_NAMESPACE_DEFAULT_QUOTA_SPEC) self.log('creating Quota {} for namespace {}'.format(quota_name, namespace), level=logging.DEBUG) try: self._scheduler.quota.get(namespace, quota_name) except KubeException: self._scheduler.quota.create(namespace, quota_name, data=quota_spec) try: self._scheduler.svc.get(namespace, service) except KubeException: self._scheduler.svc.create(namespace, service) except KubeException as e: # Blow it all away only if something horrible happens try: self._scheduler.ns.delete(namespace) except KubeException as e: # Just feed into the item below raise ServiceUnavailable('Could not delete the Namespace in Kubernetes') from e raise ServiceUnavailable('Kubernetes resources could not be created') from e try: # In order to create an ingress, we must first have a namespace. if settings.EXPERIMENTAL_NATIVE_INGRESS: if ingress == "": raise ServiceUnavailable('Empty hostname') try: self._scheduler.ingress.get(ingress) except KubeException: self.log("creating Ingress {}".format(namespace), level=logging.INFO) self._scheduler.ingress.create(ingress, namespace, settings.EXPERIMENTAL_NATIVE_INGRESS_HOSTNAME, settings.EXPERIMENTAL_NATIVE_INGRESS_CLASS) except KubeException as e: raise ServiceUnavailable('Could not create Ingress in Kubernetes') from e try: self.appsettings_set.latest() except AppSettings.DoesNotExist: AppSettings.objects.create(owner=self.owner, app=self) try: self.tls_set.latest() except TLS.DoesNotExist: TLS.objects.create(owner=self.owner, app=self) # Attach the platform specific application sub domain to the k8s service # Only attach it on first release in case a customer has remove the app domain if rel.version == 1 and not Domain.objects.filter(domain=self.id).exists(): Domain(owner=self.owner, app=self, domain=self.id).save()
def deploy(self, release, force_deploy=False, rollback_on_failure=True): # noqa """ Deploy a new release to this application force_deploy can be used when a deployment is broken, such as for Rollback """ if release.build is None: raise DeisException('No build associated with this release') # use create to make sure minimum resources are created self.create() # set processes structure to default if app is new. if self.structure == {}: self.structure = self._default_structure(release) self.procfile_structure = self._default_structure(release) self.save() # reset canonical process types if build type has changed. else: # find the previous release's build type prev_release = release.previous() if prev_release and prev_release.build: if prev_release.build.type != release.build.type: structure = self.structure.copy() # zero out canonical pod counts for proctype in ['cmd', 'web']: if proctype in structure: structure[proctype] = 0 # update with the default process type. structure.update(self._default_structure(release)) self.structure = structure # if procfile structure exists then we use it if release.build.procfile and \ release.build.sha and not \ release.build.dockerfile: self.procfile_structure = release.build.procfile self.save() # always set the procfile structure for any new release if release.build.procfile: self.procfile_structure = release.build.procfile self.save() # deploy application to k8s. Also handles initial scaling app_settings = self.appsettings_set.latest() deploys = {} for scale_type, replicas in self.structure.items(): deploys[scale_type] = self._gather_app_settings(release, app_settings, scale_type, replicas) # noqa # Sort deploys so routable comes first deploys = OrderedDict(sorted(deploys.items(), key=lambda d: d[1].get('routable'))) # Check if any proc type has a Deployment in progress self._check_deployment_in_progress(deploys, force_deploy) # use slugrunner image for app if buildpack app otherwise use normal image image = settings.SLUGRUNNER_IMAGE if release.build.type == 'buildpack' else release.image try: # create the application config in k8s (secret in this case) for all deploy objects self.set_application_config(release) # only buildpack apps need access to object storage if release.build.type == 'buildpack': self.create_object_store_secret() # gather all proc types to be deployed tasks = [ functools.partial( self._scheduler.deploy, namespace=self.id, name=self._get_job_id(scale_type), image=image, entrypoint=self._get_entrypoint(scale_type), command=self._get_command(scale_type), **kwargs ) for scale_type, kwargs in deploys.items() ] try: async_run(tasks) except KubeException as e: # Don't rollback if the previous release doesn't have a build which means # this is the first build and all the previous releases are just config changes. if rollback_on_failure and release.previous().build is not None: err = 'There was a problem deploying {}. Rolling back process types to release {}.'.format('v{}'.format(release.version), "v{}".format(release.previous().version)) # noqa # This goes in the log before the rollback starts self.log(err, logging.ERROR) # revert all process types to old release self.deploy(release.previous(), force_deploy=True, rollback_on_failure=False) # let it bubble up raise DeisException('{}\n{}'.format(err, str(e))) from e # otherwise just re-raise raise except Exception as e: # This gets shown to the end user err = '(app::deploy): {}'.format(e) self.log(err, logging.ERROR) raise ServiceUnavailable(err) from e app_type = 'web' if 'web' in deploys else 'cmd' if 'cmd' in deploys else None # Make sure the application is routable and uses the correct port done after the fact to # let initial deploy settle before routing traffic to the application if deploys and app_type: app_settings = self.appsettings_set.latest() if app_settings.whitelist: addresses = ",".join(address for address in app_settings.whitelist) else: addresses = None service_annotations = { 'maintenance': app_settings.maintenance, 'whitelist': addresses } routable = deploys[app_type].get('routable') port = deploys[app_type].get('envs', {}).get('PORT', None) self._update_application_service(self.id, app_type, port, routable, service_annotations) # noqa # Wait until application is available in the router # Only run when there is no previous build / release old = release.previous() if old is None or old.build is None: self.verify_application_health(**deploys[app_type]) # cleanup old release objects from kubernetes release.cleanup_old()