class WatchedPullRequest(models.Model): """ Represents a single watched pull request; holds the ID of the Instance created for that PR, if any """ # TODO: Remove 'ref_type' ? # TODO: Remove parameters from 'update_instance_from_pr'; make it fetch PR details from the # api (including the head commit sha hash, which does not require a separate API call as # is currently used.) branch_name = models.CharField(max_length=50, default='master') ref_type = models.CharField(max_length=50, default='heads') github_organization_name = models.CharField(max_length=200, db_index=True) github_repository_name = models.CharField(max_length=200, db_index=True) github_pr_url = models.URLField(blank=False) instance = models.OneToOneField('instance.OpenEdXInstance', null=True, blank=True, on_delete=models.SET_NULL) objects = WatchedPullRequestQuerySet.as_manager() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.logger = ModelLoggerAdapter(logger, {'obj': self}) @property def fork_name(self): """ Fork name (eg. 'open-craft/edx-platform') """ return '{0.github_organization_name}/{0.github_repository_name}'.format(self) @property def reference_name(self): """ A descriptive name for the PR, which includes meaningful attributes """ return '{0.github_organization_name}/{0.branch_name}'.format(self) @property def github_base_url(self): """ Base GitHub URL of the fork (eg. 'https://github.com/open-craft/edx-platform') """ return 'https://github.com/{0.fork_name}'.format(self) @property def github_branch_url(self): """ GitHub URL of the branch tree """ return '{0.github_base_url}/tree/{0.branch_name}'.format(self) @property def github_pr_number(self): """ Get the PR number from the URL of the PR. """ if not self.github_pr_url: return None return int(self.github_pr_url.split('/')[-1]) @property def target_fork_name(self): """ Get the full name of the target repo/fork (e.g. 'edx/edx-platform') """ # Split up a URL like https://github.com/edx/edx-platform/pull/12345678 org, repo, pull, dummy = self.github_pr_url.split('/')[-4:] assert pull == "pull" return "{}/{}".format(org, repo) @property def repository_url(self): """ URL of the git repository (eg. 'https://github.com/open-craft/edx-platform.git') """ return '{0.github_base_url}.git'.format(self) @property def updates_feed(self): """ RSS/Atom feed of commits made on the repository/branch """ return '{0.github_base_url}/commits/{0.branch_name}.atom'.format(self) def get_log_message_annotation(self): """ Format a log message annotation for this PR. """ if self.instance: return self.instance.get_log_message_annotation() return None def get_branch_tip(self): """ Get the `commit_id` of the current tip of the branch """ self.logger.info('Fetching commit ID of the tip of branch %s', self.branch_name) try: new_commit_id = github.get_commit_id_from_ref( self.fork_name, self.branch_name, ref_type=self.ref_type) except github.ObjectDoesNotExist: self.logger.error("Branch '%s' not found. Has it been deleted on GitHub?", self.branch_name) raise return new_commit_id def set_fork_name(self, fork_name): """ Set the organization and repository based on the GitHub fork name """ assert not self.github_organization_name assert not self.github_repository_name self.logger.info('Setting fork name: %s', fork_name) fork_org, fork_repo = github.fork_name2tuple(fork_name) self.github_organization_name = fork_org self.github_repository_name = fork_repo def update_instance_from_pr(self, pr): """ Update/create the associated sandbox instance with settings from the given pull request. This will not spawn a new AppServer. This method will automatically save this WatchedPullRequest's 'instance' field. """ # The following fields should never change: assert self.github_pr_url == pr.github_pr_url assert self.fork_name == pr.fork_name assert self.branch_name == pr.branch_name # Create an instance if necessary: instance = self.instance or OpenEdXInstance() instance.internal_lms_domain = generate_internal_lms_domain('pr{number}.sandbox'.format(number=pr.number)) instance.edx_platform_repository_url = self.repository_url instance.edx_platform_commit = self.get_branch_tip() instance.name = ( 'PR#{pr.number}: {pr.truncated_title} ({pr.username}) - {i.reference_name} ({commit_short_id})' .format(pr=pr, i=self, commit_short_id=instance.edx_platform_commit[:7]) ) instance.configuration_extra_settings = pr.extra_settings instance.use_ephemeral_databases = pr.use_ephemeral_databases(instance.domain) instance.configuration_source_repo_url = pr.get_extra_setting( 'edx_ansible_source_repo', default=instance.configuration_source_repo_url ) instance.configuration_version = pr.get_extra_setting( 'configuration_version', default=instance.configuration_version ) # Save atomically. (because if the instance gets created but self.instance failed to # update, then any subsequent call to update_instance_from_pr() would try to create # another instance, which would fail due to unique domain name constraints.) with transaction.atomic(): instance.save() if not self.instance: self.instance = instance self.save(update_fields=["instance"]) # pylint: disable=no-member
class LoadBalancingServer(ValidateModelMixin, TimeStampedModel): """ A model representing a configured load-balancing server. """ objects = LoadBalancingServerManager() domain = models.CharField(max_length=100, unique=True) # The username used to ssh into the server ssh_username = models.CharField(max_length=32) # Whether new backends can be assigned to this load-balancing server accepts_new_backends = models.BooleanField(default=True) # A random postfix appended to the haproxy configuration file names to make collisions # impossible. fragment_name_postfix = models.CharField(max_length=8, blank=True) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.logger = ModelLoggerAdapter(logger, {'obj': self}) def set_field_defaults(self): if not self.fragment_name_postfix: # Set a unique fragment_name_postfix to avoid clashes between multiple instance # managers sharing the same load balancer. bits = self._meta.get_field("fragment_name_postfix").max_length * 4 self.fragment_name_postfix = format(random.getrandbits(bits), "x") super().set_field_defaults() def __str__(self): return self.domain def get_log_message_annotation(self): """ Annotate log messages for the load-balancing server. """ return "load_balancer={} ({!s:.15})".format(self.pk, self.domain) def get_instances(self): """ Yield all instances configured to use this load balancer. """ # Local import due to avoid problems with circular dependencies. from instance.models.mixins.load_balanced import LoadBalancedInstance for field in self._meta.get_fields(): if field.one_to_many and issubclass(field.related_model, LoadBalancedInstance): yield from getattr(self, field.get_accessor_name()).iterator() def get_configuration(self, triggering_instance_id=None): """ Collect the backend maps and configuration fragments from all associated instances. This function also appends fragment_name_postfix to all backend names to avoid name clashes between multiple instance managers using the same load balancer (e.g. for the integration tests). The triggering_instance_id indicates the id of the instance reference that initiated the reconfiguration of the load balancer. """ backend_map = [] backend_conf = [] for instance in self.get_instances(): triggered_by_instance = instance.ref.pk == triggering_instance_id map_entries, conf_entries = instance.get_load_balancer_configuration( triggered_by_instance ) backend_map.extend( " ".join([domain.lower(), backend + self.fragment_name_postfix]) for domain, backend in map_entries ) backend_conf.extend( "backend {}\n{}\n".format(backend + self.fragment_name_postfix, conf) for backend, conf in conf_entries ) return "\n".join(backend_map), "\n".join(backend_conf) def get_ansible_vars(self, triggering_instance_id=None): """ Render the configuration script to be executed on the load balancer. The triggering_instance_id indicates the id of the instance reference that initiated the reconfiguration of the load balancer. """ backend_map, backend_conf = self.get_configuration(triggering_instance_id) fragment_name = settings.LOAD_BALANCER_FRAGMENT_NAME_PREFIX + self.fragment_name_postfix return ( "FRAGMENT_NAME: {fragment_name}\n" "BACKEND_CONFIG_FRAGMENT: |\n" "{backend_conf}\n" "BACKEND_MAP_FRAGMENT: |\n" "{backend_map}\n" ).format( fragment_name=fragment_name, backend_conf=textwrap.indent(backend_conf, " "), backend_map=textwrap.indent(backend_map, " "), ) def run_playbook(self, ansible_vars): """ Run the playbook to perform the server reconfiguration. This is factored out into a separate method so it can be mocked out in the tests. """ playbook_path = pathlib.Path(settings.SITE_ROOT) / "playbooks/load_balancer_conf/load_balancer_conf.yml" with cache.lock("load_balancer_reconfigure:{}".format(self.domain), timeout=900): returncode = ansible.capture_playbook_output( requirements_path=str(playbook_path.parent / "requirements.txt"), inventory_str=self.domain, vars_str=ansible_vars, playbook_path=str(playbook_path), username=self.ssh_username, logger_=self.logger, ) if returncode != 0: self.logger.error("Playbook to reconfigure load-balancing server %s failed.", self) raise ReconfigurationFailed def reconfigure(self, triggering_instance_id=None): """ Regenerate the configuration fragments on the load-balancing server. The triggering_instance_id indicates the id of the instance reference that initiated the reconfiguration of the load balancer. """ self.logger.info("Reconfiguring load-balancing server %s", self.domain) self.run_playbook(self.get_ansible_vars(triggering_instance_id)) def deconfigure(self): """ Remove the configuration fragment from the load-balancing server. """ fragment_name = settings.LOAD_BALANCER_FRAGMENT_NAME_PREFIX + self.fragment_name_postfix self.run_playbook( "FRAGMENT_NAME: {fragment_name}\nREMOVE_FRAGMENT: True".format(fragment_name=fragment_name) ) def delete(self, *args, **kwargs): """ Delete the LoadBalancingServer from the database. """ self.deconfigure() super().delete(*args, **kwargs) # pylint: disable=no-member
class LoadBalancingServer(ValidateModelMixin, TimeStampedModel): """ A model representing a configured load-balancing server. """ objects = LoadBalancingServerManager() domain = models.CharField(max_length=100, unique=True) # The username used to ssh into the server ssh_username = models.CharField(max_length=32) # Whether new backends can be assigned to this load-balancing server accepts_new_backends = models.BooleanField(default=True) # A random postfix appended to the haproxy configuration file names to avoid clashes between # multiple instance managers (or multiple concurrently running integration tests) sharing the # same load balancer. fragment_name_postfix = models.CharField( max_length=8, blank=True, default=functools.partial(generate_fragment_name, length=8), ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.logger = ModelLoggerAdapter(logger, {'obj': self}) def __str__(self): return self.domain def get_log_message_annotation(self): """ Annotate log messages for the load-balancing server. """ return "load_balancer={} ({!s:.15})".format(self.pk, self.domain) def get_instances(self): """ Yield all instances configured to use this load balancer. """ # Local import due to avoid problems with circular dependencies. from instance.models.mixins.load_balanced import LoadBalancedInstance for field in self._meta.get_fields(): if field.one_to_many and issubclass(field.related_model, LoadBalancedInstance): yield from getattr(self, field.get_accessor_name()).iterator() def get_configuration(self, triggering_instance_id=None): """ Collect the backend maps and configuration fragments from all associated instances. This function also appends fragment_name_postfix to all backend names to avoid name clashes between multiple instance managers using the same load balancer (e.g. for the integration tests). The triggering_instance_id indicates the id of the instance reference that initiated the reconfiguration of the load balancer. """ backend_map = [] backend_conf = [] for instance in self.get_instances(): triggered_by_instance = instance.ref.pk == triggering_instance_id map_entries, conf_entries = instance.get_load_balancer_configuration( triggered_by_instance ) backend_map.extend( " ".join([domain.lower(), backend + self.fragment_name_postfix]) for domain, backend in map_entries ) backend_conf.extend( "backend {}\n{}\n".format(backend + self.fragment_name_postfix, conf) for backend, conf in conf_entries ) return "\n".join(backend_map), "\n".join(backend_conf) def get_ansible_vars(self, triggering_instance_id=None): """ Render the configuration script to be executed on the load balancer. The triggering_instance_id indicates the id of the instance reference that initiated the reconfiguration of the load balancer. """ backend_map, backend_conf = self.get_configuration(triggering_instance_id) fragment_name = settings.LOAD_BALANCER_FRAGMENT_NAME_PREFIX + self.fragment_name_postfix return ( "FRAGMENT_NAME: {fragment_name}\n" "BACKEND_CONFIG_FRAGMENT: |\n" "{backend_conf}\n" "BACKEND_MAP_FRAGMENT: |\n" "{backend_map}\n" ).format( fragment_name=fragment_name, backend_conf=textwrap.indent(backend_conf, " "), backend_map=textwrap.indent(backend_map, " "), ) def run_playbook(self, ansible_vars): """ Run the playbook to perform the server reconfiguration. This is factored out into a separate method so it can be mocked out in the tests. """ playbook_path = pathlib.Path(settings.SITE_ROOT) / "playbooks/load_balancer_conf/load_balancer_conf.yml" returncode = ansible.capture_playbook_output( requirements_path=str(playbook_path.parent / "requirements.txt"), inventory_str=self.domain, vars_str=ansible_vars, playbook_path=str(playbook_path), username=self.ssh_username, logger_=self.logger, ) if returncode != 0: self.logger.error("Playbook to reconfigure load-balancing server %s failed.", self) raise ReconfigurationFailed def _configuration_lock(self): """ A Redis lock to protect reconfigurations of this load balancer instance. """ return cache.lock("load_balancer_reconfigure:{}".format(self.domain), timeout=900) def reconfigure(self, triggering_instance_id=None): """ Regenerate the configuration fragments on the load-balancing server. The triggering_instance_id indicates the id of the instance reference that initiated the reconfiguration of the load balancer. """ self.logger.info("Reconfiguring load-balancing server %s", self.domain) with self._configuration_lock(): self.run_playbook(self.get_ansible_vars(triggering_instance_id)) def deconfigure(self): """ Remove the configuration fragment from the load-balancing server. """ fragment_name = settings.LOAD_BALANCER_FRAGMENT_NAME_PREFIX + self.fragment_name_postfix with self._configuration_lock(): self.run_playbook( "FRAGMENT_NAME: {fragment_name}\nREMOVE_FRAGMENT: True".format(fragment_name=fragment_name) ) def delete(self, *args, **kwargs): """ Delete the LoadBalancingServer from the database. """ self.deconfigure() super().delete(*args, **kwargs)
class WatchedPullRequest(models.Model): """ Represents a single watched pull request; holds the ID of the Instance created for that PR, if any """ # TODO: Remove 'ref_type' ? # TODO: Remove parameters from 'update_instance_from_pr'; make it fetch PR details from the # api (including the head commit sha hash, which does not require a separate API call as # is currently used.) watched_fork = models.ForeignKey(WatchedFork, blank=True, null=True, on_delete=models.CASCADE) branch_name = models.CharField(max_length=255, default='master') ref_type = models.CharField(max_length=50, default='heads') github_organization_name = models.CharField(max_length=200, db_index=True) github_repository_name = models.CharField(max_length=200, db_index=True) github_pr_url = models.URLField(blank=False) instance = models.OneToOneField('instance.OpenEdXInstance', null=True, blank=True, on_delete=models.SET_NULL) objects = WatchedPullRequestQuerySet.as_manager() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.logger = ModelLoggerAdapter(logger, {'obj': self}) @property def fork_name(self): """ Fork name (eg. 'open-craft/edx-platform') """ return '{0.github_organization_name}/{0.github_repository_name}'.format( self) @property def reference_name(self): """ A descriptive name for the PR, which includes meaningful attributes """ return '{0.github_organization_name}/{0.branch_name}'.format(self) @property def github_base_url(self): """ Base GitHub URL of the fork (eg. 'https://github.com/open-craft/edx-platform') """ return 'https://github.com/{0.fork_name}'.format(self) @property def github_branch_url(self): """ GitHub URL of the branch tree """ return '{0.github_base_url}/tree/{0.branch_name}'.format(self) @property def github_pr_number(self): """ Get the PR number from the URL of the PR. """ if not self.github_pr_url: return None return int(self.github_pr_url.split('/')[-1]) @property def target_fork_name(self): """ Get the full name of the target repo/fork (e.g. 'edx/edx-platform') """ # Split up a URL like https://github.com/edx/edx-platform/pull/12345678 org, repo, pull, dummy = self.github_pr_url.split('/')[-4:] assert pull == "pull" return "{}/{}".format(org, repo) @property def repository_url(self): """ URL of the git repository (eg. 'https://github.com/open-craft/edx-platform.git') """ return '{0.github_base_url}.git'.format(self) @property def updates_feed(self): """ RSS/Atom feed of commits made on the repository/branch """ return '{0.github_base_url}/commits/{0.branch_name}.atom'.format(self) def get_log_message_annotation(self): """ Format a log message annotation for this PR. """ if self.instance: return self.instance.get_log_message_annotation() return None def get_branch_tip(self): """ Get the `commit_id` of the current tip of the branch """ self.logger.info('Fetching commit ID of the tip of branch %s', self.branch_name) try: new_commit_id = github.get_commit_id_from_ref( self.fork_name, self.branch_name, ref_type=self.ref_type) except github.ObjectDoesNotExist: self.logger.error( "Branch '%s' not found. Has it been deleted on GitHub?", self.branch_name) raise return new_commit_id def set_fork_name(self, fork_name): """ Set the organization and repository based on the GitHub fork name """ assert not self.github_organization_name assert not self.github_repository_name self.logger.info('Setting fork name: %s', fork_name) fork_org, fork_repo = github.fork_name2tuple(fork_name) self.github_organization_name = fork_org self.github_repository_name = fork_repo def update_instance_from_pr(self, pr): """ Update/create the associated sandbox instance with settings from the given pull request. This will not spawn a new AppServer. This method will automatically save this WatchedPullRequest's 'instance' field. """ # The following fields should never change: assert self.github_pr_url == pr.github_pr_url assert self.fork_name == pr.fork_name assert self.branch_name == pr.branch_name # Create an instance if necessary: instance = self.instance or OpenEdXInstance() is_external_pr = self.watched_fork is None instance.internal_lms_domain = generate_internal_lms_domain( '{prefix}pr{number}.sandbox'.format( prefix='ext' if is_external_pr else '', number=pr.number, )) instance.edx_platform_repository_url = self.repository_url instance.edx_platform_commit = self.get_branch_tip() instance.name = ( '{prefix}PR#{pr.number}: {pr.truncated_title} ({pr.username}) - {i.reference_name} ({commit_short_id})' .format( pr=pr, i=self, commit_short_id=instance.edx_platform_commit[:7], prefix='EXT' if is_external_pr else '', )) if is_external_pr: instance.configuration_extra_settings = pr.extra_settings else: instance.configuration_extra_settings = yaml_merge( self.watched_fork.configuration_extra_settings, pr.extra_settings) if not instance.ref.creator or not instance.ref.owner: try: user = UserProfile.objects.get(github_username=pr.username) instance.ref.creator = user instance.ref.owner = user.organization except UserProfile.DoesNotExist: # PR is not associated with an Ocim user pass # Configuration repo and version and edx release follow this precedence: # 1) PR settings. 2) WatchedFork settings. 3) instance model defaults instance.configuration_source_repo_url = pr.get_extra_setting( 'edx_ansible_source_repo', default=((self.watched_fork and self.watched_fork.configuration_source_repo_url) or instance.configuration_source_repo_url)) instance.configuration_version = pr.get_extra_setting( 'configuration_version', default=((self.watched_fork and self.watched_fork.configuration_version) or instance.configuration_version)) instance.openedx_release = pr.get_extra_setting( 'openedx_release', default=((self.watched_fork and self.watched_fork.openedx_release) or instance.openedx_release)) # Save atomically. (because if the instance gets created but self.instance failed to # update, then any subsequent call to update_instance_from_pr() would try to create # another instance, which would fail due to unique domain name constraints.) with transaction.atomic(): instance.save() if not self.instance: self.instance = instance self.save(update_fields=["instance"])
class LoadBalancingServer(ValidateModelMixin, TimeStampedModel): """ A model representing a configured load-balancing server. """ objects = LoadBalancingServerManager() domain = models.CharField(max_length=100, unique=True) ssh_username = models.CharField( max_length=32, help_text='The username used to SSH into the server.' ) accepts_new_backends = models.BooleanField( default=False, help_text='Whether new backends can be assigned to this load-balancing server.' ) fragment_name_postfix = models.CharField( max_length=8, blank=True, default=functools.partial(generate_fragment_name, length=8), help_text=( 'A random postfix appended to the haproxy configuration file names to avoid clashes between ' 'multiple instance managers (or multiple concurrently running integration tests) sharing the ' 'same load balancer.' ) ) configuration_version = models.PositiveIntegerField( default=1, help_text=( 'The current version of configuration for this load balancer. ' 'The version value is the total number of requests ever made to reconfigure the load balancer.' ) ) deployed_configuration_version = models.PositiveIntegerField( default=1, help_text=( 'The currently active configuration version of the load balancer. ' 'If it is less than the configuration version, the load balancer is dirty. ' 'If it is equal to it, then no new reconfiguration is currently required.' ) ) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.logger = ModelLoggerAdapter(logger, {'obj': self}) def __str__(self): return self.domain def get_log_message_annotation(self): """ Annotate log messages for the load-balancing server. """ return "load_balancer={} ({!s:.15})".format(self.pk, self.domain) def get_instances(self): """ Yield all instances configured to use this load balancer. """ # Local import due to avoid problems with circular dependencies. from instance.models.mixins.load_balanced import LoadBalancedInstance for field in self._meta.get_fields(): if field.one_to_many and issubclass(field.related_model, LoadBalancedInstance): yield from getattr(self, field.get_accessor_name()).iterator() def get_configuration(self, triggering_instance_id=None): """ Collect the backend maps and configuration fragments from all associated instances. This function also appends fragment_name_postfix to all backend names to avoid name clashes between multiple instance managers using the same load balancer (e.g. for the integration tests). The triggering_instance_id indicates the id of the instance reference that initiated the reconfiguration of the load balancer. """ backend_map = [] backend_conf = [] for instance in self.get_instances(): triggered_by_instance = instance.ref.pk == triggering_instance_id map_entries, conf_entries = instance.get_load_balancer_configuration( triggered_by_instance ) backend_map.extend( " ".join([domain.lower(), backend + self.fragment_name_postfix]) for domain, backend in map_entries ) backend_conf.extend( "backend {}\n{}\n".format(backend + self.fragment_name_postfix, conf) for backend, conf in conf_entries ) return "\n".join(backend_map), "\n".join(backend_conf) def get_ansible_vars(self, triggering_instance_id=None): """ Render the configuration script to be executed on the load balancer. The triggering_instance_id indicates the id of the instance reference that initiated the reconfiguration of the load balancer. """ backend_map, backend_conf = self.get_configuration(triggering_instance_id) fragment_name = settings.LOAD_BALANCER_FRAGMENT_NAME_PREFIX + self.fragment_name_postfix return ( "FRAGMENT_NAME: {fragment_name}\n" "BACKEND_CONFIG_FRAGMENT: |\n" "{backend_conf}\n" "BACKEND_MAP_FRAGMENT: |\n" "{backend_map}\n" ).format( fragment_name=fragment_name, backend_conf=textwrap.indent(backend_conf, " "), backend_map=textwrap.indent(backend_map, " "), ) def run_playbook(self, ansible_vars): """ Run the playbook to perform the server reconfiguration. This is factored out into a separate method so it can be mocked out in the tests. """ playbook_path = pathlib.Path(settings.SITE_ROOT) / "playbooks/load_balancer_conf/load_balancer_conf.yml" returncode = ansible.capture_playbook_output( requirements_path=str(playbook_path.parent / "requirements.txt"), inventory_str=self.domain, vars_str=ansible_vars, playbook_path=str(playbook_path), username=self.ssh_username, logger_=self.logger, ) if returncode != 0: self.logger.error("Playbook to reconfigure load-balancing server %s failed.", self) raise ReconfigurationFailed def reconfigure(self, triggering_instance_id=None, mark_dirty=True): """ Regenerate the configuration fragments on the load-balancing server. The triggering_instance_id indicates the id of the instance reference that initiated the reconfiguration of the load balancer. The mark_dirty flag indicates whether the LB configuration should be marked as dirty. If this method is called because the configuration changed, the flag should be set to True (the default). If this method is called because the LB was marked dirty earlier, the flag should be set to False. """ if mark_dirty: # We need to use an F expression here. The problem is not other processes trying to # increase this counter concurrently – that wouldn't matter, since we don't care whether # we increase this counter by one or by two, since both marks the LB as dirty. However, if # another process is making a completely unrelated change to the LB object we might lose # the increment altogether. LoadBalancingServer.objects.filter(pk=self.pk).update( configuration_version=models.F("configuration_version") + 1 ) try: with self._configuration_lock(blocking=False): # Memorize the configuration version, in case new threads change it. self.refresh_from_db() candidate_configuration_version = self.configuration_version self.logger.info("Reconfiguring load-balancing server %s", self.domain) self.run_playbook(self.get_ansible_vars(triggering_instance_id)) LoadBalancingServer.objects.filter(pk=self.pk).update( deployed_configuration_version=candidate_configuration_version ) self.refresh_from_db() except OtherReconfigurationInProgress: pass def deconfigure(self): """ Remove the configuration fragment from the load-balancing server. """ self.logger.info("Deconfiguring load-balancing server %s", self.domain) fragment_name = settings.LOAD_BALANCER_FRAGMENT_NAME_PREFIX + self.fragment_name_postfix with self._configuration_lock(): self.run_playbook( "FRAGMENT_NAME: {fragment_name}\nREMOVE_FRAGMENT: True".format(fragment_name=fragment_name) ) def delete(self, *args, **kwargs): """ Delete the LoadBalancingServer from the database. """ self.deconfigure() super().delete(*args, **kwargs) @contextlib.contextmanager def _configuration_lock(self, *, blocking=True): """ A Redis lock to protect reconfigurations of this load balancer instance. """ lock = cache.lock( "load_balancer_reconfigure:{}".format(self.domain), timeout=settings.REDIS_LOCK_TIMEOUT, ) if not lock.acquire(blocking): raise OtherReconfigurationInProgress try: yield lock finally: lock.release()
class Server(ValidateModelMixin, TimeStampedModel): """ A single server VM """ name_prefix = models.SlugField(max_length=20, blank=False) Status = Status status = ModelResourceStateDescriptor(state_classes=Status.states, default_state=Status.Pending, model_field_name='_status') _status = models.CharField( max_length=20, default=status.default_state_class.state_id, choices=status.model_field_choices, db_index=True, db_column='status', ) # State transitions: _status_to_building = status.transition(from_states=(Status.Pending, Status.Unknown), to_state=Status.Building) _status_to_build_failed = status.transition(from_states=(Status.Building, Status.Unknown), to_state=Status.BuildFailed) _status_to_booting = status.transition(from_states=(Status.Building, Status.Ready, Status.Unknown), to_state=Status.Booting) _status_to_ready = status.transition(from_states=(Status.Booting, Status.Unknown), to_state=Status.Ready) _status_to_terminated = status.transition(to_state=Status.Terminated) _status_to_unknown = status.transition(from_states=(Status.Building, Status.Booting, Status.Ready), to_state=Status.Unknown) objects = ServerQuerySet().as_manager() class Meta: abstract = True def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.logger = ModelLoggerAdapter(logger, {'obj': self}) @property def name(self): """ Get a name for this server (slug-friendly) """ assert self.id is not None return "{prefix}-{num}".format(prefix=self.name_prefix, num=self.id) @property def event_context(self): """ Context dictionary to include in events """ return {'server_id': self.pk} def get_log_message_annotation(self): """ Format a log line annotation for this server. """ return 'server={} ({!s:.20})'.format(self.pk, self.name) def sleep_until(self, condition, timeout=3600): """ Sleep in a loop until condition related to server status is fulfilled, or until timeout (provided in seconds) is reached. Raises an exception if the desired condition can not be fulfilled. This can happen if the server is in a steady state (i.e., a state that is not expected to change) that does not fulfill the desired condition. The default timeout is 1h. Use as follows: server.sleep_until(lambda: server.status.is_steady_state) server.sleep_until(lambda: server.status.accepts_ssh_commands) """ # Check if we received a valid timeout value # to avoid the possibility of entering an infinite loop (if timeout is negative) # or reaching the timeout right away (if timeout is zero) assert timeout > 0, "Timeout must be greater than 0 to be able to do anything useful" self.logger.info( 'Waiting to reach status from which we can proceed...') while timeout > 0: self.update_status() if condition(): self.logger.info( 'Reached appropriate status ({name}). Proceeding.'.format( name=self.status.name)) return else: if self.status.is_steady_state: raise SteadyStateException( "The current status ({name}) does not fulfill the desired condition " "and is not expected to change.".format( name=self.status.name)) time.sleep(1) timeout -= 1 # If we get here, this means we've reached the timeout raise TimeoutError( "Waited {minutes:.2f} to reach appropriate status, and got nowhere. " "Aborting with a status of {status}.".format( minutes=timeout / 60, status=self.status.name)) def save(self, *args, **kwargs): """ Save this Server """ super().save(*args, **kwargs) publish_data('notification', { 'type': 'server_update', 'server_pk': self.pk, }) def update_status(self): """ Check the current status and update it if it has changed """ raise NotImplementedError
class Server(ValidateModelMixin, TimeStampedModel): """ A single server VM """ name_prefix = models.SlugField(max_length=20, blank=False) Status = Status status = ModelResourceStateDescriptor( state_classes=Status.states, default_state=Status.Pending, model_field_name="_status" ) _status = models.CharField( max_length=20, default=status.default_state_class.state_id, choices=status.model_field_choices, db_index=True, db_column="status", ) # State transitions: _status_to_building = status.transition(from_states=(Status.Pending, Status.Unknown), to_state=Status.Building) _status_to_build_failed = status.transition( from_states=(Status.Building, Status.Unknown), to_state=Status.BuildFailed ) _status_to_booting = status.transition( from_states=(Status.Building, Status.Ready, Status.Unknown), to_state=Status.Booting ) _status_to_ready = status.transition(from_states=(Status.Booting, Status.Unknown), to_state=Status.Ready) _status_to_terminated = status.transition(to_state=Status.Terminated) _status_to_unknown = status.transition( from_states=(Status.Building, Status.Booting, Status.Ready), to_state=Status.Unknown ) objects = ServerQuerySet().as_manager() class Meta: abstract = True def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.logger = ModelLoggerAdapter(logger, {"obj": self}) @property def name(self): """ Get a name for this server (slug-friendly) """ assert self.id is not None return "{prefix}-{num}".format(prefix=self.name_prefix, num=self.id) @property def event_context(self): """ Context dictionary to include in events """ return {"server_id": self.pk} def get_log_message_annotation(self): """ Format a log line annotation for this server. """ return "server={!s:.20}".format(self.name) def sleep_until(self, condition, timeout=3600): """ Sleep in a loop until condition related to server status is fulfilled, or until timeout (provided in seconds) is reached. Raises an exception if the desired condition can not be fulfilled. This can happen if the server is in a steady state (i.e., a state that is not expected to change) that does not fulfill the desired condition. The default timeout is 1h. Use as follows: server.sleep_until(lambda: server.status.is_steady_state) server.sleep_until(lambda: server.status.accepts_ssh_commands) """ # Check if we received a valid timeout value # to avoid the possibility of entering an infinite loop (if timeout is negative) # or reaching the timeout right away (if timeout is zero) assert timeout > 0, "Timeout must be greater than 0 to be able to do anything useful" self.logger.info("Waiting to reach status from which we can proceed...") while timeout > 0: self.update_status() if condition(): self.logger.info("Reached appropriate status ({name}). Proceeding.".format(name=self.status.name)) return else: if self.status.is_steady_state: raise SteadyStateException( "The current status ({name}) does not fulfill the desired condition " "and is not expected to change.".format(name=self.status.name) ) time.sleep(1) timeout -= 1 # If we get here, this means we've reached the timeout raise TimeoutError( "Waited {minutes:.2f} to reach appropriate status, and got nowhere. " "Aborting with a status of {status}.".format(minutes=timeout / 60, status=self.status.name) ) def save(self, *args, **kwargs): """ Save this Server """ super().save(*args, **kwargs) publish_data("notification", {"type": "server_update", "server_pk": self.pk}) def update_status(self): """ Check the current status and update it if it has changed """ raise NotImplementedError