class StorageContainer(models.Model): """ Base class selecting the storage type. """ S3_STORAGE = 's3' SWIFT_STORAGE = 'swift' FILE_STORAGE = 'filesystem' storage_type = models.CharField( max_length=16, blank=True, default=default_setting("INSTANCE_STORAGE_TYPE")) class Meta: abstract = True
class Instance(ValidateModelMixin, models.Model): """ Instance: A web application or suite of web applications. An 'Instance' consists of an 'active' AppServer which is available at the instance's URL and handles all requests from users; the instance may also own some 'terminated' AppServers that are no longer used, and 'upcoming' AppServers that are used for testing before being designated as 'active'. In the future, we may add a scalable instance type, which owns a pool of active AppServers that all handle requests; currently at most one AppServer is active at any time. """ # Reverse accessor to get the 'InstanceReference' set. This is a 1:1 relation, so use the # 'ref' property instead of accessing this directly. The only time to use this directly is # in a query, e.g. to do .select_related('ref_set') ref_set = GenericRelation(InstanceReference, content_type_field='instance_type', object_id_field='instance_id') openstack_region = models.CharField( max_length=16, blank=False, default=default_setting('OPENSTACK_REGION'), ) tags = models.ManyToManyField( 'InstanceTag', blank=True, help_text='Custom tags associated with the instance.', ) class Meta: abstract = True def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.logger = ModelLoggerAdapter(logger, {'obj': self}) def __str__(self): return str(self.ref) @cached_property def ref(self): """ Get the InstanceReference for this Instance """ try: # This is a 1:1 relation, but django's ORM does not know that. # We use all() instead of get() or first() because all()[0] can be optimized better by django's ORM # (e.g. when using prefetch_related). return self.ref_set.all()[0] except IndexError: # The InstanceReference does not yet exist - create it: return InstanceReference(instance=self) @property def name(self): """ Get this instance's name, which is stored in the InstanceReference """ return self.ref.name @name.setter def name(self, new_name): """ Change the 'name' """ self.ref.name = new_name @property def created(self): """ Get this instance's created date, which is stored in the InstanceReference """ return self.ref.created @property def modified(self): """ Get this instance's modified date, which is stored in the InstanceReference """ return self.ref.modified def save(self, *args, **kwargs): """ Save this Instance """ super().save(*args, **kwargs) # Ensure an InstanceReference exists, and update its 'modified' field: if self.ref.instance_id is None: self.ref.instance_id = self.pk # <- Fix needed when self.ref is accessed before the first self.save() self.ref.save() def refresh_from_db(self, using=None, fields=None, **kwargs): """ Reload from DB, or load related field. We override this to ensure InstanceReference is reloaded too. Otherwise, the name/created/modified properties could be out of date, even after Instance.refresh_from_db() is called. """ if fields is None: self.ref.refresh_from_db() super().refresh_from_db(using=using, fields=fields, **kwargs) @property def event_context(self): """ Context dictionary to include in events """ return { 'instance_id': self.ref.pk, 'instance_type': self.__class__.__name__ } def get_log_message_annotation(self): """ Get annotation for log message for this instance. """ return 'instance={} ({!s:.15})'.format(self.ref.pk, self.ref.name) @property def log_entries(self): """ Return the list of log entry instances for this Instance. Does NOT include log entries of associated AppServers or Servers (VMs) """ limit = settings.LOG_LIMIT instance_type = ContentType.objects.get_for_model(self) entries = LogEntry.objects.filter(content_type=instance_type, object_id=self.pk) # TODO: Filter out log entries for which the user doesn't have view rights return reversed(list(entries[:limit])) def archive(self): """ Mark this instance as archived. Subclasses should override this to shut down any active resources being used by this instance. """ self.ref.is_archived = True self.ref.save() def delete(self, *args, **kwargs): """ Delete this Instance. This will delete the InstanceReference at the same time. """ if not kwargs.pop('ref_already_deleted', False): self.ref.delete(instance_already_deleted=True) super().delete(*args, **kwargs)
class SwiftContainerInstanceMixin(models.Model): """ Mixin to provision Swift containers for an instance. """ swift_openstack_user = models.CharField( max_length=32, blank=True, default=default_setting('SWIFT_OPENSTACK_USER'), ) swift_openstack_password = models.CharField( max_length=64, blank=True, default=default_setting('SWIFT_OPENSTACK_PASSWORD'), ) swift_openstack_tenant = models.CharField( max_length=32, blank=True, default=default_setting('SWIFT_OPENSTACK_TENANT'), ) swift_openstack_auth_url = models.URLField( blank=True, default=default_setting('SWIFT_OPENSTACK_AUTH_URL'), ) swift_openstack_region = models.CharField( max_length=16, blank=True, default=default_setting('SWIFT_OPENSTACK_REGION'), ) swift_provisioned = models.BooleanField(default=False) class Meta: abstract = True @property def swift_container_names(self): """ An iterable of Swift container names. """ return NotImplementedError def provision_swift(self): """ Create the Swift containers if necessary. """ if self.storage_type == self.SWIFT_STORAGE: for container_name in self.swift_container_names: openstack_utils.create_swift_container( container_name, user=self.swift_openstack_user, password=self.swift_openstack_password, tenant=self.swift_openstack_tenant, auth_url=self.swift_openstack_auth_url, region=self.swift_openstack_region, ) self.swift_provisioned = True self.save() def deprovision_swift(self): """ Delete the Swift containers. """ self.logger.info('Deprovisioning swift started.') if self.storage_type == self.SWIFT_STORAGE and self.swift_provisioned: for container_name in self.swift_container_names: self.logger.info('Deleting swift container: %s', container_name) try: openstack_utils.delete_swift_container( container_name, user=self.swift_openstack_user, password=self.swift_openstack_password, tenant=self.swift_openstack_tenant, auth_url=self.swift_openstack_auth_url, region=self.swift_openstack_region, ) except SwiftClientException: # If deleting a Swift container fails, we still want to continue. self.logger.exception( 'Could not delete Swift container "%s".', container_name) self.swift_provisioned = False self.save() self.logger.info('Deprovisioning swift finished.')
class S3BucketInstanceMixin(models.Model): """ Mixin to provision S3 bucket for an instance. """ class Meta: abstract = True s3_access_key = models.CharField(max_length=50, blank=True) s3_secret_access_key = models.CharField(max_length=50, blank=True) s3_bucket_name = models.CharField(max_length=50, blank=True) s3_region = models.CharField( max_length=50, blank=True, default=default_setting('AWS_S3_DEFAULT_REGION'), help_text= ('The region must support Signature Version 2.' ' See https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region for options.' ' When set empty, the bucket is created in the default region us-east-1.' )) def get_s3_policy(self): """ Return s3 policy with access to create and update bucket """ policy = { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Action": [ "s3:ListBucket", "s3:CreateBucket", "s3:DeleteBucket", "s3:PutBucketCORS" ], "Resource": ["arn:aws:s3:::{}".format(self.s3_bucket_name)] }, { "Effect": "Allow", "Action": ["s3:*Object*"], "Resource": ["arn:aws:s3:::{}/*".format(self.s3_bucket_name)] }] } return json.dumps(policy, indent=2) @property def bucket_name(self): """ Return bucket name truncated to 50 characters """ return truncate_name('{}-{}'.format( settings.AWS_S3_BUCKET_PREFIX, self.database_name.replace('_', '-')), length=50) @property def iam_username(self): """ Return IAM username truncated to 50 characters """ return truncate_name('{}-{}'.format(settings.AWS_IAM_USER_PREFIX, self.database_name), length=50) @property def s3_hostname(self): """ Return the S3 hostname to use when creating a connection. """ s3_hostname = settings.AWS_S3_DEFAULT_HOSTNAME if self.s3_region: s3_hostname = settings.AWS_S3_CUSTOM_REGION_HOSTNAME.format( region=self.s3_region) return s3_hostname @property def s3_custom_domain(self): """ The custom domain name built based on the bucket name. """ return "{}.s3.amazonaws.com".format(self.s3_bucket_name) def create_iam_user(self): """ Create IAM user with access only to the s3 bucket set in s3_bucket_name """ if not (settings.AWS_ACCESS_KEY_ID or settings.AWS_SECRET_ACCESS_KEY): return iam = get_master_iam_connection() iam.create_user(self.iam_username) iam.put_user_policy(self.iam_username, 'allow_access_s3_bucket', self.get_s3_policy()) key_response = iam.create_access_key(self.iam_username) keys = key_response['create_access_key_response'][ 'create_access_key_result']['access_key'] self.s3_access_key = keys['access_key_id'] self.s3_secret_access_key = keys['secret_access_key'] self.save() def get_s3_connection(self): """ Create connection to S3 service """ return boto.connect_s3( self.s3_access_key, self.s3_secret_access_key, # The host parameter is required when connecting to non-default regions and using # AWS signature version 4. host=self.s3_hostname) def _create_bucket(self, attempts=4, ongoing_attempt=1, retry_delay=4, location=None): """ Create bucket, retry up to defined attempts if it fails If you specify a location (e.g. 'EU', 'us-west-1'), this method will use it. If the location is not specified, the value of the instance's 's3_region' field is used. """ time.sleep(retry_delay) location = location or self.s3_region try: s3 = self.get_s3_connection() try: bucket = s3.create_bucket(self.s3_bucket_name, location=location) except boto.exception.S3CreateError as e: if e.error_code == 'BucketAlreadyOwnedByYou': # Bucket already exists. Continue with set_cors bucket = s3.get_bucket(self.s3_bucket_name) else: raise e bucket.set_cors(get_s3_cors_config()) except boto.exception.S3ResponseError: if ongoing_attempt > attempts: raise self.logger.info( 'Retrying bucket creation. IAM keys are not propagated yet, attempt %s of %s.', ongoing_attempt, attempts) ongoing_attempt += 1 self._create_bucket(attempts=attempts, ongoing_attempt=ongoing_attempt, retry_delay=retry_delay, location=location) def provision_s3(self): """ Create S3 Bucket if it doesn't exist """ if not self.storage_type == self.S3_STORAGE: return if not self.s3_bucket_name: self.s3_bucket_name = self.bucket_name if not self.s3_access_key and not self.s3_secret_access_key: self.create_iam_user() self._create_bucket(location=self.s3_region) def deprovision_s3(self): """ Deprovision S3 by deleting S3 bucket and IAM user """ self.logger.info('Deprovisioning S3 started.') if (not self.storage_type == self.S3_STORAGE or not (self.s3_access_key or self.s3_secret_access_key or self.s3_bucket_name)): return if self.s3_bucket_name: try: s3 = self.get_s3_connection() bucket = s3.get_bucket(self.s3_bucket_name) self.logger.info('Deleting s3 bucket: %s', self.s3_bucket_name) for key in bucket: key.delete() s3.delete_bucket(self.s3_bucket_name) self.s3_bucket_name = "" self.save() except boto.exception.S3ResponseError: self.logger.exception( 'There was an error trying to remove S3 bucket "%s".', self.s3_bucket_name) try: self.logger.info('Deleting IAM user: %s', self.iam_username) iam = get_master_iam_connection() # Access keys and policies need to be deleted before removing the user iam.delete_access_key(self.s3_access_key, user_name=self.iam_username) iam.delete_user_policy(self.iam_username, 'allow_access_s3_bucket') iam.delete_user(self.iam_username) self.s3_access_key = "" self.s3_secret_access_key = "" self.save() except boto.exception.BotoServerError: self.logger.exception( 'There was an error trying to remove IAM user "%s".', self.iam_username) self.logger.info('Deprovisioning S3 finished.')
class OpenEdXAppConfiguration(models.Model): """ Configuration fields used by OpenEdX Instance and AppServer. Mutable on the instance but immutable on the AppServer. """ class Meta: abstract = True email = models.EmailField( default='*****@*****.**', help_text= ('The default contact email for this instance; also used as the from address for emails ' 'sent by the server.')) openedx_release = models.CharField( max_length=128, blank=False, default=default_setting('DEFAULT_OPENEDX_RELEASE'), help_text=format_help_text(""" Set this to a release tag like "named-release/dogwood" to build a specific release of Open edX. This setting becomes the default value for edx_platform_version, forum_version, notifier_version, xqueue_version, and certs_version so it should be a git branch that exists in all of those repositories. Note: to build a specific branch of edx-platform, you should just override edx_platform_commit rather than changing this setting. Note 2: This value does not affect the default value of configuration_version. """), ) # Ansible-specific settings: configuration_source_repo_url = models.URLField( max_length=256, blank=False, default=default_setting('DEFAULT_CONFIGURATION_REPO_URL'), ) configuration_version = models.CharField( max_length=50, blank=False, default=default_setting('DEFAULT_CONFIGURATION_VERSION'), ) configuration_extra_settings = models.TextField( blank=True, help_text="YAML config vars that override all others") edx_platform_repository_url = models.CharField( max_length=256, blank=False, default=default_setting('DEFAULT_EDX_PLATFORM_REPO_URL'), help_text= ('URL to the edx-platform repository to use. Leave blank for default.' ), ) edx_platform_commit = models.CharField( max_length=256, blank=False, help_text= ('edx-platform commit hash or branch or tag to use. Leave blank to use the default, ' 'which is equal to the value of "openedx_release".')) # OpenStack VM settings openstack_server_flavor = JSONField( null=True, blank=True, default=default_setting('OPENSTACK_SANDBOX_FLAVOR'), help_text='JSON openstack flavor selector, e.g. {"name": "vps-ssd-1"}.' ' Defaults to settings.OPENSTACK_SANDBOX_FLAVOR on server creation.', ) openstack_server_base_image = JSONField( null=True, blank=True, default=default_setting('OPENSTACK_SANDBOX_BASE_IMAGE'), help_text= 'JSON openstack base image selector, e.g. {"name": "xenial-16.04-unmodified"}' ' Defaults to settings.OPENSTACK_SANDBOX_BASE_IMAGE on server creation.', ) openstack_server_ssh_keyname = models.CharField( max_length=256, null=True, blank=True, default=default_setting('OPENSTACK_SANDBOX_SSH_KEYNAME'), help_text= 'SSH key name used when setting up access to the openstack project.' ' Defaults to settings.OPENSTACK_SANDBOX_SSH_KEYNAME on server creation.', ) # Misc settings: lms_users = models.ManyToManyField( settings.AUTH_USER_MODEL, blank=True, help_text= 'Instance manager users that should be made staff users on the instance.', ) additional_security_groups = ArrayField( models.CharField(max_length=200), default=list, blank=True, help_text= ("Optional: A list of extra OpenStack security group names to use for this instance's VMs. " "A typical use case is to grant this instance access to a private database server that is " "behind a firewall. (In the django admin, separate group names with a comma.)" )) additional_monitoring_emails = ArrayField( models.CharField(max_length=200), default=list, blank=True, help_text= ("Optional: A list of additional email addresses other than settings.ADMINS " "who should receive alerts from New Relic Synthetics Monitors when this instance " "becomes unavailable.")) @classmethod def get_config_fields(cls): """ Get the names of each field declared on this model (except the automatic ID field). This is used to copy the current values from an Instance to an AppServer when creating a new AppServer. """ return [ field.name for field in cls._meta.fields if field.name not in ('id', ) ]
class OpenEdXAppConfiguration(models.Model): """ Configuration fields used by OpenEdX Instance and AppServer. Mutable on the instance but immutable on the AppServer. """ class Meta: abstract = True email = models.EmailField(default='*****@*****.**', help_text=( 'The default contact email for this instance; also used as the from address for emails ' 'sent by the server.' )) privacy_policy_url = models.URLField( verbose_name='URL to Privacy Policy', help_text=('URL to the privacy policy.'), blank=True, default='', ) openedx_release = models.CharField( max_length=128, blank=False, default=default_setting('DEFAULT_OPENEDX_RELEASE'), help_text=format_help_text( """ Set this to a release tag like "named-release/dogwood" to build a specific release of Open edX. This setting becomes the default value for edx_platform_version, forum_version, notifier_version, xqueue_version, and certs_version so it should be a git branch that exists in all of those repositories. Note: to build a specific branch of edx-platform, you should just override edx_platform_commit rather than changing this setting. Note 2: This value does not affect the default value of configuration_version. """ ), ) # Ansible-specific settings: configuration_source_repo_url = models.URLField( max_length=256, blank=False, default=default_setting('DEFAULT_CONFIGURATION_REPO_URL'), ) configuration_version = models.CharField( max_length=50, blank=False, default=default_setting('DEFAULT_CONFIGURATION_VERSION'), ) configuration_extra_settings = models.TextField(blank=True, help_text="YAML config vars that override all others") configuration_playbook_name = models.CharField( max_length=100, blank=True, ) edx_platform_repository_url = models.CharField( max_length=256, blank=False, default=default_setting('DEFAULT_EDX_PLATFORM_REPO_URL'), help_text=( 'URL to the edx-platform repository to use. Leave blank for default.' ), ) edx_platform_commit = models.CharField(max_length=256, blank=False, help_text=( 'edx-platform commit hash or branch or tag to use. Leave blank to use the default, ' 'which is equal to the value of "openedx_release".' )) # Settings related to default ansible playbook ansible_appserver_repo_url = models.URLField( max_length=256, blank=False, default=default_setting('ANSIBLE_APPSERVER_REPO'), help_text=('The repository to pull the default Ansible playbook from.') ) ansible_appserver_playbook = models.CharField( max_length=256, blank=False, default=default_setting('ANSIBLE_APPSERVER_PLAYBOOK'), help_text=('The path to the common appserver playbook to run on all appservers.') ) # pylint: disable=invalid-name ansible_appserver_requirements_path = models.CharField( max_length=256, blank=False, default=default_setting('ANSIBLE_APPSERVER_REQUIREMENTS_PATH'), help_text=('The path to the requirements file for the common appserver playbook.') ) ansible_appserver_version = models.CharField( max_length=256, blank=False, default=default_setting('ANSIBLE_APPSERVER_VERSION'), help_text=('The version of the Ansible playbook repository to checkout.') ) # OpenStack VM settings openstack_server_flavor = JSONField( null=True, blank=True, default=default_setting('OPENSTACK_SANDBOX_FLAVOR'), help_text='JSON openstack flavor selector, e.g. {"name": "vps-ssd-1"}.' ' Defaults to settings.OPENSTACK_SANDBOX_FLAVOR on server creation.', ) openstack_server_base_image = JSONField( null=True, blank=True, default=default_setting('OPENSTACK_SANDBOX_BASE_IMAGE'), help_text='JSON openstack base image selector, e.g. {"name": "focal-20.04-unmodified"}' ' Defaults to settings.OPENSTACK_SANDBOX_BASE_IMAGE on server creation.', ) openstack_server_ssh_keyname = models.CharField( max_length=256, null=True, blank=True, default=default_setting('OPENSTACK_SANDBOX_SSH_KEYNAME'), help_text='SSH key name used when setting up access to the openstack project.' ' Defaults to settings.OPENSTACK_SANDBOX_SSH_KEYNAME on server creation.', ) # Misc settings: lms_users = models.ManyToManyField( settings.AUTH_USER_MODEL, blank=True, help_text='Instance manager users that should be made staff users on the instance.', ) additional_security_groups = ArrayField( models.CharField(max_length=200), default=list, blank=True, help_text=( "Optional: A list of extra OpenStack security group names to use for this instance's VMs. " "A typical use case is to grant this instance access to a private database server that is " "behind a firewall. (In the django admin, separate group names with a comma.)" ) ) additional_monitoring_emails = ArrayField( models.CharField(max_length=200), default=list, blank=True, help_text=( "Optional: A list of additional email addresses other than settings.ADMINS " "who should receive alerts from New Relic Synthetics Monitors when this instance " "becomes unavailable." ) ) provisioning_failure_notification_emails = ArrayField( # pylint: disable=invalid-name models.CharField(max_length=200), default=list, blank=True, help_text=( "Optional: A list of additional email addresses other than settings.ADMINS " "who should receive alerts when an AppServer fails to provision." ) ) openedx_appserver_count = models.IntegerField( default=1, help_text=( "The number of Open edX AppServers to deploy for this instance." ) ) @property def public_contact_email(self): """ Helper to provide similar API to get email as BetaTestApplication """ return self.email @property def base_playbook_name(self): """ Get the correct base playbook name for the openedx_release Automatically fills the field if left empty """ if not self.configuration_playbook_name: self.configuration_playbook_name = get_base_playbook_name(self.openedx_release) self.save() return self.configuration_playbook_name @classmethod def get_config_fields(cls): """ Get the names of each field declared on this model (except the automatic ID field). This is used to copy the current values from an Instance to an AppServer when creating a new AppServer. """ return [field.name for field in cls._meta.fields if field.name not in ('id', )]
class S3BucketInstanceMixin(models.Model): """ Mixin to provision S3 bucket for an instance. """ class Meta: abstract = True s3_access_key = models.CharField(max_length=50, blank=True) s3_secret_access_key = models.CharField(max_length=50, blank=True) s3_bucket_name = models.CharField(max_length=50, blank=True) s3_region = models.CharField( max_length=50, blank=True, default=default_setting('AWS_S3_DEFAULT_REGION'), help_text= ('The region must support Signature Version 2.' ' See https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region for options.' ' When set empty, the bucket is created in the default region us-east-1.' )) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._iam_client = None self._s3_client = None def get_s3_policy(self): """ Return s3 policy with access to create and update bucket """ return { "Version": "2012-10-17", "Statement": [{ "Effect": "Allow", "Action": [ "s3:CreateBucket", "s3:DeleteBucket", "s3:DeleteObjects", "s3:GetBucketCORS", "s3:GetBucketVersioning", "s3:GetLifecycleConfiguration", "s3:ListBucket", "s3:ListBucketVersions", "s3:PutBucketCORS", "s3:PutBucketVersioning", "s3:PutLifecycleConfiguration", ], "Resource": ["arn:aws:s3:::{}".format(self.s3_bucket_name)] }, { "Effect": "Allow", "Action": ["s3:*Object*"], "Resource": ["arn:aws:s3:::{}/*".format(self.s3_bucket_name)] }] } @property def bucket_name(self): """ Return bucket name truncated to 50 characters """ return truncate_name('{}-{}'.format( settings.AWS_S3_BUCKET_PREFIX, self.database_name.replace('_', '-')), length=50) @property def iam_username(self): """ Return IAM username truncated to 50 characters """ return truncate_name('{}-{}'.format(settings.AWS_IAM_USER_PREFIX, self.database_name), length=50) @property def s3_hostname(self): """ Return the S3 hostname to use when creating a connection. """ s3_hostname = settings.AWS_S3_DEFAULT_HOSTNAME if self.s3_region: s3_hostname = settings.AWS_S3_CUSTOM_REGION_HOSTNAME.format( region=self.s3_region) return s3_hostname @property def s3_custom_domain(self): """ The custom domain name built based on the bucket name. """ # If s3_region is empty, boto3 will use S3 default region 'N. Virginia (us-east-1)' # Reference: https://docs.aws.amazon.com/general/latest/gr/rande.html return "{}.s3.{}.amazonaws.com".format(self.s3_bucket_name, self.s3_region or 'us-east-1') @property def iam(self): """ Create connection to S3 service """ if self._iam_client is None: self._iam_client = boto3.client( service_name='iam', region_name=self.s3_region or None, aws_access_key_id=settings.AWS_ACCESS_KEY_ID, aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY) return self._iam_client def create_iam_user(self): """ Create IAM user with access only to the s3 bucket set in s3_bucket_name """ try: self.iam.create_user(UserName=self.iam_username, ) except ClientError as e: if e.response.get('Error', {}).get('Code') == 'EntityAlreadyExists': # Continue if IAM user already exists, i.e. reprovisioning self.logger.info('IAM user %s already exists', self.iam_username) access_key = self.iam.create_access_key( UserName=self.iam_username)['AccessKey'] self.s3_access_key = access_key['AccessKeyId'] self.s3_secret_access_key = access_key['SecretAccessKey'] self.save() @property def s3(self): """ Create connection to S3 service """ if self._s3_client is None: self._s3_client = boto3.client( service_name='s3', region_name=self.s3_region or None, aws_access_key_id=self.s3_access_key, aws_secret_access_key=self.s3_secret_access_key) return self._s3_client def _create_bucket(self, max_tries=4, retry_delay=15, location=None): """ Create bucket, retry up to defined attempts if it fails If you specify a location (e.g. 'EU', 'us-west-1'), this method will use it. If the location is not specified, the value of the instance's 's3_region' field is used. """ location_constraint = location or self.s3_region for attempt in range(1, max_tries + 1): try: self._perform_create_bucket(location_constraint) # Log success self.logger.info('Successfully created S3 bucket.', ) break except ClientError as e: if e.response.get('Error', {}).get('Code') == 'BucketAlreadyOwnedByYou': # Continue if bucket already exists, i.e. reprovisioning # This is only raised outside of us-east-1 self.logger.info('Bucket %s already exists', self.s3_bucket_name) break # Retry up to `max_tries` times self.logger.info( 'Retrying bucket creation due to "%s", attempt %s of %s.', e.response.get('Error', {}).get('Code'), attempt, max_tries) if attempt == max_tries: raise time.sleep(retry_delay) self._update_bucket_policies(max_tries, retry_delay) def _update_bucket_policies(self, max_tries, retry_delay): """ Update bucket policies, including cors, lifecycle config, and versioning. Retry up to `max_tries` times, with `retry_delay` seconds between each attempt. """ for attempt in range(1, max_tries + 1): try: # Update bucket cors self._update_bucket_cors() # Update bucket lifecycle configuration self._update_bucket_lifecycle() # Enable bucket versioning self._enable_bucket_versioning() # Log success self.logger.info('Successfully updated bucket policies.', ) return except ClientError as e: self.logger.info( 'Retrying bucket configuration due to "%s", attempt %s of %s.', e.response.get('Error', {}).get('Code'), attempt, max_tries) if attempt == max_tries: raise time.sleep(retry_delay) def _enable_bucket_versioning(self): """ Enable S3 bucket versioning for instance """ self.s3.put_bucket_versioning( Bucket=self.s3_bucket_name, VersioningConfiguration={'Status': 'Enabled'}) def _update_bucket_lifecycle(self): """ Update lifecycle configuration for instance S3 bucket """ self.s3.put_bucket_lifecycle_configuration( Bucket=self.s3_bucket_name, LifecycleConfiguration=S3_LIFECYCLE) def _update_bucket_cors(self): """ Update S3 bucket CORS configuration for instance """ self.s3.put_bucket_cors(Bucket=self.s3_bucket_name, CORSConfiguration=S3_CORS) def _perform_create_bucket(self, location_constraint): """ Helper method to create S3 bucket :param location_constraint: AWS location or '' """ if not location_constraint or location_constraint == 'us-east-1': # oddly enough, boto3 uses 'us-east-1' as default and doesn't accept it explicitly # https://github.com/boto/boto3/issues/125 bucket = self.s3.create_bucket(Bucket=self.s3_bucket_name) else: bucket = self.s3.create_bucket( Bucket=self.s3_bucket_name, CreateBucketConfiguration={ 'LocationConstraint': location_constraint }, ) return bucket def _update_iam_policy(self): """ Update S3 IAM user policy """ self.iam.put_user_policy(UserName=self.iam_username, PolicyName=USER_POLICY_NAME, PolicyDocument=json.dumps( self.get_s3_policy())) # Force a new connection with the updated policy self._s3_client = None def provision_s3(self): """ Create S3 Bucket if it doesn't exist """ if not self.storage_type == self.S3_STORAGE: return if not self.s3_bucket_name: self.s3_bucket_name = self.bucket_name if not self.s3_access_key and not self.s3_secret_access_key: self.create_iam_user() self._update_iam_policy() self._create_bucket(location=self.s3_region) def _get_bucket_objects(self): """ Get list of objects in bucket for deletion """ response = self.s3.list_object_versions(Bucket=self.s3_bucket_name) return response.get('Versions', []) + response.get('DeleteMarkers', []) def deprovision_s3(self): """ Deprovision S3 by deleting S3 bucket and IAM user """ self.logger.info('Deprovisioning S3 started.') if (not self.storage_type == self.S3_STORAGE or not (self.s3_access_key or self.s3_secret_access_key or self.s3_bucket_name)): return try: to_delete = self._get_bucket_objects() while to_delete: self.s3.delete_objects(Bucket=self.s3_bucket_name, Delete={ 'Objects': [{ 'Key': d['Key'], 'VersionId': d['VersionId'] } for d in to_delete] }) to_delete = self._get_bucket_objects() # Remove bucket self.s3.delete_bucket(Bucket=self.s3_bucket_name) except ClientError as e: if e.response['Error']['Code'] != '404': self.logger.exception( 'There was an error trying to remove S3 bucket "%s".', self.s3_bucket_name) else: self.s3_bucket_name = "" self.save() try: # Access keys and policies need to be deleted before removing the user self.iam.delete_access_key(UserName=self.iam_username, AccessKeyId=self.s3_access_key) self.iam.delete_user_policy(UserName=self.iam_username, PolicyName=USER_POLICY_NAME) self.iam.delete_user(UserName=self.iam_username) self.s3_access_key = "" self.s3_secret_access_key = "" self.save() except ClientError: self.logger.exception( 'There was an error trying to remove IAM user "%s".', self.iam_username) self.logger.info('Deprovisioning S3 finished.')
class OpenStackServer(Server): """ A Server VM hosted on an OpenStack cloud """ openstack_region = models.CharField( max_length=16, blank=False, default=default_setting('OPENSTACK_REGION'), ) openstack_id = models.CharField(max_length=250, db_index=True, blank=True) _public_ip = models.GenericIPAddressField(blank=True, null=True, db_column="public_ip") class Meta: verbose_name = 'OpenStack VM' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.nova = openstack_utils.get_nova_client(self.openstack_region) def __str__(self): if self.openstack_id: return self.openstack_id else: return 'Pending OpenStack Server' @property def os_server(self): """ OpenStack nova server API endpoint """ if not self.openstack_id: assert self.status == Status.Pending self.start() return self.nova.servers.get(self.openstack_id) @property def public_ip(self): """ Return one of the public address(es) """ if not self.vm_created: # No VM means no public IP. return None if not self._public_ip: # This branch will only be reached the first time the public IP address of a server is # requested. We let any exceptions occurring during the Nova API calls propagate to the # caller. We previously caught and ignored all exceptions here, which led to # hard-to-debug bugs. public_addr = openstack_utils.get_server_public_address( self.os_server) if not public_addr: return None self._public_ip = public_addr['addr'] self.save() return self._public_ip @property def vm_created(self): """ Return True if this server has a VM, False otherwise """ return self.status.vm_available or (self.status == Status.Building and self.openstack_id) @property def vm_not_yet_requested(self): """ Return True if the VM has not been requested, and name_prefix can be changed. """ return self.status == Status.Pending def _update_status_from_nova(self, os_server): """ Update the status from the Nova Server object given in os_server. """ self.logger.debug('Updating status from nova (currently %s):\n%s', self.status, to_json(os_server)) if self.status == Status.Unknown: if os_server.status in ('INITIALIZED', 'BUILDING'): # OpenStack has multiple API versions; INITIALIZED is current; BUILDING was used in the past self._status_to_building() if self.status in (Status.Building, Status.Unknown): self.logger.debug('OpenStack: loaded="%s" status="%s"', os_server._loaded, os_server.status) if os_server._loaded and os_server.status == 'ACTIVE': self._status_to_booting() if self.status in (Status.Booting, Status.Unknown) and self.public_ip and is_port_open( self.public_ip, 22): self._status_to_ready() def update_status(self): """ Refresh the status by querying the openstack server via nova """ # TODO: Check when server is stopped # First check if it makes sense to update the current status. # This is not the case if we can not interact with the server: if self.status not in [ Status.BuildFailed, Status.Terminated, Status.Pending ]: try: os_server = self.os_server except novaclient.exceptions.NotFound: # This exception is raised before the server is created, and after it has been # terminated. Because of the first "if", we can't get her in Pending state, so the # server must have been terminated. self.logger.debug('Server does not exist anymore: %s', self) self._status_to_terminated() except (requests.RequestException, novaclient.exceptions.ClientException): self.logger.debug('Could not reach the OpenStack API') if self.status != Status.Unknown: self._status_to_unknown() else: self._update_status_from_nova(os_server) return self.status @Server.status.only_for(Status.Pending) def start(self, flavor_selector=settings.OPENSTACK_SANDBOX_FLAVOR, image_selector=settings.OPENSTACK_SANDBOX_BASE_IMAGE, key_name=settings.OPENSTACK_SANDBOX_SSH_KEYNAME, **kwargs): """ Get a server instance started and an openstack_id assigned TODO: Add handling of quota limitations & waiting list TODO: Create the key dynamically """ self.logger.info('Starting server (status=%s)...', self.status) self._status_to_building() try: os_server = openstack_utils.create_server( self.nova, self.name, flavor_selector=flavor_selector, image_selector=image_selector, key_name=key_name, **kwargs) except novaclient.exceptions.ClientException as e: self.logger.error('Failed to start server: %s', e) self._status_to_build_failed() else: self.openstack_id = os_server.id self.logger.info('Server got assigned OpenStack id %s', self.openstack_id) # Persist OpenStack ID self.save() @Server.status.only_for(Status.Ready, Status.Booting) def reboot(self, reboot_type='SOFT'): """ Reboot the server This requires to switch the status to 'booting'. If the current state doesn't allow to switch to this status, a WrongStateException exception is thrown. """ if self.status == Status.Booting: return self._status_to_booting() self.os_server.reboot(reboot_type=reboot_type) # TODO: Find a better way to wait for the server shutdown and reboot # Currently, without sleeping here, the status would immediately switch back to ready, # as SSH is still available until the reboot terminates the SSHD process time.sleep(30) def terminate(self): """ Terminate the server """ self.logger.info('Terminating server (status=%s)...', self.status) if self.status == Status.Terminated: return elif not self.vm_created: self._status_to_terminated() return self._status_to_terminated() try: self.os_server.delete() except novaclient.exceptions.NotFound: self.logger.error( 'Error while attempting to terminate server: could not find OS server' )
class OpenStackServer(Server): """ A Server VM hosted on an OpenStack cloud """ openstack_region = models.CharField( max_length=16, blank=False, default=default_setting('OPENSTACK_REGION'), ) openstack_id = models.CharField(max_length=250, db_index=True, blank=True) _public_ip = models.GenericIPAddressField(blank=True, null=True, db_column="public_ip") class Meta: verbose_name = 'OpenStack VM' def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.nova = openstack_utils.get_nova_client(self.openstack_region) def __str__(self): if self.openstack_id: return self.openstack_id else: return 'Pending OpenStack Server' @property def os_server(self): """ OpenStack nova server API endpoint """ if not self.openstack_id: assert self.status == Status.Pending self.start() return self.nova.servers.get(self.openstack_id) @property def public_ip(self): """ Return one of the public address(es) """ if not self.vm_created: # No VM means no public IP. return None if not self._public_ip: # This branch will only be reached the first time the public IP address of a server is # requested. We let any exceptions occurring during the Nova API calls propagate to the # caller. We previously caught and ignored all exceptions here, which led to # hard-to-debug bugs. public_addr = openstack_utils.get_server_public_address(self.os_server) if not public_addr: return None self._public_ip = public_addr['addr'] self.save() return self._public_ip @property def vm_created(self): """ Return True if this server has a VM, False otherwise """ return self.status.vm_available or (self.status == Status.Building and self.openstack_id) @property def vm_not_yet_requested(self): """ Return True if the VM has not been requested, and name_prefix can be changed. """ return self.status == Status.Pending def _update_status_from_nova(self, os_server): """ Update the status from the Nova Server object given in os_server. """ self.logger.debug('Updating status from nova (currently %s):\n%s', self.status, to_json(os_server)) if self.status == Status.Unknown: if os_server.status in ('INITIALIZED', 'BUILDING'): # OpenStack has multiple API versions; INITIALIZED is current; BUILDING was used in the past self._status_to_building() if self.status in (Status.Building, Status.Unknown): self.logger.debug('OpenStack: loaded="%s" status="%s"', os_server._loaded, os_server.status) if os_server._loaded and os_server.status == 'ACTIVE': self._status_to_booting() if self.status in (Status.Booting, Status.Unknown) and self.public_ip and is_port_open(self.public_ip, 22): self._status_to_ready() def update_status(self): """ Refresh the status by querying the openstack server via nova """ # TODO: Check when server is stopped # First check if it makes sense to update the current status. # This is not the case if we can not interact with the server: if self.status not in [Status.BuildFailed, Status.Terminated, Status.Pending]: try: os_server = self.os_server except novaclient.exceptions.NotFound: # This exception is raised before the server is created, and after it has been # terminated. Because of the first "if", we can't get her in Pending state, so the # server must have been terminated. self.logger.debug('Server does not exist anymore: %s', self) self._status_to_terminated() except (requests.RequestException, novaclient.exceptions.ClientException, novaclient.exceptions.EndpointNotFound) as exc: self.logger.debug('Could not reach the OpenStack API due to %s', exc) if self.status != Status.Unknown: self._status_to_unknown() else: self._update_status_from_nova(os_server) return self.status @Server.status.only_for(Status.Pending) def start(self, flavor_selector=settings.OPENSTACK_SANDBOX_FLAVOR, image_selector=settings.OPENSTACK_SANDBOX_BASE_IMAGE, key_name=settings.OPENSTACK_SANDBOX_SSH_KEYNAME, **kwargs): """ Get a server instance started and an openstack_id assigned TODO: Add handling of quota limitations & waiting list TODO: Create the key dynamically """ self.logger.info('Starting server (status=%s)...', self.status) self._status_to_building() try: os_server = openstack_utils.create_server( self.nova, self.name, flavor_selector=flavor_selector, image_selector=image_selector, key_name=key_name, **kwargs ) except (novaclient.exceptions.ClientException, novaclient.exceptions.EndpointNotFound) as exc: self.logger.error('Failed to start server: %s', exc) self._status_to_build_failed() else: self.openstack_id = os_server.id self.logger.info('Server got assigned OpenStack id %s', self.openstack_id) # Persist OpenStack ID self.save() @Server.status.only_for(Status.Ready, Status.Booting) def reboot(self, reboot_type='SOFT'): """ Reboot the server This requires to switch the status to 'booting'. If the current state doesn't allow to switch to this status, a WrongStateException exception is thrown. """ if self.status == Status.Booting: return self._status_to_booting() self.os_server.reboot(reboot_type=reboot_type) # TODO: Find a better way to wait for the server shutdown and reboot # Currently, without sleeping here, the status would immediately switch back to ready, # as SSH is still available until the reboot terminates the SSHD process time.sleep(30) def terminate(self): """ Stop and terminate the server. We explicitly stop and wait for a graceful shutdown of the server before terminating it. This ensures any daemons that need to perform cleanup tasks can do so, or that any shutdown scripts/tasks get executed. """ # We should delete SSH key before terminating a server. # Edge cases: server is still being configured or has incorrect status due to an error. self._delete_ssh_key() if self.status == Status.Terminated: return self.logger.info('Terminating server (status=%s)...', self.status) if not self.vm_created: self.logger.info('Note: server was not created when terminated.') self._status_to_terminated() return try: os_server = self.os_server server_closed = self._shutdown(os_server) if not server_closed: self.logger.warning("Server has not reached SHUTOFF state after max wait time; terminating forcefully.") os_server.delete() except novaclient.exceptions.NotFound: self.logger.error('Error while attempting to terminate server: could not find OS server') self._status_to_terminated() except (requests.RequestException, novaclient.exceptions.ClientException, novaclient.exceptions.EndpointNotFound) as exc: self.logger.error('Unable to reach the OpenStack API due to %s', exc) if self.status != Status.Unknown: self._status_to_unknown() else: self._status_to_terminated() def _shutdown(self, os_server, poll_interval=10, max_wait=None): """ Shutdown the server and wait to return until shutdown or wait threshold reached. We don't have an explicit state for this and don't catch exceptions, which is why this is private. The caller is expected to handle any exceptions and retry if necessary. """ # We purposely check for `None` rather than generically for falseness # because it allows `max_wait = 0`. max_wait = max_wait if max_wait is not None else settings.SHUTDOWN_TIMEOUT os_server.stop() while max_wait > 0 and os_server.status != 'SHUTOFF': time.sleep(poll_interval) max_wait -= poll_interval os_server = self.os_server return os_server.status == 'SHUTOFF' def _delete_ssh_key(self) -> None: """ Delete SSH key from `~/.ssh/known_hosts`. We can safely ignore the command's return code, because we just need to be sure that the key has been removed for non-existing server - we don't care about non-existing keys. """ self.logger.info('Deleting SSH key of "%s" host.', self.public_ip) command = f'ssh-keygen -R {self.public_ip}' try: subprocess.Popen(command, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except subprocess.SubprocessError as e: self.logger.error('Failed to delete SSH key of "%s" host: %s', self.public_ip, e)