class StorageContainer(models.Model):
"""
Base class selecting the storage type.
"""
S3_STORAGE = 's3'
SWIFT_STORAGE = 'swift'
FILE_STORAGE = 'filesystem'
storage_type = models.CharField(
max_length=16,
blank=True,
default=default_setting("INSTANCE_STORAGE_TYPE"))
class Meta:
abstract = True
class Instance(ValidateModelMixin, models.Model):
"""
Instance: A web application or suite of web applications.
An 'Instance' consists of an 'active' AppServer which is available at the instance's URL and
handles all requests from users; the instance may also own some 'terminated' AppServers that
are no longer used, and 'upcoming' AppServers that are used for testing before being
designated as 'active'.
In the future, we may add a scalable instance type, which owns a pool of active AppServers
that all handle requests; currently at most one AppServer is active at any time.
"""
# Reverse accessor to get the 'InstanceReference' set. This is a 1:1 relation, so use the
# 'ref' property instead of accessing this directly. The only time to use this directly is
# in a query, e.g. to do .select_related('ref_set')
ref_set = GenericRelation(InstanceReference,
content_type_field='instance_type',
object_id_field='instance_id')
openstack_region = models.CharField(
max_length=16,
blank=False,
default=default_setting('OPENSTACK_REGION'),
)
tags = models.ManyToManyField(
'InstanceTag',
blank=True,
help_text='Custom tags associated with the instance.',
)
class Meta:
abstract = True
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.logger = ModelLoggerAdapter(logger, {'obj': self})
def __str__(self):
return str(self.ref)
@cached_property
def ref(self):
""" Get the InstanceReference for this Instance """
try:
# This is a 1:1 relation, but django's ORM does not know that.
# We use all() instead of get() or first() because all()[0] can be optimized better by django's ORM
# (e.g. when using prefetch_related).
return self.ref_set.all()[0]
except IndexError:
# The InstanceReference does not yet exist - create it:
return InstanceReference(instance=self)
@property
def name(self):
""" Get this instance's name, which is stored in the InstanceReference """
return self.ref.name
@name.setter
def name(self, new_name):
""" Change the 'name' """
self.ref.name = new_name
@property
def created(self):
""" Get this instance's created date, which is stored in the InstanceReference """
return self.ref.created
@property
def modified(self):
""" Get this instance's modified date, which is stored in the InstanceReference """
return self.ref.modified
def save(self, *args, **kwargs):
""" Save this Instance """
super().save(*args, **kwargs)
# Ensure an InstanceReference exists, and update its 'modified' field:
if self.ref.instance_id is None:
self.ref.instance_id = self.pk # <- Fix needed when self.ref is accessed before the first self.save()
self.ref.save()
def refresh_from_db(self, using=None, fields=None, **kwargs):
"""
Reload from DB, or load related field.
We override this to ensure InstanceReference is reloaded too.
Otherwise, the name/created/modified properties could be out of date, even after
Instance.refresh_from_db() is called.
"""
if fields is None:
self.ref.refresh_from_db()
super().refresh_from_db(using=using, fields=fields, **kwargs)
@property
def event_context(self):
"""
Context dictionary to include in events
"""
return {
'instance_id': self.ref.pk,
'instance_type': self.__class__.__name__
}
def get_log_message_annotation(self):
"""
Get annotation for log message for this instance.
"""
return 'instance={} ({!s:.15})'.format(self.ref.pk, self.ref.name)
@property
def log_entries(self):
"""
Return the list of log entry instances for this Instance.
Does NOT include log entries of associated AppServers or Servers (VMs)
"""
limit = settings.LOG_LIMIT
instance_type = ContentType.objects.get_for_model(self)
entries = LogEntry.objects.filter(content_type=instance_type,
object_id=self.pk)
# TODO: Filter out log entries for which the user doesn't have view rights
return reversed(list(entries[:limit]))
def archive(self):
"""
Mark this instance as archived.
Subclasses should override this to shut down any active resources being used by this instance.
"""
self.ref.is_archived = True
self.ref.save()
def delete(self, *args, **kwargs):
"""
Delete this Instance.
This will delete the InstanceReference at the same time.
"""
if not kwargs.pop('ref_already_deleted', False):
self.ref.delete(instance_already_deleted=True)
super().delete(*args, **kwargs)
class SwiftContainerInstanceMixin(models.Model):
"""
Mixin to provision Swift containers for an instance.
"""
swift_openstack_user = models.CharField(
max_length=32,
blank=True,
default=default_setting('SWIFT_OPENSTACK_USER'),
)
swift_openstack_password = models.CharField(
max_length=64,
blank=True,
default=default_setting('SWIFT_OPENSTACK_PASSWORD'),
)
swift_openstack_tenant = models.CharField(
max_length=32,
blank=True,
default=default_setting('SWIFT_OPENSTACK_TENANT'),
)
swift_openstack_auth_url = models.URLField(
blank=True,
default=default_setting('SWIFT_OPENSTACK_AUTH_URL'),
)
swift_openstack_region = models.CharField(
max_length=16,
blank=True,
default=default_setting('SWIFT_OPENSTACK_REGION'),
)
swift_provisioned = models.BooleanField(default=False)
class Meta:
abstract = True
@property
def swift_container_names(self):
"""
An iterable of Swift container names.
"""
return NotImplementedError
def provision_swift(self):
"""
Create the Swift containers if necessary.
"""
if self.storage_type == self.SWIFT_STORAGE:
for container_name in self.swift_container_names:
openstack_utils.create_swift_container(
container_name,
user=self.swift_openstack_user,
password=self.swift_openstack_password,
tenant=self.swift_openstack_tenant,
auth_url=self.swift_openstack_auth_url,
region=self.swift_openstack_region,
)
self.swift_provisioned = True
self.save()
def deprovision_swift(self):
"""
Delete the Swift containers.
"""
self.logger.info('Deprovisioning swift started.')
if self.storage_type == self.SWIFT_STORAGE and self.swift_provisioned:
for container_name in self.swift_container_names:
self.logger.info('Deleting swift container: %s',
container_name)
try:
openstack_utils.delete_swift_container(
container_name,
user=self.swift_openstack_user,
password=self.swift_openstack_password,
tenant=self.swift_openstack_tenant,
auth_url=self.swift_openstack_auth_url,
region=self.swift_openstack_region,
)
except SwiftClientException:
# If deleting a Swift container fails, we still want to continue.
self.logger.exception(
'Could not delete Swift container "%s".',
container_name)
self.swift_provisioned = False
self.save()
self.logger.info('Deprovisioning swift finished.')
class S3BucketInstanceMixin(models.Model):
"""
Mixin to provision S3 bucket for an instance.
"""
class Meta:
abstract = True
s3_access_key = models.CharField(max_length=50, blank=True)
s3_secret_access_key = models.CharField(max_length=50, blank=True)
s3_bucket_name = models.CharField(max_length=50, blank=True)
s3_region = models.CharField(
max_length=50,
blank=True,
default=default_setting('AWS_S3_DEFAULT_REGION'),
help_text=
('The region must support Signature Version 2.'
' See https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region for options.'
' When set empty, the bucket is created in the default region us-east-1.'
))
def get_s3_policy(self):
"""
Return s3 policy with access to create and update bucket
"""
policy = {
"Version":
"2012-10-17",
"Statement": [{
"Effect":
"Allow",
"Action": [
"s3:ListBucket", "s3:CreateBucket", "s3:DeleteBucket",
"s3:PutBucketCORS"
],
"Resource": ["arn:aws:s3:::{}".format(self.s3_bucket_name)]
}, {
"Effect":
"Allow",
"Action": ["s3:*Object*"],
"Resource": ["arn:aws:s3:::{}/*".format(self.s3_bucket_name)]
}]
}
return json.dumps(policy, indent=2)
@property
def bucket_name(self):
"""
Return bucket name truncated to 50 characters
"""
return truncate_name('{}-{}'.format(
settings.AWS_S3_BUCKET_PREFIX,
self.database_name.replace('_', '-')),
length=50)
@property
def iam_username(self):
"""
Return IAM username truncated to 50 characters
"""
return truncate_name('{}-{}'.format(settings.AWS_IAM_USER_PREFIX,
self.database_name),
length=50)
@property
def s3_hostname(self):
"""
Return the S3 hostname to use when creating a connection.
"""
s3_hostname = settings.AWS_S3_DEFAULT_HOSTNAME
if self.s3_region:
s3_hostname = settings.AWS_S3_CUSTOM_REGION_HOSTNAME.format(
region=self.s3_region)
return s3_hostname
@property
def s3_custom_domain(self):
"""
The custom domain name built based on the bucket name.
"""
return "{}.s3.amazonaws.com".format(self.s3_bucket_name)
def create_iam_user(self):
"""
Create IAM user with access only to the s3 bucket set in s3_bucket_name
"""
if not (settings.AWS_ACCESS_KEY_ID or settings.AWS_SECRET_ACCESS_KEY):
return
iam = get_master_iam_connection()
iam.create_user(self.iam_username)
iam.put_user_policy(self.iam_username, 'allow_access_s3_bucket',
self.get_s3_policy())
key_response = iam.create_access_key(self.iam_username)
keys = key_response['create_access_key_response'][
'create_access_key_result']['access_key']
self.s3_access_key = keys['access_key_id']
self.s3_secret_access_key = keys['secret_access_key']
self.save()
def get_s3_connection(self):
"""
Create connection to S3 service
"""
return boto.connect_s3(
self.s3_access_key,
self.s3_secret_access_key,
# The host parameter is required when connecting to non-default regions and using
# AWS signature version 4.
host=self.s3_hostname)
def _create_bucket(self,
attempts=4,
ongoing_attempt=1,
retry_delay=4,
location=None):
"""
Create bucket, retry up to defined attempts if it fails
If you specify a location (e.g. 'EU', 'us-west-1'), this method will use it. If the location is
not specified, the value of the instance's 's3_region' field is used.
"""
time.sleep(retry_delay)
location = location or self.s3_region
try:
s3 = self.get_s3_connection()
try:
bucket = s3.create_bucket(self.s3_bucket_name,
location=location)
except boto.exception.S3CreateError as e:
if e.error_code == 'BucketAlreadyOwnedByYou':
# Bucket already exists. Continue with set_cors
bucket = s3.get_bucket(self.s3_bucket_name)
else:
raise e
bucket.set_cors(get_s3_cors_config())
except boto.exception.S3ResponseError:
if ongoing_attempt > attempts:
raise
self.logger.info(
'Retrying bucket creation. IAM keys are not propagated yet, attempt %s of %s.',
ongoing_attempt, attempts)
ongoing_attempt += 1
self._create_bucket(attempts=attempts,
ongoing_attempt=ongoing_attempt,
retry_delay=retry_delay,
location=location)
def provision_s3(self):
"""
Create S3 Bucket if it doesn't exist
"""
if not self.storage_type == self.S3_STORAGE:
return
if not self.s3_bucket_name:
self.s3_bucket_name = self.bucket_name
if not self.s3_access_key and not self.s3_secret_access_key:
self.create_iam_user()
self._create_bucket(location=self.s3_region)
def deprovision_s3(self):
"""
Deprovision S3 by deleting S3 bucket and IAM user
"""
self.logger.info('Deprovisioning S3 started.')
if (not self.storage_type == self.S3_STORAGE
or not (self.s3_access_key or self.s3_secret_access_key
or self.s3_bucket_name)):
return
if self.s3_bucket_name:
try:
s3 = self.get_s3_connection()
bucket = s3.get_bucket(self.s3_bucket_name)
self.logger.info('Deleting s3 bucket: %s', self.s3_bucket_name)
for key in bucket:
key.delete()
s3.delete_bucket(self.s3_bucket_name)
self.s3_bucket_name = ""
self.save()
except boto.exception.S3ResponseError:
self.logger.exception(
'There was an error trying to remove S3 bucket "%s".',
self.s3_bucket_name)
try:
self.logger.info('Deleting IAM user: %s', self.iam_username)
iam = get_master_iam_connection()
# Access keys and policies need to be deleted before removing the user
iam.delete_access_key(self.s3_access_key,
user_name=self.iam_username)
iam.delete_user_policy(self.iam_username, 'allow_access_s3_bucket')
iam.delete_user(self.iam_username)
self.s3_access_key = ""
self.s3_secret_access_key = ""
self.save()
except boto.exception.BotoServerError:
self.logger.exception(
'There was an error trying to remove IAM user "%s".',
self.iam_username)
self.logger.info('Deprovisioning S3 finished.')
class OpenEdXAppConfiguration(models.Model):
"""
Configuration fields used by OpenEdX Instance and AppServer.
Mutable on the instance but immutable on the AppServer.
"""
class Meta:
abstract = True
email = models.EmailField(
default='*****@*****.**',
help_text=
('The default contact email for this instance; also used as the from address for emails '
'sent by the server.'))
openedx_release = models.CharField(
max_length=128,
blank=False,
default=default_setting('DEFAULT_OPENEDX_RELEASE'),
help_text=format_help_text("""
Set this to a release tag like "named-release/dogwood" to build a specific release of
Open edX. This setting becomes the default value for edx_platform_version,
forum_version, notifier_version, xqueue_version, and certs_version so it should be a git
branch that exists in all of those repositories.
Note: to build a specific branch of edx-platform, you should just override
edx_platform_commit rather than changing this setting.
Note 2: This value does not affect the default value of configuration_version.
"""),
)
# Ansible-specific settings:
configuration_source_repo_url = models.URLField(
max_length=256,
blank=False,
default=default_setting('DEFAULT_CONFIGURATION_REPO_URL'),
)
configuration_version = models.CharField(
max_length=50,
blank=False,
default=default_setting('DEFAULT_CONFIGURATION_VERSION'),
)
configuration_extra_settings = models.TextField(
blank=True, help_text="YAML config vars that override all others")
edx_platform_repository_url = models.CharField(
max_length=256,
blank=False,
default=default_setting('DEFAULT_EDX_PLATFORM_REPO_URL'),
help_text=
('URL to the edx-platform repository to use. Leave blank for default.'
),
)
edx_platform_commit = models.CharField(
max_length=256,
blank=False,
help_text=
('edx-platform commit hash or branch or tag to use. Leave blank to use the default, '
'which is equal to the value of "openedx_release".'))
# OpenStack VM settings
openstack_server_flavor = JSONField(
null=True,
blank=True,
default=default_setting('OPENSTACK_SANDBOX_FLAVOR'),
help_text='JSON openstack flavor selector, e.g. {"name": "vps-ssd-1"}.'
' Defaults to settings.OPENSTACK_SANDBOX_FLAVOR on server creation.',
)
openstack_server_base_image = JSONField(
null=True,
blank=True,
default=default_setting('OPENSTACK_SANDBOX_BASE_IMAGE'),
help_text=
'JSON openstack base image selector, e.g. {"name": "xenial-16.04-unmodified"}'
' Defaults to settings.OPENSTACK_SANDBOX_BASE_IMAGE on server creation.',
)
openstack_server_ssh_keyname = models.CharField(
max_length=256,
null=True,
blank=True,
default=default_setting('OPENSTACK_SANDBOX_SSH_KEYNAME'),
help_text=
'SSH key name used when setting up access to the openstack project.'
' Defaults to settings.OPENSTACK_SANDBOX_SSH_KEYNAME on server creation.',
)
# Misc settings:
lms_users = models.ManyToManyField(
settings.AUTH_USER_MODEL,
blank=True,
help_text=
'Instance manager users that should be made staff users on the instance.',
)
additional_security_groups = ArrayField(
models.CharField(max_length=200),
default=list,
blank=True,
help_text=
("Optional: A list of extra OpenStack security group names to use for this instance's VMs. "
"A typical use case is to grant this instance access to a private database server that is "
"behind a firewall. (In the django admin, separate group names with a comma.)"
))
additional_monitoring_emails = ArrayField(
models.CharField(max_length=200),
default=list,
blank=True,
help_text=
("Optional: A list of additional email addresses other than settings.ADMINS "
"who should receive alerts from New Relic Synthetics Monitors when this instance "
"becomes unavailable."))
@classmethod
def get_config_fields(cls):
"""
Get the names of each field declared on this model (except the automatic ID field).
This is used to copy the current values from an Instance to an AppServer when creating
a new AppServer.
"""
return [
field.name for field in cls._meta.fields
if field.name not in ('id', )
]
class OpenEdXAppConfiguration(models.Model):
"""
Configuration fields used by OpenEdX Instance and AppServer.
Mutable on the instance but immutable on the AppServer.
"""
class Meta:
abstract = True
email = models.EmailField(default='*****@*****.**', help_text=(
'The default contact email for this instance; also used as the from address for emails '
'sent by the server.'
))
privacy_policy_url = models.URLField(
verbose_name='URL to Privacy Policy',
help_text=('URL to the privacy policy.'),
blank=True,
default='',
)
openedx_release = models.CharField(
max_length=128,
blank=False,
default=default_setting('DEFAULT_OPENEDX_RELEASE'),
help_text=format_help_text(
"""
Set this to a release tag like "named-release/dogwood" to build a specific release of
Open edX. This setting becomes the default value for edx_platform_version,
forum_version, notifier_version, xqueue_version, and certs_version so it should be a git
branch that exists in all of those repositories.
Note: to build a specific branch of edx-platform, you should just override
edx_platform_commit rather than changing this setting.
Note 2: This value does not affect the default value of configuration_version.
"""
),
)
# Ansible-specific settings:
configuration_source_repo_url = models.URLField(
max_length=256,
blank=False,
default=default_setting('DEFAULT_CONFIGURATION_REPO_URL'),
)
configuration_version = models.CharField(
max_length=50,
blank=False,
default=default_setting('DEFAULT_CONFIGURATION_VERSION'),
)
configuration_extra_settings = models.TextField(blank=True, help_text="YAML config vars that override all others")
configuration_playbook_name = models.CharField(
max_length=100,
blank=True,
)
edx_platform_repository_url = models.CharField(
max_length=256,
blank=False,
default=default_setting('DEFAULT_EDX_PLATFORM_REPO_URL'),
help_text=(
'URL to the edx-platform repository to use. Leave blank for default.'
),
)
edx_platform_commit = models.CharField(max_length=256, blank=False, help_text=(
'edx-platform commit hash or branch or tag to use. Leave blank to use the default, '
'which is equal to the value of "openedx_release".'
))
# Settings related to default ansible playbook
ansible_appserver_repo_url = models.URLField(
max_length=256,
blank=False,
default=default_setting('ANSIBLE_APPSERVER_REPO'),
help_text=('The repository to pull the default Ansible playbook from.')
)
ansible_appserver_playbook = models.CharField(
max_length=256,
blank=False,
default=default_setting('ANSIBLE_APPSERVER_PLAYBOOK'),
help_text=('The path to the common appserver playbook to run on all appservers.')
)
# pylint: disable=invalid-name
ansible_appserver_requirements_path = models.CharField(
max_length=256,
blank=False,
default=default_setting('ANSIBLE_APPSERVER_REQUIREMENTS_PATH'),
help_text=('The path to the requirements file for the common appserver playbook.')
)
ansible_appserver_version = models.CharField(
max_length=256,
blank=False,
default=default_setting('ANSIBLE_APPSERVER_VERSION'),
help_text=('The version of the Ansible playbook repository to checkout.')
)
# OpenStack VM settings
openstack_server_flavor = JSONField(
null=True,
blank=True,
default=default_setting('OPENSTACK_SANDBOX_FLAVOR'),
help_text='JSON openstack flavor selector, e.g. {"name": "vps-ssd-1"}.'
' Defaults to settings.OPENSTACK_SANDBOX_FLAVOR on server creation.',
)
openstack_server_base_image = JSONField(
null=True,
blank=True,
default=default_setting('OPENSTACK_SANDBOX_BASE_IMAGE'),
help_text='JSON openstack base image selector, e.g. {"name": "focal-20.04-unmodified"}'
' Defaults to settings.OPENSTACK_SANDBOX_BASE_IMAGE on server creation.',
)
openstack_server_ssh_keyname = models.CharField(
max_length=256,
null=True,
blank=True,
default=default_setting('OPENSTACK_SANDBOX_SSH_KEYNAME'),
help_text='SSH key name used when setting up access to the openstack project.'
' Defaults to settings.OPENSTACK_SANDBOX_SSH_KEYNAME on server creation.',
)
# Misc settings:
lms_users = models.ManyToManyField(
settings.AUTH_USER_MODEL,
blank=True,
help_text='Instance manager users that should be made staff users on the instance.',
)
additional_security_groups = ArrayField(
models.CharField(max_length=200),
default=list,
blank=True,
help_text=(
"Optional: A list of extra OpenStack security group names to use for this instance's VMs. "
"A typical use case is to grant this instance access to a private database server that is "
"behind a firewall. (In the django admin, separate group names with a comma.)"
)
)
additional_monitoring_emails = ArrayField(
models.CharField(max_length=200),
default=list,
blank=True,
help_text=(
"Optional: A list of additional email addresses other than settings.ADMINS "
"who should receive alerts from New Relic Synthetics Monitors when this instance "
"becomes unavailable."
)
)
provisioning_failure_notification_emails = ArrayField( # pylint: disable=invalid-name
models.CharField(max_length=200),
default=list,
blank=True,
help_text=(
"Optional: A list of additional email addresses other than settings.ADMINS "
"who should receive alerts when an AppServer fails to provision."
)
)
openedx_appserver_count = models.IntegerField(
default=1,
help_text=(
"The number of Open edX AppServers to deploy for this instance."
)
)
@property
def public_contact_email(self):
""" Helper to provide similar API to get email as BetaTestApplication """
return self.email
@property
def base_playbook_name(self):
"""
Get the correct base playbook name for the openedx_release
Automatically fills the field if left empty
"""
if not self.configuration_playbook_name:
self.configuration_playbook_name = get_base_playbook_name(self.openedx_release)
self.save()
return self.configuration_playbook_name
@classmethod
def get_config_fields(cls):
"""
Get the names of each field declared on this model (except the automatic ID field).
This is used to copy the current values from an Instance to an AppServer when creating
a new AppServer.
"""
return [field.name for field in cls._meta.fields if field.name not in ('id', )]
class S3BucketInstanceMixin(models.Model):
"""
Mixin to provision S3 bucket for an instance.
"""
class Meta:
abstract = True
s3_access_key = models.CharField(max_length=50, blank=True)
s3_secret_access_key = models.CharField(max_length=50, blank=True)
s3_bucket_name = models.CharField(max_length=50, blank=True)
s3_region = models.CharField(
max_length=50,
blank=True,
default=default_setting('AWS_S3_DEFAULT_REGION'),
help_text=
('The region must support Signature Version 2.'
' See https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region for options.'
' When set empty, the bucket is created in the default region us-east-1.'
))
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._iam_client = None
self._s3_client = None
def get_s3_policy(self):
"""
Return s3 policy with access to create and update bucket
"""
return {
"Version":
"2012-10-17",
"Statement": [{
"Effect":
"Allow",
"Action": [
"s3:CreateBucket",
"s3:DeleteBucket",
"s3:DeleteObjects",
"s3:GetBucketCORS",
"s3:GetBucketVersioning",
"s3:GetLifecycleConfiguration",
"s3:ListBucket",
"s3:ListBucketVersions",
"s3:PutBucketCORS",
"s3:PutBucketVersioning",
"s3:PutLifecycleConfiguration",
],
"Resource": ["arn:aws:s3:::{}".format(self.s3_bucket_name)]
}, {
"Effect":
"Allow",
"Action": ["s3:*Object*"],
"Resource": ["arn:aws:s3:::{}/*".format(self.s3_bucket_name)]
}]
}
@property
def bucket_name(self):
"""
Return bucket name truncated to 50 characters
"""
return truncate_name('{}-{}'.format(
settings.AWS_S3_BUCKET_PREFIX,
self.database_name.replace('_', '-')),
length=50)
@property
def iam_username(self):
"""
Return IAM username truncated to 50 characters
"""
return truncate_name('{}-{}'.format(settings.AWS_IAM_USER_PREFIX,
self.database_name),
length=50)
@property
def s3_hostname(self):
"""
Return the S3 hostname to use when creating a connection.
"""
s3_hostname = settings.AWS_S3_DEFAULT_HOSTNAME
if self.s3_region:
s3_hostname = settings.AWS_S3_CUSTOM_REGION_HOSTNAME.format(
region=self.s3_region)
return s3_hostname
@property
def s3_custom_domain(self):
"""
The custom domain name built based on the bucket name.
"""
# If s3_region is empty, boto3 will use S3 default region 'N. Virginia (us-east-1)'
# Reference: https://docs.aws.amazon.com/general/latest/gr/rande.html
return "{}.s3.{}.amazonaws.com".format(self.s3_bucket_name,
self.s3_region or 'us-east-1')
@property
def iam(self):
"""
Create connection to S3 service
"""
if self._iam_client is None:
self._iam_client = boto3.client(
service_name='iam',
region_name=self.s3_region or None,
aws_access_key_id=settings.AWS_ACCESS_KEY_ID,
aws_secret_access_key=settings.AWS_SECRET_ACCESS_KEY)
return self._iam_client
def create_iam_user(self):
"""
Create IAM user with access only to the s3 bucket set in s3_bucket_name
"""
try:
self.iam.create_user(UserName=self.iam_username, )
except ClientError as e:
if e.response.get('Error',
{}).get('Code') == 'EntityAlreadyExists':
# Continue if IAM user already exists, i.e. reprovisioning
self.logger.info('IAM user %s already exists',
self.iam_username)
access_key = self.iam.create_access_key(
UserName=self.iam_username)['AccessKey']
self.s3_access_key = access_key['AccessKeyId']
self.s3_secret_access_key = access_key['SecretAccessKey']
self.save()
@property
def s3(self):
"""
Create connection to S3 service
"""
if self._s3_client is None:
self._s3_client = boto3.client(
service_name='s3',
region_name=self.s3_region or None,
aws_access_key_id=self.s3_access_key,
aws_secret_access_key=self.s3_secret_access_key)
return self._s3_client
def _create_bucket(self, max_tries=4, retry_delay=15, location=None):
"""
Create bucket, retry up to defined attempts if it fails
If you specify a location (e.g. 'EU', 'us-west-1'), this method will use it. If the location is
not specified, the value of the instance's 's3_region' field is used.
"""
location_constraint = location or self.s3_region
for attempt in range(1, max_tries + 1):
try:
self._perform_create_bucket(location_constraint)
# Log success
self.logger.info('Successfully created S3 bucket.', )
break
except ClientError as e:
if e.response.get('Error',
{}).get('Code') == 'BucketAlreadyOwnedByYou':
# Continue if bucket already exists, i.e. reprovisioning
# This is only raised outside of us-east-1
self.logger.info('Bucket %s already exists',
self.s3_bucket_name)
break
# Retry up to `max_tries` times
self.logger.info(
'Retrying bucket creation due to "%s", attempt %s of %s.',
e.response.get('Error', {}).get('Code'), attempt,
max_tries)
if attempt == max_tries:
raise
time.sleep(retry_delay)
self._update_bucket_policies(max_tries, retry_delay)
def _update_bucket_policies(self, max_tries, retry_delay):
"""
Update bucket policies, including cors, lifecycle config, and versioning.
Retry up to `max_tries` times, with `retry_delay` seconds between each attempt.
"""
for attempt in range(1, max_tries + 1):
try:
# Update bucket cors
self._update_bucket_cors()
# Update bucket lifecycle configuration
self._update_bucket_lifecycle()
# Enable bucket versioning
self._enable_bucket_versioning()
# Log success
self.logger.info('Successfully updated bucket policies.', )
return
except ClientError as e:
self.logger.info(
'Retrying bucket configuration due to "%s", attempt %s of %s.',
e.response.get('Error', {}).get('Code'), attempt,
max_tries)
if attempt == max_tries:
raise
time.sleep(retry_delay)
def _enable_bucket_versioning(self):
"""
Enable S3 bucket versioning for instance
"""
self.s3.put_bucket_versioning(
Bucket=self.s3_bucket_name,
VersioningConfiguration={'Status': 'Enabled'})
def _update_bucket_lifecycle(self):
"""
Update lifecycle configuration for instance S3 bucket
"""
self.s3.put_bucket_lifecycle_configuration(
Bucket=self.s3_bucket_name, LifecycleConfiguration=S3_LIFECYCLE)
def _update_bucket_cors(self):
"""
Update S3 bucket CORS configuration for instance
"""
self.s3.put_bucket_cors(Bucket=self.s3_bucket_name,
CORSConfiguration=S3_CORS)
def _perform_create_bucket(self, location_constraint):
"""
Helper method to create S3 bucket
:param location_constraint: AWS location or ''
"""
if not location_constraint or location_constraint == 'us-east-1':
# oddly enough, boto3 uses 'us-east-1' as default and doesn't accept it explicitly
# https://github.com/boto/boto3/issues/125
bucket = self.s3.create_bucket(Bucket=self.s3_bucket_name)
else:
bucket = self.s3.create_bucket(
Bucket=self.s3_bucket_name,
CreateBucketConfiguration={
'LocationConstraint': location_constraint
},
)
return bucket
def _update_iam_policy(self):
"""
Update S3 IAM user policy
"""
self.iam.put_user_policy(UserName=self.iam_username,
PolicyName=USER_POLICY_NAME,
PolicyDocument=json.dumps(
self.get_s3_policy()))
# Force a new connection with the updated policy
self._s3_client = None
def provision_s3(self):
"""
Create S3 Bucket if it doesn't exist
"""
if not self.storage_type == self.S3_STORAGE:
return
if not self.s3_bucket_name:
self.s3_bucket_name = self.bucket_name
if not self.s3_access_key and not self.s3_secret_access_key:
self.create_iam_user()
self._update_iam_policy()
self._create_bucket(location=self.s3_region)
def _get_bucket_objects(self):
"""
Get list of objects in bucket for deletion
"""
response = self.s3.list_object_versions(Bucket=self.s3_bucket_name)
return response.get('Versions', []) + response.get('DeleteMarkers', [])
def deprovision_s3(self):
"""
Deprovision S3 by deleting S3 bucket and IAM user
"""
self.logger.info('Deprovisioning S3 started.')
if (not self.storage_type == self.S3_STORAGE
or not (self.s3_access_key or self.s3_secret_access_key
or self.s3_bucket_name)):
return
try:
to_delete = self._get_bucket_objects()
while to_delete:
self.s3.delete_objects(Bucket=self.s3_bucket_name,
Delete={
'Objects': [{
'Key':
d['Key'],
'VersionId':
d['VersionId']
} for d in to_delete]
})
to_delete = self._get_bucket_objects()
# Remove bucket
self.s3.delete_bucket(Bucket=self.s3_bucket_name)
except ClientError as e:
if e.response['Error']['Code'] != '404':
self.logger.exception(
'There was an error trying to remove S3 bucket "%s".',
self.s3_bucket_name)
else:
self.s3_bucket_name = ""
self.save()
try:
# Access keys and policies need to be deleted before removing the user
self.iam.delete_access_key(UserName=self.iam_username,
AccessKeyId=self.s3_access_key)
self.iam.delete_user_policy(UserName=self.iam_username,
PolicyName=USER_POLICY_NAME)
self.iam.delete_user(UserName=self.iam_username)
self.s3_access_key = ""
self.s3_secret_access_key = ""
self.save()
except ClientError:
self.logger.exception(
'There was an error trying to remove IAM user "%s".',
self.iam_username)
self.logger.info('Deprovisioning S3 finished.')
class OpenStackServer(Server):
"""
A Server VM hosted on an OpenStack cloud
"""
openstack_region = models.CharField(
max_length=16,
blank=False,
default=default_setting('OPENSTACK_REGION'),
)
openstack_id = models.CharField(max_length=250, db_index=True, blank=True)
_public_ip = models.GenericIPAddressField(blank=True,
null=True,
db_column="public_ip")
class Meta:
verbose_name = 'OpenStack VM'
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.nova = openstack_utils.get_nova_client(self.openstack_region)
def __str__(self):
if self.openstack_id:
return self.openstack_id
else:
return 'Pending OpenStack Server'
@property
def os_server(self):
"""
OpenStack nova server API endpoint
"""
if not self.openstack_id:
assert self.status == Status.Pending
self.start()
return self.nova.servers.get(self.openstack_id)
@property
def public_ip(self):
"""
Return one of the public address(es)
"""
if not self.vm_created:
# No VM means no public IP.
return None
if not self._public_ip:
# This branch will only be reached the first time the public IP address of a server is
# requested. We let any exceptions occurring during the Nova API calls propagate to the
# caller. We previously caught and ignored all exceptions here, which led to
# hard-to-debug bugs.
public_addr = openstack_utils.get_server_public_address(
self.os_server)
if not public_addr:
return None
self._public_ip = public_addr['addr']
self.save()
return self._public_ip
@property
def vm_created(self):
"""
Return True if this server has a VM, False otherwise
"""
return self.status.vm_available or (self.status == Status.Building
and self.openstack_id)
@property
def vm_not_yet_requested(self):
"""
Return True if the VM has not been requested, and name_prefix can be changed.
"""
return self.status == Status.Pending
def _update_status_from_nova(self, os_server):
"""
Update the status from the Nova Server object given in os_server.
"""
self.logger.debug('Updating status from nova (currently %s):\n%s',
self.status, to_json(os_server))
if self.status == Status.Unknown:
if os_server.status in ('INITIALIZED', 'BUILDING'):
# OpenStack has multiple API versions; INITIALIZED is current; BUILDING was used in the past
self._status_to_building()
if self.status in (Status.Building, Status.Unknown):
self.logger.debug('OpenStack: loaded="%s" status="%s"',
os_server._loaded, os_server.status)
if os_server._loaded and os_server.status == 'ACTIVE':
self._status_to_booting()
if self.status in (Status.Booting,
Status.Unknown) and self.public_ip and is_port_open(
self.public_ip, 22):
self._status_to_ready()
def update_status(self):
"""
Refresh the status by querying the openstack server via nova
"""
# TODO: Check when server is stopped
# First check if it makes sense to update the current status.
# This is not the case if we can not interact with the server:
if self.status not in [
Status.BuildFailed, Status.Terminated, Status.Pending
]:
try:
os_server = self.os_server
except novaclient.exceptions.NotFound:
# This exception is raised before the server is created, and after it has been
# terminated. Because of the first "if", we can't get her in Pending state, so the
# server must have been terminated.
self.logger.debug('Server does not exist anymore: %s', self)
self._status_to_terminated()
except (requests.RequestException,
novaclient.exceptions.ClientException):
self.logger.debug('Could not reach the OpenStack API')
if self.status != Status.Unknown:
self._status_to_unknown()
else:
self._update_status_from_nova(os_server)
return self.status
@Server.status.only_for(Status.Pending)
def start(self,
flavor_selector=settings.OPENSTACK_SANDBOX_FLAVOR,
image_selector=settings.OPENSTACK_SANDBOX_BASE_IMAGE,
key_name=settings.OPENSTACK_SANDBOX_SSH_KEYNAME,
**kwargs):
"""
Get a server instance started and an openstack_id assigned
TODO: Add handling of quota limitations & waiting list
TODO: Create the key dynamically
"""
self.logger.info('Starting server (status=%s)...', self.status)
self._status_to_building()
try:
os_server = openstack_utils.create_server(
self.nova,
self.name,
flavor_selector=flavor_selector,
image_selector=image_selector,
key_name=key_name,
**kwargs)
except novaclient.exceptions.ClientException as e:
self.logger.error('Failed to start server: %s', e)
self._status_to_build_failed()
else:
self.openstack_id = os_server.id
self.logger.info('Server got assigned OpenStack id %s',
self.openstack_id)
# Persist OpenStack ID
self.save()
@Server.status.only_for(Status.Ready, Status.Booting)
def reboot(self, reboot_type='SOFT'):
"""
Reboot the server
This requires to switch the status to 'booting'.
If the current state doesn't allow to switch to this status,
a WrongStateException exception is thrown.
"""
if self.status == Status.Booting:
return
self._status_to_booting()
self.os_server.reboot(reboot_type=reboot_type)
# TODO: Find a better way to wait for the server shutdown and reboot
# Currently, without sleeping here, the status would immediately switch back to ready,
# as SSH is still available until the reboot terminates the SSHD process
time.sleep(30)
def terminate(self):
"""
Terminate the server
"""
self.logger.info('Terminating server (status=%s)...', self.status)
if self.status == Status.Terminated:
return
elif not self.vm_created:
self._status_to_terminated()
return
self._status_to_terminated()
try:
self.os_server.delete()
except novaclient.exceptions.NotFound:
self.logger.error(
'Error while attempting to terminate server: could not find OS server'
)
class OpenStackServer(Server):
"""
A Server VM hosted on an OpenStack cloud
"""
openstack_region = models.CharField(
max_length=16,
blank=False,
default=default_setting('OPENSTACK_REGION'),
)
openstack_id = models.CharField(max_length=250, db_index=True, blank=True)
_public_ip = models.GenericIPAddressField(blank=True, null=True, db_column="public_ip")
class Meta:
verbose_name = 'OpenStack VM'
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.nova = openstack_utils.get_nova_client(self.openstack_region)
def __str__(self):
if self.openstack_id:
return self.openstack_id
else:
return 'Pending OpenStack Server'
@property
def os_server(self):
"""
OpenStack nova server API endpoint
"""
if not self.openstack_id:
assert self.status == Status.Pending
self.start()
return self.nova.servers.get(self.openstack_id)
@property
def public_ip(self):
"""
Return one of the public address(es)
"""
if not self.vm_created:
# No VM means no public IP.
return None
if not self._public_ip:
# This branch will only be reached the first time the public IP address of a server is
# requested. We let any exceptions occurring during the Nova API calls propagate to the
# caller. We previously caught and ignored all exceptions here, which led to
# hard-to-debug bugs.
public_addr = openstack_utils.get_server_public_address(self.os_server)
if not public_addr:
return None
self._public_ip = public_addr['addr']
self.save()
return self._public_ip
@property
def vm_created(self):
"""
Return True if this server has a VM, False otherwise
"""
return self.status.vm_available or (self.status == Status.Building and self.openstack_id)
@property
def vm_not_yet_requested(self):
"""
Return True if the VM has not been requested, and name_prefix can be changed.
"""
return self.status == Status.Pending
def _update_status_from_nova(self, os_server):
"""
Update the status from the Nova Server object given in os_server.
"""
self.logger.debug('Updating status from nova (currently %s):\n%s', self.status, to_json(os_server))
if self.status == Status.Unknown:
if os_server.status in ('INITIALIZED', 'BUILDING'):
# OpenStack has multiple API versions; INITIALIZED is current; BUILDING was used in the past
self._status_to_building()
if self.status in (Status.Building, Status.Unknown):
self.logger.debug('OpenStack: loaded="%s" status="%s"', os_server._loaded, os_server.status)
if os_server._loaded and os_server.status == 'ACTIVE':
self._status_to_booting()
if self.status in (Status.Booting, Status.Unknown) and self.public_ip and is_port_open(self.public_ip, 22):
self._status_to_ready()
def update_status(self):
"""
Refresh the status by querying the openstack server via nova
"""
# TODO: Check when server is stopped
# First check if it makes sense to update the current status.
# This is not the case if we can not interact with the server:
if self.status not in [Status.BuildFailed, Status.Terminated, Status.Pending]:
try:
os_server = self.os_server
except novaclient.exceptions.NotFound:
# This exception is raised before the server is created, and after it has been
# terminated. Because of the first "if", we can't get her in Pending state, so the
# server must have been terminated.
self.logger.debug('Server does not exist anymore: %s', self)
self._status_to_terminated()
except (requests.RequestException,
novaclient.exceptions.ClientException,
novaclient.exceptions.EndpointNotFound) as exc:
self.logger.debug('Could not reach the OpenStack API due to %s', exc)
if self.status != Status.Unknown:
self._status_to_unknown()
else:
self._update_status_from_nova(os_server)
return self.status
@Server.status.only_for(Status.Pending)
def start(self,
flavor_selector=settings.OPENSTACK_SANDBOX_FLAVOR,
image_selector=settings.OPENSTACK_SANDBOX_BASE_IMAGE,
key_name=settings.OPENSTACK_SANDBOX_SSH_KEYNAME,
**kwargs):
"""
Get a server instance started and an openstack_id assigned
TODO: Add handling of quota limitations & waiting list
TODO: Create the key dynamically
"""
self.logger.info('Starting server (status=%s)...', self.status)
self._status_to_building()
try:
os_server = openstack_utils.create_server(
self.nova,
self.name,
flavor_selector=flavor_selector,
image_selector=image_selector,
key_name=key_name,
**kwargs
)
except (novaclient.exceptions.ClientException, novaclient.exceptions.EndpointNotFound) as exc:
self.logger.error('Failed to start server: %s', exc)
self._status_to_build_failed()
else:
self.openstack_id = os_server.id
self.logger.info('Server got assigned OpenStack id %s', self.openstack_id)
# Persist OpenStack ID
self.save()
@Server.status.only_for(Status.Ready, Status.Booting)
def reboot(self, reboot_type='SOFT'):
"""
Reboot the server
This requires to switch the status to 'booting'.
If the current state doesn't allow to switch to this status,
a WrongStateException exception is thrown.
"""
if self.status == Status.Booting:
return
self._status_to_booting()
self.os_server.reboot(reboot_type=reboot_type)
# TODO: Find a better way to wait for the server shutdown and reboot
# Currently, without sleeping here, the status would immediately switch back to ready,
# as SSH is still available until the reboot terminates the SSHD process
time.sleep(30)
def terminate(self):
"""
Stop and terminate the server.
We explicitly stop and wait for a graceful shutdown of the server before terminating it.
This ensures any daemons that need to perform cleanup tasks can do so, or that any
shutdown scripts/tasks get executed.
"""
# We should delete SSH key before terminating a server.
# Edge cases: server is still being configured or has incorrect status due to an error.
self._delete_ssh_key()
if self.status == Status.Terminated:
return
self.logger.info('Terminating server (status=%s)...', self.status)
if not self.vm_created:
self.logger.info('Note: server was not created when terminated.')
self._status_to_terminated()
return
try:
os_server = self.os_server
server_closed = self._shutdown(os_server)
if not server_closed:
self.logger.warning("Server has not reached SHUTOFF state after max wait time; terminating forcefully.")
os_server.delete()
except novaclient.exceptions.NotFound:
self.logger.error('Error while attempting to terminate server: could not find OS server')
self._status_to_terminated()
except (requests.RequestException,
novaclient.exceptions.ClientException,
novaclient.exceptions.EndpointNotFound) as exc:
self.logger.error('Unable to reach the OpenStack API due to %s', exc)
if self.status != Status.Unknown:
self._status_to_unknown()
else:
self._status_to_terminated()
def _shutdown(self, os_server, poll_interval=10, max_wait=None):
"""
Shutdown the server and wait to return until shutdown or wait threshold reached.
We don't have an explicit state for this and don't catch exceptions, which is why this is private.
The caller is expected to handle any exceptions and retry if necessary.
"""
# We purposely check for `None` rather than generically for falseness
# because it allows `max_wait = 0`.
max_wait = max_wait if max_wait is not None else settings.SHUTDOWN_TIMEOUT
os_server.stop()
while max_wait > 0 and os_server.status != 'SHUTOFF':
time.sleep(poll_interval)
max_wait -= poll_interval
os_server = self.os_server
return os_server.status == 'SHUTOFF'
def _delete_ssh_key(self) -> None:
"""
Delete SSH key from `~/.ssh/known_hosts`.
We can safely ignore the command's return code, because we just need to be sure that the key has been removed
for non-existing server - we don't care about non-existing keys.
"""
self.logger.info('Deleting SSH key of "%s" host.', self.public_ip)
command = f'ssh-keygen -R {self.public_ip}'
try:
subprocess.Popen(command, shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
except subprocess.SubprocessError as e:
self.logger.error('Failed to delete SSH key of "%s" host: %s', self.public_ip, e)
