class Server(ValidateModelMixin, TimeStampedModel):
"""
A single server VM
"""
Status = Status
status = ModelResourceStateDescriptor(
state_classes=Status.states, default_state=Status.Pending, model_field_name='_status'
)
_status = models.CharField(
max_length=20,
default=status.default_state_class.state_id,
choices=status.model_field_choices,
db_index=True,
db_column='status',
)
# State transitions:
_status_to_building = status.transition(from_states=Status.Pending, to_state=Status.Building)
_status_to_build_failed = status.transition(from_states=Status.Building, to_state=Status.BuildFailed)
_status_to_booting = status.transition(from_states=(Status.Building, Status.Ready), to_state=Status.Booting)
_status_to_ready = status.transition(from_states=Status.Booting, to_state=Status.Ready)
_status_to_terminated = status.transition(to_state=Status.Terminated)
_status_to_unknown = status.transition(
from_states=(Status.Building, Status.Booting, Status.Ready), to_state=Status.Unknown
)
instance = models.ForeignKey(SingleVMOpenEdXInstance, related_name='server_set')
objects = ServerQuerySet().as_manager()
logger = ServerLoggerAdapter(logger, {})
class Meta:
abstract = True
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.logger = ServerLoggerAdapter(logger, {'obj': self})
@property
def event_context(self):
"""
Context dictionary to include in events
"""
return {
'instance_id': self.instance.pk,
'server_id': self.pk,
}
def sleep_until(self, condition, timeout=3600):
"""
Sleep in a loop until condition related to server status is fulfilled,
or until timeout (provided in seconds) is reached.
Raises an exception if the desired condition can not be fulfilled.
This can happen if the server is in a steady state (i.e., a state that is not expected to change)
that does not fulfill the desired condition.
The default timeout is 1h.
Use as follows:
server.sleep_until(lambda: server.status.is_steady_state)
server.sleep_until(lambda: server.status.accepts_ssh_commands)
"""
# Check if we received a valid timeout value
# to avoid the possibility of entering an infinite loop (if timeout is negative)
# or reaching the timeout right away (if timeout is zero)
assert timeout > 0, "Timeout must be greater than 0 to be able to do anything useful"
self.logger.info('Waiting to reach status from which we can proceed...')
while timeout > 0:
self.update_status()
if condition():
self.logger.info(
'Reached appropriate status ({name}). Proceeding.'.format(name=self.status.name)
)
return
else:
if self.status.is_steady_state:
raise SteadyStateException(
"The current status ({name}) does not fulfill the desired condition "
"and is not expected to change.".format(name=self.status.name)
)
time.sleep(1)
timeout -= 1
# If we get here, this means we've reached the timeout
raise TimeoutError(
"Waited {minutes:.2f} to reach appropriate status, and got nowhere. "
"Aborting with a status of {status}.".format(minutes=timeout / 60, status=self.status.name)
)
@staticmethod
def on_post_save(sender, instance, created, **kwargs):
"""
Called when an instance is saved
"""
self = instance
publish_data('notification', {
'type': 'server_update',
'server_pk': self.pk,
})
def update_status(self):
"""
Check the current status and update it if it has changed
"""
raise NotImplementedError
class Server(ValidateModelMixin, TimeStampedModel):
"""
A single server VM
"""
NEW = 'new'
STARTED = 'started'
ACTIVE = 'active'
BOOTED = 'booted'
PROVISIONING = 'provisioning'
REBOOTING = 'rebooting'
READY = 'ready'
LIVE = 'live'
STOPPING = 'stopping'
STOPPED = 'stopped'
TERMINATING = 'terminating'
TERMINATED = 'terminated'
STATUS_CHOICES = (
(NEW, 'New - Not yet loaded'),
(STARTED, 'Started - Running but not active yet'),
(ACTIVE, 'Active - Running but not booted yet'),
(BOOTED, 'Booted - Booted but not ready to be added to the application'),
(PROVISIONING, 'Provisioning - Provisioning is in progress'),
(REBOOTING, 'Rebooting - Reboot in progress, to apply changes from provisioning'),
(READY, 'Ready - Rebooted and ready to add to the application'),
(LIVE, 'Live - Is actively used in the application and/or accessed by users'),
(STOPPING, 'Stopping - Stopping temporarily'),
(STOPPED, 'Stopped - Stopped temporarily'),
(TERMINATING, 'Terminating - Stopping forever'),
(TERMINATED, 'Terminated - Stopped forever'),
)
PROGRESS_RUNNING = 'running'
PROGRESS_SUCCESS = 'success'
PROGRESS_FAILED = 'failed'
PROGRESS_CHOICES = (
(PROGRESS_RUNNING, 'Running'),
(PROGRESS_SUCCESS, 'Success'),
(PROGRESS_FAILED, 'Failed'),
)
instance = models.ForeignKey(OpenEdXInstance, related_name='server_set')
status = models.CharField(max_length=20, default=NEW, choices=STATUS_CHOICES, db_index=True)
progress = models.CharField(max_length=7, default=PROGRESS_RUNNING, choices=PROGRESS_CHOICES)
objects = ServerQuerySet().as_manager()
logger = ServerLoggerAdapter(logger, {})
class Meta:
abstract = True
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.logger = ServerLoggerAdapter(logger, {'obj': self})
@property
def event_context(self):
"""
Context dictionary to include in events
"""
return {
'instance_id': self.instance.pk,
'server_id': self.pk,
}
def _set_status(self, status, progress):
"""
Update the current status variable, to be called when a status change is detected
"""
if status not in (s[0] for s in self.STATUS_CHOICES):
raise ValueError(status)
self.status = status
self.progress = progress
self.logger.info('Changed status: %s (%s)', self.status, self.progress)
self.save()
return self.status
def sleep_until_status(self, target_status):
"""
Sleep in a loop until the server reaches one of the specified status
"""
target_status_list = [target_status] if isinstance(target_status, str) else target_status
self.logger.info('Waiting to reach status %s...', target_status_list)
while True:
self.update_status()
if self.status in target_status and self.progress != self.PROGRESS_RUNNING:
break
time.sleep(1)
return self.status
@staticmethod
def on_post_save(sender, instance, created, **kwargs):
"""
Called when an instance is saved
"""
self = instance
publish_data('notification', {
'type': 'server_update',
'server_pk': self.pk,
})
def update_status(self, provisioning=False, rebooting=False, failed=None):
"""
Check the current status and update it if it has changed
"""
raise NotImplementedError