def launch_periodic_builds():
"""
Automatically deploy new servers for all Open edX instances configured for periodic builds.
"""
instances = OpenEdXInstance.objects.filter(periodic_builds_enabled=True)
now = datetime.datetime.now(tz=datetime.timezone.utc)
for instance in instances:
appservers = instance.appserver_set.order_by("-created").all()
# NOTE: 'created' is the time when the appserver was created, which is
# before provisioning begins.
# if the instance has no appservers or latest appserver is past the
# interval time, then we spawn a new appserver
if not appservers or (now - appservers[0].created
) >= instance.periodic_builds_interval:
# check for appservers in-progress; if so, we don't want to launch
# a new one on top
for appserver in appservers:
if appserver.status in (
Status.New,
Status.WaitingForServer,
Status.ConfiguringServer,
):
break
else:
# NOTE: this is async - enqueues as a new huey task and returns immediately
create_new_deployment(
instance,
num_attempts=instance.periodic_builds_retries + 1,
mark_active_on_success=True,
deployment_type=DeploymentType.periodic,
)
def _do_redeployment(self):
"""
Run the redeployment in batches, logging the status for each loop.
"""
num_attempts = self.options['num_attempts']
batch_size = self.options['batch_size']
update = self.options.get('update', {})
sleep_seconds = self.options['batch_frequency']
activate_on_success = not self.options['no_activate']
# Loop termination is handled at the end.
while True:
# 1. Log instances that failed or succeeded.
for instance in self.ongoing_tag.openedxinstance_set.order_by(
'id').iterator():
instance_tags = instance.tags.all()
if self.success_tag in instance_tags:
LOG.info("SUCCESS: %s [%s]", instance, instance.id)
instance.tags.remove(self.ongoing_tag)
elif self.failure_tag in instance_tags:
LOG.info("FAILED: %s [%s]", instance, instance.id)
instance.tags.remove(self.ongoing_tag)
# 2. Spawn the next batch of instances, if there's room.
next_batch_size = batch_size - self.ongoing_tag.openedxinstance_set.count(
)
for instance in self._pending_instances()[0:next_batch_size]:
# 2.1 Execute any custom MySQL commands (useful for complex upgrades).
self._do_mysql_commands(instance)
# 2.2 Update any fields that need to change
if update:
for field, value in update.items():
setattr(instance, field, value)
instance.save()
# 2.3 Redeploy.
# Note that if the appserver succeeds or fails to deploy, they'll be marked with the appropriate
# tag through `spawn_appserver`'s logic. New appservers will be marked active and old ones will
# be deactivated.
LOG.info("SPAWNING: %s [%s]", instance, instance.id)
instance.tags.add(self.ongoing_tag)
create_new_deployment(
instance,
success_tag=self.success_tag,
failure_tag=self.failure_tag,
num_attempts=num_attempts,
mark_active_on_success=activate_on_success,
deployment_type=DeploymentType.batch,
)
# 3. Give a status update.
self._log_status()
# 4. Sleep for the time it takes to configure the new appserver batch, and loop again, or break if done.
if self._redeployment_complete():
break
LOG.info("Sleeping for %s", self._format_batch_frequency())
time.sleep(sleep_seconds)
def commit_changes_to_instance(self,
deploy_on_commit=False,
retry_attempts=2,
creator=None,
deployment_type=None,
cancel_pending_deployments=False):
"""
Copies over configuration changes stored in this model to the related instance,
and optionally spawn a new instance.
:param deploy_on_commit: Initiate new deployment after committing changes
:param deployment_type: Type of deployment
:param creator: User initiating deployment
:param retry_attempts: Number of times to retry deployment
"""
instance = self.instance
if instance is None:
return
instance.theme_config = self.draft_theme_config
instance.static_content_overrides = self.draft_static_content_overrides
instance.name = self.instance_name
instance.privacy_policy_url = self.privacy_policy_url
instance.email = self.public_contact_email
instance.save()
if deploy_on_commit:
create_new_deployment(
instance,
creator=creator,
deployment_type=deployment_type,
mark_active_on_success=True,
num_attempts=retry_attempts,
cancel_pending=cancel_pending_deployments,
add_delay=True,
)
def watch_pr():
"""
Automatically create sandboxes for PRs opened by members of the watched
organization on the watched repository
"""
if not settings.WATCH_PRS:
return
try:
for watched_fork in WatchedFork.objects.filter(enabled=True):
usernames = list(
UserProfile.objects.filter(
organization=watched_fork.organization, ).exclude(
github_username__isnull=True, ).values_list(
'github_username', flat=True))
for pr in get_pr_list_from_usernames(usernames, watched_fork.fork):
instance, created = WatchedPullRequest.objects.get_or_create_from_pr(
pr, watched_fork)
if created:
logger.info('New PR found, creating sandbox: %s', pr)
create_new_deployment(
instance,
mark_active_on_success=True,
num_attempts=2,
deployment_type=DeploymentType.pr,
)
except RateLimitExceeded as err:
logger.warning('Could not complete PR scan due to an error: %s',
str(err))
def handle(self, *args, **options):
pr_url = urlparse(options['pr_url'])
target_fork, pr_number = pr_url.path[1:].split('/pull/')
pr = get_pr_by_number(target_fork, pr_number)
instance, created = WatchedPullRequest.objects.get_or_create_from_pr(
pr, None)
# Set the playbook name to None so that it is auto-generated based on the release version
instance.configuration_playbook_name = None
# Set release-specific configuration parameters
for release_name, release_config in RELEASE_BRANCH_MAP.items():
if release_name in pr.target_branch:
for key, val in release_config.items():
setattr(instance, key, val)
instance.save()
if not created:
self.stderr.write(
self.style.WARNING(
"The specified PR already has a sandbox instance. Updating existing instance."
))
create_new_deployment(
instance,
mark_active_on_success=True,
num_attempts=2,
deployment_type=DeploymentType.pr,
)
def test_ansible_failignore(self, heartbeat_active, get_playbooks,
manage_instance_services):
"""
Ensure failures that are ignored aren't reflected in the instance
"""
heartbeat_active.return_value = True
get_playbooks.return_value = [
Playbook(
source_repo=os.path.join(os.path.dirname(__file__), 'ansible'),
requirements_path='requirements.txt',
playbook_path='playbooks/failignore.yml',
version=None,
variables='{}',
)
]
# Mocking the manage_services.yml playbook because the services it tries to manage
# will not be installed in the appserver provisioned by the dummy failignore.yml
# playbook.
manage_instance_services.return_value = True
instance = OpenEdXInstanceFactory(
name='Integration - test_ansible_failignore',
configuration_playbook_name='playbooks/failignore.yml')
create_new_deployment(instance,
mark_active_on_success=True,
num_attempts=1)
self.assert_server_ready(instance)
def _provision_instance(sender, **kwargs):
"""Provision a new instance once all email addresses of a user are confirmed."""
user = sender
if not all(email.is_confirmed
for email in user.email_address_set.iterator()):
return
try:
application = user.betatestapplication
except BetaTestApplication.DoesNotExist:
logger.info('Email confirmed for user %s, who is not a beta tester.',
user.username)
return
if application.status == BetaTestApplication.REJECTED:
logger.info(
'Email confirmed for user %s, but application was rejected.',
user.username)
return
if application.instance is not None:
logger.info(
'Email confirmed for user %s, but instance already provisioned.',
user.username)
return
with transaction.atomic():
application.instance = production_instance_factory(
sub_domain=application.subdomain,
name=application.instance_name,
email=application.public_contact_email,
privacy_policy_url=application.privacy_policy_url,
)
application.instance.lms_users.add(user)
application.instance.ref.creator = user.profile
application.instance.ref.save()
# Check if simple theme is set up and add it to instance
if application.draft_theme_config:
application.instance.theme_config = application.draft_theme_config
application.instance.deploy_simpletheme = True
application.instance.save()
# If using external domain, set it up
if application.external_domain:
application.instance.external_lms_domain = application.external_domain
application.instance.save()
if settings.PROD_APPSERVER_FAIL_EMAILS:
application.instance.provisioning_failure_notification_emails = settings.PROD_APPSERVER_FAIL_EMAILS
application.instance.additional_monitoring_emails = settings.PROD_APPSERVER_FAIL_EMAILS
application.instance.save()
application.save()
# At this point we know the user has confirmed their email and set up an instance.
# So we can go ahead and send the account info email.
transaction.on_commit(lambda: send_account_info_email(application))
create_new_deployment(
application.instance,
mark_active_on_success=True,
num_attempts=2,
creator=user,
deployment_type=DeploymentType.registration,
)
def create(self, request):
"""
Spawn a new AppServer for an existing OpenEdXInstance
Must pass a parameter called 'instance_id' which is the ID of the InstanceReference of
the OpenEdXInstance that this AppServer is for.
"""
if self.request.user.is_superuser:
default_deployment_type = DeploymentType.admin.name
else:
default_deployment_type = DeploymentType.user.name
# Allow overriding deployment type in case deployment is created by API in some other way.
deployment_type = request.query_params.get("deployment_type", default_deployment_type)
serializer = SpawnAppServerSerializer(data=request.data)
if not serializer.is_valid():
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
instance_id = serializer.validated_data['instance_id']
# Limit by organization. Instance managers can't spawn servers for other organizations
filtered_instances = IsOrganizationOwnerFilterBackendInstance().filter_queryset(
request,
InstanceReference.objects.all(),
view=None,
)
try:
instance_ref = filtered_instances.get(pk=instance_id)
except ObjectDoesNotExist:
raise NotFound('InstanceReference with ID {} not found.'.format(instance_id))
instance = instance_ref.instance
if not isinstance(instance, OpenEdXInstance):
raise serializers.ValidationError('Invalid InstanceReference ID: Not an OpenEdXInstance.')
create_new_deployment(
instance,
creator=self.request.user,
deployment_type=deployment_type,
)
return Response({'status': 'Instance provisioning started'})