def handle(self, *args, **options):
"""
Disable ephemeral databases for all instances that are currently using them
and provision new app servers pointing to persistent databases.
"""
# Identify instances that need updating
refs = {
ref
for ref in InstanceReference.objects.filter(is_archived=False)
if ref.instance.use_ephemeral_databases
}
LOG.info('Found "%d" instances using ephemeral databases', len(refs))
for ref in refs:
# Disable ephemeral support and spawn a new AppServer
LOG.info('Migrating %s to use persistent databases',
ref.instance.name)
ref.instance.use_ephemeral_databases = False
ref.instance.save()
spawn_appserver(instance_ref_id=ref.id,
mark_active_on_success=True,
deactivate_old_appservers=True,
num_attempts=3)
LOG.info(
'Migrated and started provisioning a new app server for %s',
ref.instance.name)
def _provision_instance(sender, **kwargs):
"""Provision a new instance once all email addresses of a user are confirmed."""
user = sender
if not all(email.is_confirmed for email in user.email_address_set.iterator()):
return
try:
application = user.betatestapplication
except BetaTestApplication.DoesNotExist:
logger.info('Email confirmed for user %s, who is not a beta tester.', user.username)
return
if application.status == BetaTestApplication.REJECTED:
logger.info('Email confirmed for user %s, but application was rejected.', user.username)
return
if application.instance is not None:
logger.info('Email confirmed for user %s, but instance already provisioned.', user.username)
return
with transaction.atomic():
application.instance = production_instance_factory(
sub_domain=application.subdomain,
name=application.instance_name,
email=application.public_contact_email,
privacy_policy_url=application.privacy_policy_url,
deploy_simpletheme=True,
)
application.instance.lms_users.add(user)
if settings.PROD_APPSERVER_FAIL_EMAILS:
application.instance.provisioning_failure_notification_emails = settings.PROD_APPSERVER_FAIL_EMAILS
application.instance.save()
application.save()
# At this point we know the user has confirmed their email and set up an instance.
# So we can go ahead and send the account info email.
transaction.on_commit(lambda: send_account_info_email(application))
spawn_appserver(application.instance.ref.pk, mark_active_on_success=True, num_attempts=2)
def _provision_instance(sender, **kwargs):
"""Provision a new instance once all email addresses of a user are confirmed."""
user = sender
if not all(email.is_confirmed for email in user.email_address_set.iterator()):
return
try:
application = user.betatestapplication
except BetaTestApplication.DoesNotExist:
logger.info('Email confirmed for user %s, who is not a beta tester.', user.username)
return
if application.status == BetaTestApplication.REJECTED:
logger.info('Email confirmed for user %s, but application was rejected.', user.username)
return
if application.instance is not None:
logger.info('Email confirmed for user %s, but instance already provisioned.', user.username)
return
with transaction.atomic():
application.instance = production_instance_factory(
sub_domain=application.subdomain,
name=application.instance_name,
email=application.public_contact_email,
deploy_simpletheme=True,
)
application.instance.lms_users.add(user)
application.save()
spawn_appserver(application.instance.ref.pk, mark_active_on_success=True, num_attempts=2)
def create(self, request): # pylint: disable=no-self-use
"""
Spawn a new AppServer for an existing OpenEdXInstance
Must pass a parameter called 'instance_id' which is the ID of the InstanceReference of
the OpenEdXInstance that this AppServer is for.
"""
serializer = SpawnAppServerSerializer(data=request.data)
if not serializer.is_valid():
return Response(serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
instance_id = serializer.validated_data['instance_id']
try:
instance_ref = InstanceReference.objects.get(pk=instance_id)
except ObjectDoesNotExist:
raise NotFound(
'InstanceReference with ID {} not found.'.format(instance_id))
instance = instance_ref.instance
if not isinstance(instance, OpenEdXInstance):
raise serializers.ValidationError(
'Invalid InstanceReference ID: Not an OpenEdXInstance.')
spawn_appserver(instance_id)
return Response({'status': 'Instance provisioning started'})
def create(self, request):
"""
Spawn a new AppServer for an existing OpenEdXInstance
Must pass a parameter called 'instance_id' which is the ID of the InstanceReference of
the OpenEdXInstance that this AppServer is for.
"""
serializer = SpawnAppServerSerializer(data=request.data)
if not serializer.is_valid():
return Response(serializer.errors,
status=status.HTTP_400_BAD_REQUEST)
instance_id = serializer.validated_data['instance_id']
# Limit by organization. Instance managers can't spawn servers for other organizations
filtered_instances = IsOrganizationOwnerFilterBackendInstance(
).filter_queryset(
request,
InstanceReference.objects.all(),
view=None,
)
try:
instance_ref = filtered_instances.get(pk=instance_id)
except ObjectDoesNotExist:
raise NotFound(
'InstanceReference with ID {} not found.'.format(instance_id))
instance = instance_ref.instance
if not isinstance(instance, OpenEdXInstance):
raise serializers.ValidationError(
'Invalid InstanceReference ID: Not an OpenEdXInstance.')
spawn_appserver(instance_id)
return Response({'status': 'Instance provisioning started'})
def test_num_attempts(self, mock_spawn):
"""
Test that if num_attempts > 1, the spawn_appserver task will automatically re-try
provisioning.
"""
instance = OpenEdXInstanceFactory()
# Disable mocking of retry-enabled spawn_appserver
self.spawn_appserver_patcher.stop()
self.addCleanup(self.spawn_appserver_patcher.start)
# Mock provisioning failure
mock_spawn.return_value = None
tasks.spawn_appserver(instance.ref.pk,
num_attempts=3,
mark_active_on_success=True)
# Check mocked functions call count
self.assertEqual(mock_spawn.call_count, 3)
self.assertEqual(self.mock_make_appserver_active.call_count, 0)
# Confirm logs
self.assertTrue(
any("Spawning new AppServer, attempt 1 of 3" in log.text
for log in instance.log_entries))
self.assertTrue(
any("Spawning new AppServer, attempt 2 of 3" in log.text
for log in instance.log_entries))
self.assertTrue(
any("Spawning new AppServer, attempt 3 of 3" in log.text
for log in instance.log_entries))
def _do_redeployment(self):
"""
Run the redeployment in batches, logging the status for each loop.
"""
num_attempts = self.options['num_attempts']
batch_size = self.options['batch_size']
update = self.options.get('update', {})
sleep_seconds = self.options['batch_frequency']
activate_on_success = not self.options['no_activate']
# Loop termination is handled at the end.
while True:
# 1. Log instances that failed or succeeded.
for instance in self.ongoing_tag.openedxinstance_set.order_by(
'id').iterator():
instance_tags = instance.tags.all()
if self.success_tag in instance_tags:
LOG.info("SUCCESS: %s [%s]", instance, instance.id)
instance.tags.remove(self.ongoing_tag)
elif self.failure_tag in instance_tags:
LOG.info("FAILED: %s [%s]", instance, instance.id)
instance.tags.remove(self.ongoing_tag)
# 2. Spawn the next batch of instances, if there's room.
next_batch_size = batch_size - self.ongoing_tag.openedxinstance_set.count(
)
for instance in self._pending_instances()[0:next_batch_size]:
# 2.1 Execute any custom MySQL commands (useful for complex upgrades).
self._do_mysql_commands(instance)
# 2.2 Update any fields that need to change
if update:
for field, value in update.items():
setattr(instance, field, value)
instance.save()
# 2.3 Redeploy.
# Note that if the appserver succeeds or fails to deploy, they'll be marked with the appropriate
# tag through `spawn_appserver`'s logic. New appservers will be marked active and old ones will
# be deactivated.
LOG.info("SPAWNING: %s [%s]", instance, instance.id)
instance.tags.add(self.ongoing_tag)
spawn_appserver(
instance.ref.pk,
success_tag=self.success_tag,
failure_tag=self.failure_tag,
num_attempts=num_attempts,
mark_active_on_success=activate_on_success,
deactivate_old_appservers=activate_on_success,
)
# 3. Give a status update.
self._log_status()
# 4. Sleep for the time it takes to configure the new appserver batch, and loop again, or break if done.
if self._redeployment_complete():
break
LOG.info("Sleeping for %s", self._format_batch_frequency())
time.sleep(sleep_seconds)
def test_num_attempts_successful(self, mock_provision, mock_spawn):
"""
Test that if num_attempts > 1, the spawn_appserver task will stop trying to provision
after a successful attempt.
"""
instance = OpenEdXInstanceFactory()
# Disable mocking of retry-enabled spawn_appserver
self.spawn_appserver_patcher.stop()
self.addCleanup(self.spawn_appserver_patcher.start)
# Mock successful provisioning
mock_provision.return_value = True
mock_spawn.return_value = make_test_appserver(instance)
tasks.spawn_appserver(instance.ref.pk,
num_attempts=3,
mark_active_on_success=True)
# Check mocked functions call count
self.assertEqual(mock_spawn.call_count, 1)
self.assertEqual(mock_provision.call_count, 1)
self.assertEqual(self.mock_make_appserver_active.call_count, 1)
# Confirm logs
self.assertTrue(
any("Spawning new AppServer, attempt 1 of 3" in log.text
for log in instance.log_entries))
self.assertFalse(
any("Spawning new AppServer, attempt 2 of 3" in log.text
for log in instance.log_entries))
self.assertFalse(
any("Spawning new AppServer, attempt 3 of 3" in log.text
for log in instance.log_entries))
def _provision_instance(sender, **kwargs):
"""Provision a new instance once all email addresses of a user are confirmed."""
user = sender
if not all(email.is_confirmed for email in user.email_address_set.iterator()):
return
try:
application = user.betatestapplication
except BetaTestApplication.DoesNotExist:
logger.info('Email confirmed for user %s, who is not a beta tester.', user.username)
return
if application.status == BetaTestApplication.REJECTED:
logger.info('Email confirmed for user %s, but application was rejected.', user.username)
return
if application.instance is not None:
logger.info('Email confirmed for user %s, but instance already provisioned.', user.username)
return
with transaction.atomic():
application.instance = production_instance_factory(
sub_domain=application.subdomain,
name=application.instance_name,
email=application.public_contact_email,
)
application.instance.lms_users.add(user)
application.save()
spawn_appserver(application.instance.ref.pk, mark_active_on_success=True, num_attempts=2)
def launch_periodic_builds():
"""
Automatically deploy new servers for all Open edX instances configured for periodic builds.
"""
instances = OpenEdXInstance.objects.filter(periodic_builds_enabled=True)
now = datetime.datetime.now(tz=datetime.timezone.utc)
for instance in instances:
appservers = instance.appserver_set.order_by("-created").all()
# NOTE: 'created' is the time when the appserver was created, which is
# before provisioning begins.
# if the instance has no appservers or latest appserver is past the
# interval time, then we spawn a new appserver
if not appservers or (now - appservers[0].created
) >= instance.periodic_builds_interval:
# check for appservers in-progress; if so, we don't want to launch
# a new one on top
for appserver in appservers:
if appserver.status in (
Status.New,
Status.WaitingForServer,
Status.ConfiguringServer,
):
break
else:
# NOTE: this is async - enqueues as a new huey task and returns immediately
spawn_appserver(
instance.ref.pk,
num_attempts=instance.periodic_builds_retries + 1,
mark_active_on_success=True,
deactivate_old_appservers=True,
)
def test_one_attempt_default(self):
"""
Test that by default, the spawn_appserver task will not re-try provisioning.
"""
instance = OpenEdXInstanceFactory()
self.mock_spawn_appserver.return_value = None # Mock provisioning failure
tasks.spawn_appserver(instance.ref.pk)
self.assertEqual(self.mock_spawn_appserver.call_count, 1)
self.assertTrue(any("Spawning new AppServer, attempt 1 of 1" in log.text for log in instance.log_entries))
def test_one_attempt_default(self):
"""
Test that by default, the spawn_appserver task will not re-try provisioning.
"""
instance = OpenEdXInstanceFactory()
self.mock_spawn_appserver.return_value = None # Mock provisioning failure
tasks.spawn_appserver(instance.ref.pk)
self.assertEqual(self.mock_spawn_appserver.call_count, 1)
self.assertTrue(any("Spawning new AppServer, attempt 1 of 1" in log.text for log in instance.log_entries))
def test_provision_sandbox_instance(self):
"""
Test the spawn_appserver() task, and that it can be used to spawn an AppServer for a new
instance.
"""
instance = OpenEdXInstanceFactory()
tasks.spawn_appserver(instance.ref.pk)
self.assertEqual(self.mock_spawn_appserver.call_count, 1)
self.mock_spawn_appserver.assert_called_once_with(instance)
# By default we don't mark_active_on_success:
self.assertEqual(self.mock_set_appserver_active.call_count, 0)
def test_provision_sandbox_instance(self):
"""
Test the spawn_appserver() task, and that it can be used to spawn an AppServer for a new
instance.
"""
instance = OpenEdXInstanceFactory()
tasks.spawn_appserver(instance.ref.pk)
self.assertEqual(self.mock_spawn_appserver.call_count, 1)
self.mock_spawn_appserver.assert_called_once_with(instance)
# By default we don't mark_active_on_success:
self.assertEqual(self.mock_add_active_appserver.call_count, 0)
def test_spawn_appserver(self):
"""
Provision an instance and spawn an AppServer
"""
OpenEdXInstanceFactory(name='Integration - test_spawn_appserver')
instance = OpenEdXInstance.objects.get()
spawn_appserver(instance.ref.pk, mark_active_on_success=True, num_attempts=2)
self.assert_instance_up(instance)
self.assertTrue(instance.successfully_provisioned)
self.assertTrue(instance.require_user_creation_success())
for appserver in instance.appserver_set.all():
self.assert_secret_keys(instance, appserver)
def test_mark_active_on_success(self, provisioning_succeeds):
"""
Test that we when mark_active_on_success=True, the spawn_appserver task will mark the
newly provisioned AppServer as active, if provisioning succeeded.
"""
self.mock_spawn_appserver.return_value = 10 if provisioning_succeeds else None
instance = OpenEdXInstanceFactory()
tasks.spawn_appserver(instance.ref.pk, mark_active_on_success=True)
self.assertEqual(self.mock_spawn_appserver.call_count, 1)
self.assertEqual(self.mock_set_appserver_active.call_count, 1 if provisioning_succeeds else 0)
def test_mark_active_on_success(self, provisioning_succeeds):
"""
Test that we when mark_active_on_success=True, the spawn_appserver task will mark the
newly provisioned AppServer as active, if provisioning succeeded.
"""
self.mock_spawn_appserver.return_value = 10 if provisioning_succeeds else None
instance = OpenEdXInstanceFactory()
tasks.spawn_appserver(instance.ref.pk, mark_active_on_success=True)
self.assertEqual(self.mock_spawn_appserver.call_count, 1)
self.assertEqual(self.mock_add_active_appserver.call_count, 1 if provisioning_succeeds else 0)
def test_ansible_failignore(self, heartbeat_active, git_checkout, git_working_dir):
"""
Ensure failures that are ignored aren't reflected in the instance
"""
git_working_dir.return_value = os.path.join(os.path.dirname(__file__), "ansible")
heartbeat_active.return_value = True
instance = OpenEdXInstanceFactory(
name='Integration - test_ansible_failignore',
configuration_playbook_name='playbooks/failignore.yml'
)
with self.settings(ANSIBLE_APPSERVER_PLAYBOOK='playbooks/failignore.yml'):
spawn_appserver(instance.ref.pk, mark_active_on_success=True, num_attempts=1)
self.assert_server_ready(instance)
def watch_pr():
"""
Automatically create sandboxes for PRs opened by members of the watched
organization on the watched repository
"""
team_username_list = get_username_list_from_team(settings.WATCH_ORGANIZATION)
for username in team_username_list:
for pr in get_pr_list_from_username(username, settings.WATCH_FORK):
instance, created = WatchedPullRequest.objects.get_or_create_from_pr(pr)
if created:
logger.info('New PR found, creating sandbox: %s', pr)
spawn_appserver(instance.ref.pk, mark_active_on_success=True, num_attempts=2)
def test_ansible_failignore(self, git_checkout, git_working_dir):
"""
Ensure failures that are ignored aren't reflected in the instance
"""
git_working_dir.return_value = os.path.join(os.path.dirname(__file__), "ansible")
instance = OpenEdXInstanceFactory(name='Integration - test_ansible_failignore')
with patch.object(OpenEdXAppServer, 'CONFIGURATION_PLAYBOOK', new="playbooks/failignore.yml"):
spawn_appserver(instance.ref.pk, mark_active_on_success=True, num_attempts=1)
instance.refresh_from_db()
self.assertIsNotNone(instance.active_appserver)
self.assertEqual(instance.active_appserver.status, AppServerStatus.Running)
self.assertEqual(instance.active_appserver.server.status, ServerStatus.Ready)
def test_spawn_appserver(self):
"""
Provision an instance and spawn an AppServer, complete with custom theme (colors)
"""
OpenEdXInstanceFactory(
name='Integration - test_spawn_appserver',
deploy_simpletheme=True,
)
instance = OpenEdXInstance.objects.get()
# Add an lms user, as happens with beta registration
user, _ = get_user_model().objects.get_or_create(
username='******', email='*****@*****.**')
instance.lms_users.add(user)
# Simulate that the application form was filled. This doesn't create another instance nor user
application = BetaTestApplication.objects.create(
user=user,
subdomain='betatestdomain',
instance_name=instance.name,
public_contact_email='*****@*****.**',
project_description='I want to beta test OpenCraft IM',
status=BetaTestApplication.PENDING,
# The presence of these colors will be checked later
# Note: avoid string like #ffbb66 because it would be shortened to #fb6 and therefore
# much harder to detect ("#ffbb66" wouldn't appear in CSS). Use e.g. #ffbb67
main_color='#13709b',
link_color='#14719c',
header_bg_color='#ffbb67',
footer_bg_color='#ddff89',
instance=instance,
)
# We don't want to simulate e-mail verification of the user who submitted the application,
# because that would start provisioning. Instead, we provision ourselves here.
spawn_appserver(instance.ref.pk,
mark_active_on_success=True,
num_attempts=2)
self.assert_instance_up(instance)
self.assert_bucket_configured(instance)
self.assert_appserver_firewalled(instance)
self.assertTrue(instance.successfully_provisioned)
for appserver in instance.appserver_set.all():
self.assert_secret_keys(instance, appserver)
self.assert_lms_users_provisioned(user, appserver)
self.assert_theme_provisioned(instance, appserver, application)
def test_num_attempts(self):
"""
Test that if num_attempts > 1, the spawn_appserver task will automatically re-try
provisioning.
"""
instance = OpenEdXInstanceFactory()
self.mock_spawn_appserver.return_value = None # Mock provisioning failure
tasks.spawn_appserver(instance.ref.pk, num_attempts=3, mark_active_on_success=True)
self.assertEqual(self.mock_spawn_appserver.call_count, 3)
self.assertEqual(self.mock_set_appserver_active.call_count, 0)
self.assertTrue(any("Spawning new AppServer, attempt 1 of 3" in log.text for log in instance.log_entries))
self.assertTrue(any("Spawning new AppServer, attempt 2 of 3" in log.text for log in instance.log_entries))
self.assertTrue(any("Spawning new AppServer, attempt 3 of 3" in log.text for log in instance.log_entries))
def upgrade_instances(self):
"""
Main upgrade method:
1. Obtains list of instances to upgrade
2. Updates instances' fields
3. Saves updated instances to DB
4. Schedules jobs to spawn new appservers with new instance settings
"""
instances = self.get_instances_to_upgrade()
for instance in instances:
instance.refresh_from_db()
logger.info("Upgrading instance %s to %s ...", instance, self.TARGET_RELEASE)
self.upgrade_instance(instance)
instance.save()
spawn_appserver(instance.ref.pk, mark_active_on_success=True, num_attempts=1)
def test_provision_sandbox_instance(self, mock_consul):
"""
Test the spawn_appserver() task, and that it can be used to spawn an AppServer for a new
instance.
"""
instance = OpenEdXInstanceFactory()
tasks.spawn_appserver(instance.ref.pk)
self.assertEqual(self.mock_spawn_appserver.call_count, 1)
self.mock_spawn_appserver.assert_called_once_with(
instance,
failure_tag=None,
success_tag=None,
num_attempts=1
)
# By default we don't mark_active_on_success:
self.assertEqual(self.mock_make_appserver_active.call_count, 0)
def test_num_attempts(self):
"""
Test that if num_attempts > 1, the spawn_appserver task will automatically re-try
provisioning.
"""
instance = OpenEdXInstanceFactory()
self.mock_spawn_appserver.return_value = None # Mock provisioning failure
tasks.spawn_appserver(instance.ref.pk, num_attempts=3, mark_active_on_success=True)
self.assertEqual(self.mock_spawn_appserver.call_count, 3)
self.assertEqual(self.mock_add_active_appserver.call_count, 0)
self.assertTrue(any("Spawning new AppServer, attempt 1 of 3" in log.text for log in instance.log_entries))
self.assertTrue(any("Spawning new AppServer, attempt 2 of 3" in log.text for log in instance.log_entries))
self.assertTrue(any("Spawning new AppServer, attempt 3 of 3" in log.text for log in instance.log_entries))
def test_not_mark_active_if_pending(self):
"""
Test that we when mark_active_on_success=True, the spawn_appserver task will not mark the
newly provisioned AppServer as active if the OpenStack server is not ready.
"""
instance = OpenEdXInstanceFactory()
appserver = make_test_appserver(instance=instance)
appserver.server = BootingOpenStackServerFactory()
appserver.save()
self.mock_spawn_appserver.return_value = appserver.pk
self.make_appserver_active_patcher.stop()
self.addCleanup(self.make_appserver_active_patcher.start)
tasks.spawn_appserver(instance.ref.pk, mark_active_on_success=True)
self.assertEqual(appserver.is_active, False)
def test_mark_active_on_success(self, provisioning_succeeds, mock_consul):
"""
Test that when mark_active_on_success=True, the spawn_appserver task will mark the
newly provisioned AppServer as active, if provisioning succeeded.
"""
instance = OpenEdXInstanceFactory()
server = ReadyOpenStackServerFactory()
appserver = make_test_appserver(instance=instance, server=server)
self.mock_spawn_appserver.return_value = appserver.pk if provisioning_succeeds else None
tasks.spawn_appserver(instance.ref.pk, mark_active_on_success=True)
self.assertEqual(self.mock_spawn_appserver.call_count, 1)
if provisioning_succeeds:
self.mock_make_appserver_active.assert_called_once_with(appserver.pk, active=True, deactivate_others=False)
else:
self.mock_make_appserver_active.assert_not_called()
def watch_pr():
"""
Automatically create sandboxes for PRs opened by members of the watched
organization on the watched repository
"""
team_username_list = get_username_list_from_team(
settings.WATCH_ORGANIZATION)
for username in team_username_list:
for pr in get_pr_list_from_username(username, settings.WATCH_FORK):
instance, created = WatchedPullRequest.objects.get_or_create_from_pr(
pr)
if created:
logger.info('New PR found, creating sandbox: %s', pr)
spawn_appserver(instance.ref.pk,
mark_active_on_success=True,
num_attempts=2)
def test_deactivate_old_appservers(self, provisioning_succeeds):
"""
If `mark_active_on_success` and `deactivate_old_appservers` are both passed in as `True`,
the spawn appserver task will mark the newly provisioned AppServer as active, and deactivate
old appservers, if provisioning succeeded.
"""
instance = OpenEdXInstanceFactory()
server = ReadyOpenStackServerFactory()
appserver = make_test_appserver(instance=instance, server=server)
self.mock_spawn_appserver.return_value = appserver.pk if provisioning_succeeds else None
tasks.spawn_appserver(instance.ref.pk, mark_active_on_success=True, deactivate_old_appservers=True)
self.assertEqual(self.mock_spawn_appserver.call_count, 1)
if provisioning_succeeds:
self.mock_make_appserver_active.assert_called_once_with(appserver.pk, active=True, deactivate_others=True)
else:
self.mock_make_appserver_active.assert_not_called()
def test_activity_csv(self):
"""
Run the activity_csv management command against a live instance.
"""
OpenEdXInstanceFactory(name='Integration - test_spawn_appserver')
instance = OpenEdXInstance.objects.get()
spawn_appserver(instance.ref.pk, mark_active_on_success=True, num_attempts=2)
self.assert_instance_up(instance)
self.assertTrue(instance.successfully_provisioned)
self.assertTrue(instance.require_user_creation_success())
user = User.objects.create_user('betatestuser', '*****@*****.**')
BetaTestApplication.objects.create(
user=user,
subdomain='betatestdomain',
instance_name='betatestinstance',
public_contact_email='*****@*****.**',
project_description='I want to beta test OpenCraft IM',
status=BetaTestApplication.ACCEPTED,
instance=instance,
)
# Run the management command and collect the CSV from stdout.
out = StringIO()
call_command('activity_csv', stdout=out)
out_lines = out.getvalue().split('\r\n')
# The output should look similar to this when one instance is launched:
#
# "Appserver IP","Internal LMS Domain","Name","Contact Email","Unique Hits","Total Users","Total Courses",
# "Age (Days)"
# "213.32.77.49","test.example.com","Instance","*****@*****.**","87","6","1",1
self.assertEqual(
'"Appserver IP","Internal LMS Domain","Name","Contact Email","Unique Hits","Total Users","Total Courses",'
'"Age (Days)"',
out_lines[0]
)
self.assertIn('"Integration - test_spawn_appserver"', out_lines[1])
self.assertIn('"*****@*****.**"', out_lines[1])
self.assertNotIn('N/A', out_lines[1])
# stdout should contain 3 lines (as opposed to 2) to account for the last newline.
self.assertEqual(len(out_lines), 3)
def test_ansible_failure(self, git_checkout, git_working_dir):
"""
Ensure failures in the ansible flow are reflected in the instance
"""
git_working_dir.return_value = os.path.join(os.path.dirname(__file__), "ansible")
instance = OpenEdXInstanceFactory(
name='Integration - test_ansible_failure',
configuration_playbook_name='playbooks/failure.yml'
)
spawn_appserver(instance.ref.pk, mark_active_on_success=True, num_attempts=1)
instance.refresh_from_db()
self.assertFalse(instance.get_active_appservers().exists())
appserver = instance.appserver_set.last()
self.assertFalse(appserver.is_active)
self.assertEqual(appserver.status, AppServerStatus.ConfigurationFailed)
self.assertEqual(appserver.server.status, ServerStatus.Ready)
def test_activity_csv(self):
"""
Run the activity_csv management command against a live instance.
"""
OpenEdXInstanceFactory(name='Integration - test_activity_csv')
instance = OpenEdXInstance.objects.get()
spawn_appserver(instance.ref.pk,
mark_active_on_success=True,
num_attempts=2)
self.assert_instance_up(instance)
self.assertTrue(instance.successfully_provisioned)
user = get_user_model().objects.create_user('betatestuser',
'*****@*****.**')
BetaTestApplication.objects.create(
user=user,
subdomain='betatestdomain',
instance_name='betatestinstance',
public_contact_email='*****@*****.**',
project_description='I want to beta test OpenCraft IM',
status=BetaTestApplication.ACCEPTED,
instance=instance,
)
# Run the management command and collect the CSV from stdout.
out = StringIO()
call_command('activity_csv', stdout=out)
out_lines = out.getvalue().split('\r\n')
# The output should look similar to this when one instance is launched:
#
# "Appserver IP","Internal LMS Domain","Name","Contact Email","Unique Hits","Total Users","Total Courses",
# "Age (Days)"
# "213.32.77.49","test.example.com","Instance","*****@*****.**","87","6","1",1
self.assertEqual(
'"Appserver IP","Internal LMS Domain","Name","Contact Email","Unique Hits","Total Users","Total Courses",'
'"Age (Days)"', out_lines[0])
self.assertIn('"Integration - test_activity_csv"', out_lines[1])
self.assertIn('"*****@*****.**"', out_lines[1])
self.assertNotIn('N/A', out_lines[1])
# stdout should contain 3 lines (as opposed to 2) to account for the last newline.
self.assertEqual(len(out_lines), 3)
def test_spawn_appserver_break_on_success(self, mark_active,
mock_run_playbook,
mock_provision,
mock_provision_rabbitmq):
"""
This test makes sure that upon a successful instance creation, further instances are not created
even when the num_attempts is more than 1.
"""
self.api_client.login(username='******', password='******')
instance = OpenEdXInstanceFactory(edx_platform_commit='1' * 40)
self.assertEqual(instance.appserver_set.count(), 0)
self.assertFalse(instance.get_active_appservers().exists())
spawn_appserver(instance.ref.pk,
mark_active_on_success=mark_active,
num_attempts=4)
self.assertEqual(mock_provision.call_count, 1)
self.assertEqual(mock_provision_rabbitmq.call_count, 1)
def test_external_databases(self):
"""
Ensure that the instance can connect to external databases
"""
if not settings.DEFAULT_INSTANCE_MYSQL_URL or not settings.DEFAULT_INSTANCE_MONGO_URL:
print('External databases not configured, skipping integration test')
return
OpenEdXInstanceFactory(name='Integration - test_external_databases', use_ephemeral_databases=False)
instance = OpenEdXInstance.objects.get()
spawn_appserver(instance.ref.pk, mark_active_on_success=True, num_attempts=2)
self.assert_swift_container_provisioned(instance)
self.assert_instance_up(instance)
self.assertTrue(instance.successfully_provisioned)
self.assertFalse(instance.require_user_creation_success())
for appserver in instance.appserver_set.all():
self.assert_secret_keys(instance, appserver)
self.assert_mysql_db_provisioned(instance)
self.assert_mongo_db_provisioned(instance)
def upgrade_instances(self):
"""
Main upgrade method:
1. Obtains list of instances to upgrade
2. Updates instances' fields
3. Saves updated instances to DB
4. Schedules jobs to spawn new appservers with new instance settings
"""
instances = self.get_instances_to_upgrade()
for instance in instances:
instance.refresh_from_db()
logger.info("Upgrading instance %s to %s ...", instance,
self.TARGET_RELEASE)
self.upgrade_instance(instance)
instance.save()
spawn_appserver(instance.ref.pk,
mark_active_on_success=True,
num_attempts=1)
def test_ansible_failignore(self, heartbeat_active, git_checkout,
git_working_dir):
"""
Ensure failures that are ignored aren't reflected in the instance
"""
git_working_dir.return_value = os.path.join(os.path.dirname(__file__),
"ansible")
heartbeat_active.return_value = True
instance = OpenEdXInstanceFactory(
name='Integration - test_ansible_failignore')
with patch.object(OpenEdXAppServer, 'CONFIGURATION_PLAYBOOK', new="playbooks/failignore.yml"), \
self.settings(ANSIBLE_APPSERVER_PLAYBOOK='playbooks/failignore.yml'):
spawn_appserver(instance.ref.pk,
mark_active_on_success=True,
num_attempts=1)
instance.refresh_from_db()
active_appservers = list(instance.get_active_appservers().all())
self.assertEqual(len(active_appservers), 1)
self.assertTrue(active_appservers[0].is_active)
self.assertEqual(active_appservers[0].status, AppServerStatus.Running)
self.assertEqual(active_appservers[0].server.status,
ServerStatus.Ready)
def test_external_databases(self):
"""
Ensure that the instance can connect to external databases
"""
if not settings.DEFAULT_INSTANCE_MYSQL_URL or not settings.DEFAULT_INSTANCE_MONGO_URL:
print(
'External databases not configured, skipping integration test')
return
OpenEdXInstanceFactory(name='Integration - test_external_databases')
instance = OpenEdXInstance.objects.get()
spawn_appserver(instance.ref.pk,
mark_active_on_success=True,
num_attempts=2)
self.assert_swift_container_provisioned(instance)
self.assert_instance_up(instance)
self.assert_appserver_firewalled(instance)
self.assertTrue(instance.successfully_provisioned)
self.assertFalse(instance.require_user_creation_success())
for appserver in instance.appserver_set.all():
self.assert_secret_keys(instance, appserver)
self.assert_mysql_db_provisioned(instance)
self.assert_mongo_db_provisioned(instance)
def test_one_attempt_default_fail(self, mock_provision, mock_spawn, mock_consul):
"""
Test that by default, the spawn_appserver task will not re-try provisioning, even when failing.
"""
instance = OpenEdXInstanceFactory()
# Disable mocking of retry-enabled spawn_appserver
self.spawn_appserver_patcher.stop()
self.addCleanup(self.spawn_appserver_patcher.start)
# Mock successful provisioning
mock_provision.return_value = False
mock_spawn.return_value = make_test_appserver(instance)
tasks.spawn_appserver(instance.ref.pk)
# Check mocked functions call count
self.assertEqual(mock_spawn.call_count, 1)
self.assertEqual(mock_provision.call_count, 1)
# Confirm logs
self.assertTrue(any("Spawning new AppServer, attempt 1 of 1" in log.text for log in instance.log_entries))
def watch_pr():
"""
Automatically create sandboxes for PRs opened by members of the watched
organization on the watched repository
"""
try:
for watched_fork in WatchedFork.objects.filter(enabled=True):
usernames = list(
UserProfile.objects.filter(
organization=watched_fork.organization, ).exclude(
github_username__isnull=True, ).values_list(
'github_username', flat=True))
for pr in get_pr_list_from_usernames(usernames, watched_fork.fork):
instance, created = WatchedPullRequest.objects.get_or_create_from_pr(
pr, watched_fork)
if created:
logger.info('New PR found, creating sandbox: %s', pr)
spawn_appserver(instance.ref.pk,
mark_active_on_success=True,
num_attempts=2)
except RateLimitExceeded as err:
logger.warning('Could not complete PR scan due to an error: %s',
str(err))
def create(self, request): # pylint: disable=no-self-use
"""
Spawn a new AppServer for an existing OpenEdXInstance
Must pass a parameter called 'instance_id' which is the ID of the InstanceReference of
the OpenEdXInstance that this AppServer is for.
"""
serializer = SpawnAppServerSerializer(data=request.data)
if not serializer.is_valid():
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
instance_id = serializer.validated_data['instance_id']
try:
instance_ref = InstanceReference.objects.get(pk=instance_id)
except ObjectDoesNotExist:
raise NotFound('InstanceReference with ID {} not found.'.format(instance_id))
instance = instance_ref.instance
if not isinstance(instance, OpenEdXInstance):
raise serializers.ValidationError('Invalid InstanceReference ID: Not an OpenEdXInstance.')
spawn_appserver(instance_id)
return Response({'status': 'Instance provisioning started'})
def commit_changes_to_instance(self,
spawn_on_commit=False,
retry_attempts=2):
"""
Copies over configuration changes stored in this model to the related instance,
and optionally spawn a new instance.
"""
instance = self.instance
if instance is None:
return
instance.theme_config = self.draft_theme_config
instance.name = self.instance_name
instance.privacy_policy_url = self.privacy_policy_url
instance.email = self.public_contact_email
instance.save()
if spawn_on_commit:
return spawn_appserver(instance.ref.pk,
mark_active_on_success=True,
num_attempts=retry_attempts)
def test_spawn_appserver(self):
"""
Provision an instance and spawn an AppServer, complete with custom theme (colors)
"""
OpenEdXInstanceFactory(
name='Integration - test_spawn_appserver',
deploy_simpletheme=True,
static_content_overrides={
'version': 0,
'static_template_about_content': 'Hello world!',
'homepage_overlay_html': '<h1>Welcome to the LMS!</h1>',
},
)
instance = OpenEdXInstance.objects.get()
# Add an lms user, as happens with beta registration
user, _ = get_user_model().objects.get_or_create(username='******', email='*****@*****.**')
instance.lms_users.add(user)
# Create user profile and update user model from db
UserProfile.objects.create(
user=user,
full_name="Test user 1",
accepted_privacy_policy=datetime.now(),
accept_paid_support=True,
subscribe_to_updates=True,
)
user.refresh_from_db()
# Simulate that the application form was filled. This doesn't create another instance nor user
application = BetaTestApplication.objects.create(
user=user,
subdomain='betatestdomain',
instance_name=instance.name,
public_contact_email='*****@*****.**',
project_description='I want to beta test OpenCraft IM',
status=BetaTestApplication.PENDING,
# The presence of these colors will be checked later
# Note: avoid string like #ffbb66 because it would be shortened to #fb6 and therefore
# much harder to detect ("#ffbb66" wouldn't appear in CSS). Use e.g. #ffbb67
main_color='#13709b',
link_color='#14719c',
header_bg_color='#ffbb67',
footer_bg_color='#ddff89',
instance=instance,
)
# We don't want to simulate e-mail verification of the user who submitted the application,
# because that would start provisioning. Instead, we provision ourselves here.
spawn_appserver(instance.ref.pk, mark_active_on_success=True, num_attempts=2)
self.assert_server_ready(instance)
self.assert_instance_up(instance)
self.assert_bucket_configured(instance)
self.assert_appserver_firewalled(instance)
self.assertTrue(instance.successfully_provisioned)
for appserver in instance.appserver_set.all():
self.assert_secret_keys(instance, appserver)
self.assert_lms_users_provisioned(user, appserver)
self.assert_theme_provisioned(instance, appserver, application)
self.assert_static_content_overrides_work(instance, appserver, page='about')
self.assert_load_balanced_domains(instance)
# Test external databases
if settings.DEFAULT_INSTANCE_MYSQL_URL and settings.DEFAULT_INSTANCE_MONGO_URL:
self.assertFalse(instance.require_user_creation_success())
self.assert_mysql_db_provisioned(instance)
self.assert_mongo_db_provisioned(instance)
# Test activity CSV
# Run the management command and collect the CSV from stdout.
out = StringIO()
call_command('activity_csv', stdout=out)
out_lines = out.getvalue().split('\r\n')
# The output should look similar to this when one instance is launched:
#
# "Appserver IP","Internal LMS Domain","Name","Contact Email","Unique Hits","Total Users","Total Courses",
# "Age (Days)"
# "213.32.77.49","test.example.com","Instance","*****@*****.**","87","6","1",1
self.assertEqual(
'"Appserver IP","Internal LMS Domain","Name","Contact Email","Unique Hits","Total Users","Total Courses",'
'"Age (Days)"',
out_lines[0]
)
self.assertIn('"Integration - test_spawn_appserver"', out_lines[1])
self.assertIn('"*****@*****.**"', out_lines[1])
self.assertNotIn('N/A', out_lines[1])
# stdout should contain 3 lines (as opposed to 2) to account for the last newline.
self.assertEqual(len(out_lines), 3)
