def terminate_obsolete_appservers(self, days=2):
"""
Terminate app servers that were created (more than) `days`
before the currently-active app server of this instance.
Do nothing if this instance doesn't have an active app server.
"""
active_appserver = self.active_appserver
if active_appserver:
for appserver in self.appserver_set.all():
if sufficient_time_passed(appserver.created, active_appserver.created, days):
appserver.terminate_vm()
def test_sufficient_time_passed(self):
"""
Test helper function for checking if sufficient time passed between two dates.
"""
date = datetime(2016, 9, 25, 8, 20, 5)
reference_date = datetime(2016, 10, 11, 15, 10, 30)
# Delta between dates greater than expected delta (actual delta: 16 days)
result = sufficient_time_passed(date, reference_date, 5)
self.assertTrue(result)
# Delta between dates equal to expected delta
result = sufficient_time_passed(date, reference_date, 16)
self.assertTrue(result)
# Delta between dates less than expected delta (actual delta: 16 days)
result = sufficient_time_passed(date, reference_date, 20)
self.assertFalse(result)
# Negative delta (reference date follows date), delta between dates greater than expected delta
result = sufficient_time_passed(reference_date, date, 5)
self.assertFalse(result)
# Negative delta (reference date follows date), delta between dates less than expected delta
result = sufficient_time_passed(reference_date, date, 20)
self.assertFalse(result)
def test_sufficient_time_passed(self):
"""
Test helper function for checking if sufficient time passed between two dates.
"""
date = datetime(2016, 9, 25, 8, 20, 5)
reference_date = datetime(2016, 10, 11, 15, 10, 30)
# Delta between dates greater than expected delta (actual delta: 16 days)
result = sufficient_time_passed(date, reference_date, 5)
self.assertTrue(result)
# Delta between dates equal to expected delta
result = sufficient_time_passed(date, reference_date, 16)
self.assertTrue(result)
# Delta between dates less than expected delta (actual delta: 16 days)
result = sufficient_time_passed(date, reference_date, 20)
self.assertFalse(result)
# Negative delta (reference date follows date), delta between dates greater than expected delta
result = sufficient_time_passed(reference_date, date, 5)
self.assertFalse(result)
# Negative delta (reference date follows date), delta between dates less than expected delta
result = sufficient_time_passed(reference_date, date, 20)
self.assertFalse(result)
def shut_down_obsolete_pr_sandboxes():
"""
Shut down instances whose PRs got merged (more than) one week ago.
"""
for instance in OpenEdXInstance.objects.filter(watchedpullrequest__isnull=False):
pr = github.get_pr_info_by_number(
instance.watchedpullrequest.target_fork_name,
instance.watchedpullrequest.github_pr_number
)
if pr['state'] == 'closed':
closed_at = github.parse_date(pr['closed_at'])
now = datetime.now()
if sufficient_time_passed(closed_at, now, 7):
instance.shut_down()
def terminate_obsolete_appservers(self, days=2):
"""
Terminate app servers that were created more than `days` before now, except:
- the active appserver(s) if there are any,
- a release candidate (rc) appserver, to allow testing before the next appserver activation
(we keep the most recent running appserver)
- a fallback appserver, for `days` after activating an appserver, to allow reverts
(we keep the most recent running appserver created before the latest activation)
"""
latest_active_appserver = None
if self.get_active_appservers().exists():
latest_active_appserver = self.get_active_appservers().latest(
'last_activated')
fallback_appserver = None
rc_appserver = None
now = timezone.now()
for appserver in self.appserver_set.all().order_by('-created'):
# Skip active appservers
if appserver.is_active:
continue
# Keep a running appserver as fallback for `days` after latest activation, to allow reverts
if latest_active_appserver and appserver.created < latest_active_appserver.last_activated:
if not sufficient_time_passed(latest_active_appserver.last_activated, now, days) \
and not fallback_appserver and appserver.status == AppServerStatus.Running:
fallback_appserver = appserver
elif sufficient_time_passed(appserver.created, now, days):
appserver.terminate_vm()
# Keep the most recent running appserver created after activation (or when none is activated)
# to allow testing of a release candidate (rc)
else:
if not rc_appserver and appserver.status == AppServerStatus.Running:
rc_appserver = appserver
elif sufficient_time_passed(appserver.created, now, days):
appserver.terminate_vm()
def shut_down_obsolete_pr_sandboxes():
"""
Shut down instances whose PRs got merged (more than) one week ago.
"""
for instance in OpenEdXInstance.objects.filter(watchedpullrequest__isnull=False):
pr = github.get_pr_info_by_number(
instance.watchedpullrequest.target_fork_name,
instance.watchedpullrequest.github_pr_number
)
if pr['state'] == 'closed' and not instance.ref.is_archived:
closed_at = github.parse_date(pr['closed_at'])
now = datetime.now()
if sufficient_time_passed(closed_at, now, 7):
instance.logger.info("Shutting down obsolete sandbox instance")
instance.archive()
