def test_cancel_pending(self):
"""Test cancelling a Job which is in state 'pending'"""
self.set_state_delayed([(self.host.lnet_configuration, "lnet_unloaded")
])
pending_jobs = Job.objects.filter(state="pending")
# stop lnet, unload lnet
self.assertEqual(pending_jobs.count(), 2)
# This is the one we cancelled explicitly
cancelled_job = pending_jobs[0]
# This one should be cancelled as a result of cancelling it's dependency
consequentially_cancelled_job = pending_jobs[1]
JobSchedulerClient.cancel_job(pending_jobs[0].id)
self.drain_progress()
cancelled_job = freshen(cancelled_job)
consequentially_cancelled_job = freshen(consequentially_cancelled_job)
self.assertEqual(cancelled_job.state, "complete")
self.assertEqual(cancelled_job.errored, False)
self.assertEqual(cancelled_job.cancelled, True)
self.assertEqual(consequentially_cancelled_job.state, "complete")
self.assertEqual(consequentially_cancelled_job.errored, False)
self.assertEqual(consequentially_cancelled_job.cancelled, True)
pending_jobs = Job.objects.filter(state="pending")
self.assertEqual(pending_jobs.count(), 0)
self.assertFalse(
self.job_scheduler._lock_cache.get_by_job(cancelled_job))
def test_late_notification(self):
"""Test that notifications are droppped when they are older than
the last change to an objects state"""
self.lnet_configuration = self.assertState(self.lnet_configuration, 'lnet_up')
awhile_ago = django.utils.timezone.now() - datetime.timedelta(seconds = 120)
job_scheduler_notify.notify(freshen(self.lnet_configuration), awhile_ago, {'state': 'lnet_down'}, ['lnet_up'])
self.assertEqual(freshen(self.lnet_configuration).state, 'lnet_up')
def test_buffered_notification(self):
"""Test that notifications for locked items are buffered and
replayed when the locking Job has completed."""
self.lnet_configuration = self.assertState(self.lnet_configuration, 'lnet_up')
# Set boot_time to something that should change.
now = django.utils.timezone.now()
job_scheduler_notify.notify(freshen(self.host), now, {'boot_time': now})
self.assertEqual(freshen(self.host).boot_time, now)
# Not much later, but later enough (fastest boot EVAR).
later = django.utils.timezone.now()
self.assertNotEqual(later, now)
# This is more direct than fooling around with trying to get the
# timing right. Contrive a locking event on the host we want to
# notify, and the notification should be buffered.
self.job_scheduler._lock_cache.all_by_item[self.host] = ["fake lock"]
job_scheduler_notify.notify(freshen(self.host), later, {'boot_time': later})
# Now, remove the lock and make sure that the second notification
# didn't get through during the lock.
del(self.job_scheduler._lock_cache.all_by_item[self.host])
self.assertEqual(freshen(self.host).boot_time, now)
# Run any job, doesn't matter -- we just want to ensure that the
# notification buffer is drained after the job completes.
self.lnet_configuration = self.set_and_assert_state(self.lnet_configuration, 'lnet_down')
self.assertEqual(freshen(self.host).boot_time, later)
# Just for completeness, check that the notification buffer for this
# host was completely drained and removed.
buffer_key = (tuple(self.host.content_type.natural_key()), self.host.pk)
self.assertEqual([], self.job_scheduler._notification_buffer.drain_notifications_for_key(buffer_key))
self.assertEqual([], self.job_scheduler._notification_buffer.notification_keys)
def test_notification(self):
"""Test that state notifications cause the state of an object to change"""
self.lnet_configuration = self.assertState(self.lnet_configuration,
"lnet_up")
now = django.utils.timezone.now()
job_scheduler_notify.notify(freshen(self.lnet_configuration), now,
{"state": "lnet_down"}, ["lnet_up"])
self.assertEqual(freshen(self.lnet_configuration).state, "lnet_down")
def test_reformat_idempotency(self):
"""
Test that if a volume format passes its initial check for existing filesystems,
then it will format successfully even if the initial format operation is stopped
and restarted. To do that it has to pass reformat=True the second time
"""
path = self.mgt.managedtargetmount_set.get().volume_node.path
try:
MockAgentRpc.fail_commands = [('format_target', {
'device': path,
'target_types': 'mgs',
'backfstype': 'ldiskfs',
'device_type': 'linux',
'target_name': 'MGS'
})]
command = self.set_and_assert_state(self.mgt.managedtarget_ptr,
'formatted',
check=False)
self.assertEqual(freshen(command).complete, True)
self.assertEqual(freshen(command).errored, True)
finally:
MockAgentRpc.fail_commands = []
# Check that the initial format did not pass the reformat flag
self.assertEqual(MockAgentRpc.skip_calls(['device_plugin']),
('format_target', {
'device': path,
'target_types': 'mgs',
'backfstype': 'ldiskfs',
'device_type': 'linux',
'target_name': 'MGS'
}))
# This one should succeed
self.set_and_assert_state(self.mgt.managedtarget_ptr,
'formatted',
check=True)
# Check that it passed the reformat flag
self.assertEqual(
MockAgentRpc.skip_calls(['device_plugin', 'export_target']),
('format_target', {
'device': path,
'target_types': 'mgs',
'backfstype': 'ldiskfs',
'device_type': 'linux',
'target_name': 'MGS',
'reformat': True
}))
def test_onejob(self):
# Our self.host is initially lnet_up
self.assertEqual(
LNetConfiguration.objects.get(pk=self.lnet_configuration.pk).state,
"lnet_up")
# This tests a state transition which is done by a single job
command_id = JobSchedulerClient.command_run_jobs(
[{
"class_name": "UpdateDevicesJob",
"args": {
"hosts": [api.get_resource_uri(self.host)]
}
}],
"Test single job action",
)
self.drain_progress()
self.assertEqual(Command.objects.get(pk=command_id).complete, True)
self.assertEqual(Command.objects.get(pk=command_id).jobs.count(), 1)
# Test that if I try to run the same again I get None
command = Command.set_state([(freshen(self.lnet_configuration),
"lnet_up")])
self.assertEqual(command, None)
def set_state(objects, message=None, **kwargs):
command = context.old_set_state(objects, message=message, **kwargs)
context.test_case.drain_progress()
if command:
return freshen(command)
else:
return command
def test_dismissing_alert(self):
"""Send a API PATCH to update Alert.dismissed to True with del obj
HostOfflineAlert.alert_item is a GenericForeignKey. This will test that
item being set, but deleted
"""
alert = self.make_alertstate(HostOfflineAlert,
dismissed=False,
severity=WARNING,
created_at=timezone.now())
self.assertEqual(alert.dismissed, False)
self.assertEqual(type(alert.alert_item), ManagedHost)
alert.alert_item.mark_deleted()
# Make sure it is deleted.
self.assertRaises(ManagedHost.DoesNotExist,
ManagedHost.objects.get,
pk=alert.alert_item.pk)
# Should not be able to PATCH this to dismissed without a failure
data = {"dismissed": 'true'}
response = self.api_client.patch("/api/alert/%s/" % alert.pk,
data=data)
self.assertHttpAccepted(response)
alert = freshen(alert)
self.assertEqual(alert.dismissed, True)
def test_removal(self):
from chroma_core.models import ManagedMgs
self.mgt.managedtarget_ptr = self.set_and_assert_state(freshen(self.mgt.managedtarget_ptr), "removed")
with self.assertRaises(ManagedMgs.DoesNotExist):
ManagedMgs.objects.get(pk=self.mgt.pk)
self.assertEqual(ManagedMgs._base_manager.get(pk=self.mgt.pk).state, "removed")
def test_target_start(self):
from chroma_core.models import ManagedMdt, ManagedOst, ManagedFilesystem
self.fs = self.set_and_assert_state(self.fs, "stopped")
self.mdt.managedtarget_ptr = self.set_and_assert_state(
freshen(self.mdt.managedtarget_ptr), "mounted")
self.assertEqual(
ManagedMdt.objects.get(pk=self.mdt.pk).state, "mounted")
self.assertEqual(
ManagedOst.objects.get(pk=self.ost.pk).state, "unmounted")
self.assertEqual(
ManagedFilesystem.objects.get(pk=self.fs.pk).state, "stopped")
self.ost.managedtarget_ptr = self.set_and_assert_state(
freshen(self.ost.managedtarget_ptr), "mounted")
self.assertState(self.fs, "available")
def test_2steps(self):
self.assertEqual(LNetConfiguration.objects.get(pk = self.lnet_configuration.pk).state, 'lnet_up')
# This tests a state transition which requires two jobs acting on the same object
# lnet_up -> lnet_down issues an StopLNetJob and a UnloadLNetJob
command_id = Command.set_state([(freshen(self.lnet_configuration), 'lnet_unloaded')]).id
self.drain_progress()
self.assertEqual(LNetConfiguration.objects.get(pk = self.lnet_configuration.pk).state, 'lnet_unloaded')
self.assertEqual(Command.objects.get(pk = command_id).complete, True)
self.assertEqual(Command.objects.get(pk = command_id).jobs.count(), 2)
def test_forget(self):
self.fs = self.set_and_assert_state(self.fs, "forgotten")
self.mgt.managedtarget_ptr = self.set_and_assert_state(
self.mgt.managedtarget_ptr, "forgotten")
with self.assertRaises(ManagedMgs.DoesNotExist):
freshen(self.mgt)
with self.assertRaises(ManagedFilesystem.DoesNotExist):
freshen(self.fs)
with self.assertRaises(ManagedMdt.DoesNotExist):
freshen(self.mdt)
with self.assertRaises(ManagedOst.DoesNotExist):
freshen(self.ost)
def create_simple_filesystem(self, host, start=True):
from chroma_core.models import ManagedMgs, ManagedMdt, ManagedOst, ManagedFilesystem
self.mgt, mgt_tms = ManagedMgs.create_for_volume(
self._test_lun(host).id, name="MGS")
self.fs = ManagedFilesystem.objects.create(mgs=self.mgt, name="testfs")
ObjectCache.add(ManagedFilesystem, self.fs)
self.mdt, mdt_tms = ManagedMdt.create_for_volume(
self._test_lun(host).id, filesystem=self.fs)
self.ost, ost_tms = ManagedOst.create_for_volume(
self._test_lun(host).id, filesystem=self.fs)
for target in [self.mgt, self.ost, self.mdt]:
ObjectCache.add(ManagedTarget, target.managedtarget_ptr)
for tm in chain(mgt_tms, mdt_tms, ost_tms):
ObjectCache.add(ManagedTargetMount, tm)
if start:
self.fs = self.set_and_assert_state(self.fs, 'available')
self.mgt = freshen(self.mgt)
self.mdt = freshen(self.mdt)
self.ost = freshen(self.ost)
def test_cancel_complete(self):
"""Test cancelling a Job which is in state 'complete': should be
a no-op
"""
self.set_state_delayed([(self.lnet_configuration, "lnet_down")])
job = Job.objects.get(state="pending")
# Run, check that it goes to successful state
self.set_state_complete()
job = freshen(job)
self.assertEqual(job.state, "complete")
self.assertEqual(job.cancelled, False)
self.assertEqual(job.errored, False)
# Try to cancel, check that it is not modified
JobSchedulerClient.cancel_job(job.id)
job = freshen(job)
self.assertEqual(job.state, "complete")
self.assertEqual(job.cancelled, False)
self.assertEqual(job.errored, False)
self.assertFalse(self.job_scheduler._lock_cache.get_by_job(job))
def test_ost_changes(self):
self.fs = self.set_and_assert_state(self.fs, "stopped")
ost_new, ost_new_tms = ManagedOst.create_for_volume(self._test_lun(
self.host).id,
filesystem=self.fs)
ObjectCache.add(ManagedTarget, ost_new.managedtarget_ptr)
for tm in ost_new_tms:
ObjectCache.add(ManagedTargetMount, tm)
self.mgt.managedtarget_ptr = self.set_and_assert_state(
freshen(self.mgt.managedtarget_ptr), "mounted")
self.mdt.managedtarget_ptr = self.set_and_assert_state(
freshen(self.mdt.managedtarget_ptr), "mounted")
self.ost.managedtarget_ptr = self.set_and_assert_state(
freshen(self.ost.managedtarget_ptr), "mounted")
ost_new.managedtarget_ptr = self.set_and_assert_state(
ost_new.managedtarget_ptr, "mounted")
self.assertState(self.fs, "available")
ost_new.managedtarget_ptr = self.set_and_assert_state(
ost_new.managedtarget_ptr, "unmounted")
self.assertState(self.fs, "unavailable")
ost_new.managedtarget_ptr = self.set_and_assert_state(
ost_new.managedtarget_ptr, "removed")
self.assertState(self.fs, "available")
def test_dismissing_alert(self):
"""Test dismissing alert by fs users is prevented"""
alert = self.make_alertstate(HostOfflineAlert,
dismissed=False,
severity=WARNING,
created_at=timezone.now())
self.assertEqual(alert.dismissed, False)
data = {"dismissed": 'true'}
response = self.api_client.patch("/api/alert/%s/" % alert.pk,
data=data)
self.assertHttpUnauthorized(response)
alert = freshen(alert)
self.assertEqual(alert.dismissed, False)
def test_dismissing_alert(self):
"""Test dismissing alert by fs admins is allowed"""
alert = self.make_alertstate(HostOfflineAlert,
dismissed=False,
severity=WARNING,
created_at=timezone.now())
self.assertEqual(alert.dismissed, False)
data = {"dismissed": "true"}
response = self.api_client.patch("/api/alert/%s/" % alert.pk,
data=data)
self.assertHttpAccepted(response)
alert = freshen(alert)
self.assertEqual(alert.dismissed, True)
def setUp(self):
super(TestDetectedFSTransitions, self).setUp()
self.create_simple_filesystem(self.host, start=False)
self.assertEqual(
ManagedMgs.objects.get(pk=self.mgt.pk).state, "unformatted")
self.assertEqual(
ManagedMdt.objects.get(pk=self.mdt.pk).state, "unformatted")
self.assertEqual(
ManagedOst.objects.get(pk=self.ost.pk).state, "unformatted")
self.fs = self.set_and_assert_state(self.fs, "available")
for obj in [self.mgt, self.mdt, self.ost, self.fs]:
obj = freshen(obj)
obj.immutable_state = True
obj.save()
def test_host_complete_job(self):
"""If a DeployHostJob completes in failure, the host should be in state "undeploy" """
job_scheduler = JobScheduler()
load_default_profile()
host = synthetic_host()
host.state = "undeployed"
host.save()
deploy_host_job = DeployHostJob.objects.create(managed_host=host)
deploy_host_job.locks_json = "{}"
job_scheduler._complete_job(deploy_host_job,
errored=True,
cancelled=False)
host = freshen(host)
self.assertEqual(host.state, "undeployed")
def test_dismissing_alert(self):
"""Test dismissing alert, not logged in is prevented"""
alert = self.make_alertstate(HostOfflineAlert,
dismissed=False,
severity=WARNING,
created_at=timezone.now())
self.assertEqual(alert.dismissed, False)
self.api_client.client.logout()
# ensure logged off
self.assertFalse(self.api_client.client.session)
data = {"dismissed": "true"}
response = self.api_client.patch("/api/alert/%s/" % alert.pk,
data=data)
self.assertHttpUnauthorized(response)
alert = freshen(alert)
self.assertEqual(alert.dismissed, False)
def test_dismissing_alert(self):
"""Send a API PATCH to update Alert.dismissed to True"""
alert = self.make_alertstate(HostOfflineAlert,
dismissed=False,
severity=WARNING,
created_at=timezone.now())
self.assertEqual(alert.dismissed, False)
path = '/api/alert/%s/' % alert.pk
# reject if severity isn't descriptive string as per the api
response = self.api_client.patch(path,
data={
'dismissed': True,
'severity': 10
})
self.assertHttpBadRequest(response)
response = self.api_client.patch(path, data={'dismissed': True})
self.assertHttpAccepted(response)
alert = freshen(alert)
self.assertEqual(alert.dismissed, True)
def test_reformat_idempotency(self):
"""
Test that if a volume format passes its initial check for existing filesystems,
then it will format successfully even if the initial format operation is stopped
and restarted. To do that it has to pass reformat=True the second time
"""
path = self.mgt.managedtargetmount_set.get().volume_node.path
try:
MockAgentRpc.fail_commands = [
(
"format_target",
{
"device": path,
"target_types": "mgs",
"backfstype": "ldiskfs",
"device_type": "linux",
"target_name": "MGS",
},
)
]
command = self.set_and_assert_state(self.mgt.managedtarget_ptr, "formatted", check=False)
self.assertEqual(freshen(command).complete, True)
self.assertEqual(freshen(command).errored, True)
finally:
MockAgentRpc.fail_commands = []
# Check that the initial format did not pass the reformat flag
self.assertEqual(
MockAgentRpc.skip_calls(["device_plugin"]),
(
"format_target",
{
"device": path,
"target_types": "mgs",
"backfstype": "ldiskfs",
"device_type": "linux",
"target_name": "MGS",
},
),
)
# This one should succeed
self.set_and_assert_state(self.mgt.managedtarget_ptr, "formatted", check=True)
# Check that it passed the reformat flag
self.assertEqual(
MockAgentRpc.skip_calls(["device_plugin", "export_target"]),
(
"format_target",
{
"device": path,
"target_types": "mgs",
"backfstype": "ldiskfs",
"device_type": "linux",
"target_name": "MGS",
"reformat": True,
},
),
)
def create_targets(*args, **kwargs):
targets, command = context.old_create_targets(*args, **kwargs)
context.test_case.drain_progress()
return [freshen(t) for t in targets], freshen(command)
def assertState(self, obj, state):
obj = freshen(obj)
self.assertEqual(obj.state, state)
return obj
