def test_cancel_pending(self): """Test cancelling a Job which is in state 'pending'""" self.set_state_delayed([(self.host.lnet_configuration, "lnet_unloaded") ]) pending_jobs = Job.objects.filter(state="pending") # stop lnet, unload lnet self.assertEqual(pending_jobs.count(), 2) # This is the one we cancelled explicitly cancelled_job = pending_jobs[0] # This one should be cancelled as a result of cancelling it's dependency consequentially_cancelled_job = pending_jobs[1] JobSchedulerClient.cancel_job(pending_jobs[0].id) self.drain_progress() cancelled_job = freshen(cancelled_job) consequentially_cancelled_job = freshen(consequentially_cancelled_job) self.assertEqual(cancelled_job.state, "complete") self.assertEqual(cancelled_job.errored, False) self.assertEqual(cancelled_job.cancelled, True) self.assertEqual(consequentially_cancelled_job.state, "complete") self.assertEqual(consequentially_cancelled_job.errored, False) self.assertEqual(consequentially_cancelled_job.cancelled, True) pending_jobs = Job.objects.filter(state="pending") self.assertEqual(pending_jobs.count(), 0) self.assertFalse( self.job_scheduler._lock_cache.get_by_job(cancelled_job))
def test_late_notification(self): """Test that notifications are droppped when they are older than the last change to an objects state""" self.lnet_configuration = self.assertState(self.lnet_configuration, 'lnet_up') awhile_ago = django.utils.timezone.now() - datetime.timedelta(seconds = 120) job_scheduler_notify.notify(freshen(self.lnet_configuration), awhile_ago, {'state': 'lnet_down'}, ['lnet_up']) self.assertEqual(freshen(self.lnet_configuration).state, 'lnet_up')
def test_buffered_notification(self): """Test that notifications for locked items are buffered and replayed when the locking Job has completed.""" self.lnet_configuration = self.assertState(self.lnet_configuration, 'lnet_up') # Set boot_time to something that should change. now = django.utils.timezone.now() job_scheduler_notify.notify(freshen(self.host), now, {'boot_time': now}) self.assertEqual(freshen(self.host).boot_time, now) # Not much later, but later enough (fastest boot EVAR). later = django.utils.timezone.now() self.assertNotEqual(later, now) # This is more direct than fooling around with trying to get the # timing right. Contrive a locking event on the host we want to # notify, and the notification should be buffered. self.job_scheduler._lock_cache.all_by_item[self.host] = ["fake lock"] job_scheduler_notify.notify(freshen(self.host), later, {'boot_time': later}) # Now, remove the lock and make sure that the second notification # didn't get through during the lock. del(self.job_scheduler._lock_cache.all_by_item[self.host]) self.assertEqual(freshen(self.host).boot_time, now) # Run any job, doesn't matter -- we just want to ensure that the # notification buffer is drained after the job completes. self.lnet_configuration = self.set_and_assert_state(self.lnet_configuration, 'lnet_down') self.assertEqual(freshen(self.host).boot_time, later) # Just for completeness, check that the notification buffer for this # host was completely drained and removed. buffer_key = (tuple(self.host.content_type.natural_key()), self.host.pk) self.assertEqual([], self.job_scheduler._notification_buffer.drain_notifications_for_key(buffer_key)) self.assertEqual([], self.job_scheduler._notification_buffer.notification_keys)
def test_notification(self): """Test that state notifications cause the state of an object to change""" self.lnet_configuration = self.assertState(self.lnet_configuration, "lnet_up") now = django.utils.timezone.now() job_scheduler_notify.notify(freshen(self.lnet_configuration), now, {"state": "lnet_down"}, ["lnet_up"]) self.assertEqual(freshen(self.lnet_configuration).state, "lnet_down")
def test_reformat_idempotency(self): """ Test that if a volume format passes its initial check for existing filesystems, then it will format successfully even if the initial format operation is stopped and restarted. To do that it has to pass reformat=True the second time """ path = self.mgt.managedtargetmount_set.get().volume_node.path try: MockAgentRpc.fail_commands = [('format_target', { 'device': path, 'target_types': 'mgs', 'backfstype': 'ldiskfs', 'device_type': 'linux', 'target_name': 'MGS' })] command = self.set_and_assert_state(self.mgt.managedtarget_ptr, 'formatted', check=False) self.assertEqual(freshen(command).complete, True) self.assertEqual(freshen(command).errored, True) finally: MockAgentRpc.fail_commands = [] # Check that the initial format did not pass the reformat flag self.assertEqual(MockAgentRpc.skip_calls(['device_plugin']), ('format_target', { 'device': path, 'target_types': 'mgs', 'backfstype': 'ldiskfs', 'device_type': 'linux', 'target_name': 'MGS' })) # This one should succeed self.set_and_assert_state(self.mgt.managedtarget_ptr, 'formatted', check=True) # Check that it passed the reformat flag self.assertEqual( MockAgentRpc.skip_calls(['device_plugin', 'export_target']), ('format_target', { 'device': path, 'target_types': 'mgs', 'backfstype': 'ldiskfs', 'device_type': 'linux', 'target_name': 'MGS', 'reformat': True }))
def test_onejob(self): # Our self.host is initially lnet_up self.assertEqual( LNetConfiguration.objects.get(pk=self.lnet_configuration.pk).state, "lnet_up") # This tests a state transition which is done by a single job command_id = JobSchedulerClient.command_run_jobs( [{ "class_name": "UpdateDevicesJob", "args": { "hosts": [api.get_resource_uri(self.host)] } }], "Test single job action", ) self.drain_progress() self.assertEqual(Command.objects.get(pk=command_id).complete, True) self.assertEqual(Command.objects.get(pk=command_id).jobs.count(), 1) # Test that if I try to run the same again I get None command = Command.set_state([(freshen(self.lnet_configuration), "lnet_up")]) self.assertEqual(command, None)
def set_state(objects, message=None, **kwargs): command = context.old_set_state(objects, message=message, **kwargs) context.test_case.drain_progress() if command: return freshen(command) else: return command
def test_dismissing_alert(self): """Send a API PATCH to update Alert.dismissed to True with del obj HostOfflineAlert.alert_item is a GenericForeignKey. This will test that item being set, but deleted """ alert = self.make_alertstate(HostOfflineAlert, dismissed=False, severity=WARNING, created_at=timezone.now()) self.assertEqual(alert.dismissed, False) self.assertEqual(type(alert.alert_item), ManagedHost) alert.alert_item.mark_deleted() # Make sure it is deleted. self.assertRaises(ManagedHost.DoesNotExist, ManagedHost.objects.get, pk=alert.alert_item.pk) # Should not be able to PATCH this to dismissed without a failure data = {"dismissed": 'true'} response = self.api_client.patch("/api/alert/%s/" % alert.pk, data=data) self.assertHttpAccepted(response) alert = freshen(alert) self.assertEqual(alert.dismissed, True)
def test_removal(self): from chroma_core.models import ManagedMgs self.mgt.managedtarget_ptr = self.set_and_assert_state(freshen(self.mgt.managedtarget_ptr), "removed") with self.assertRaises(ManagedMgs.DoesNotExist): ManagedMgs.objects.get(pk=self.mgt.pk) self.assertEqual(ManagedMgs._base_manager.get(pk=self.mgt.pk).state, "removed")
def test_target_start(self): from chroma_core.models import ManagedMdt, ManagedOst, ManagedFilesystem self.fs = self.set_and_assert_state(self.fs, "stopped") self.mdt.managedtarget_ptr = self.set_and_assert_state( freshen(self.mdt.managedtarget_ptr), "mounted") self.assertEqual( ManagedMdt.objects.get(pk=self.mdt.pk).state, "mounted") self.assertEqual( ManagedOst.objects.get(pk=self.ost.pk).state, "unmounted") self.assertEqual( ManagedFilesystem.objects.get(pk=self.fs.pk).state, "stopped") self.ost.managedtarget_ptr = self.set_and_assert_state( freshen(self.ost.managedtarget_ptr), "mounted") self.assertState(self.fs, "available")
def test_2steps(self): self.assertEqual(LNetConfiguration.objects.get(pk = self.lnet_configuration.pk).state, 'lnet_up') # This tests a state transition which requires two jobs acting on the same object # lnet_up -> lnet_down issues an StopLNetJob and a UnloadLNetJob command_id = Command.set_state([(freshen(self.lnet_configuration), 'lnet_unloaded')]).id self.drain_progress() self.assertEqual(LNetConfiguration.objects.get(pk = self.lnet_configuration.pk).state, 'lnet_unloaded') self.assertEqual(Command.objects.get(pk = command_id).complete, True) self.assertEqual(Command.objects.get(pk = command_id).jobs.count(), 2)
def test_forget(self): self.fs = self.set_and_assert_state(self.fs, "forgotten") self.mgt.managedtarget_ptr = self.set_and_assert_state( self.mgt.managedtarget_ptr, "forgotten") with self.assertRaises(ManagedMgs.DoesNotExist): freshen(self.mgt) with self.assertRaises(ManagedFilesystem.DoesNotExist): freshen(self.fs) with self.assertRaises(ManagedMdt.DoesNotExist): freshen(self.mdt) with self.assertRaises(ManagedOst.DoesNotExist): freshen(self.ost)
def create_simple_filesystem(self, host, start=True): from chroma_core.models import ManagedMgs, ManagedMdt, ManagedOst, ManagedFilesystem self.mgt, mgt_tms = ManagedMgs.create_for_volume( self._test_lun(host).id, name="MGS") self.fs = ManagedFilesystem.objects.create(mgs=self.mgt, name="testfs") ObjectCache.add(ManagedFilesystem, self.fs) self.mdt, mdt_tms = ManagedMdt.create_for_volume( self._test_lun(host).id, filesystem=self.fs) self.ost, ost_tms = ManagedOst.create_for_volume( self._test_lun(host).id, filesystem=self.fs) for target in [self.mgt, self.ost, self.mdt]: ObjectCache.add(ManagedTarget, target.managedtarget_ptr) for tm in chain(mgt_tms, mdt_tms, ost_tms): ObjectCache.add(ManagedTargetMount, tm) if start: self.fs = self.set_and_assert_state(self.fs, 'available') self.mgt = freshen(self.mgt) self.mdt = freshen(self.mdt) self.ost = freshen(self.ost)
def test_cancel_complete(self): """Test cancelling a Job which is in state 'complete': should be a no-op """ self.set_state_delayed([(self.lnet_configuration, "lnet_down")]) job = Job.objects.get(state="pending") # Run, check that it goes to successful state self.set_state_complete() job = freshen(job) self.assertEqual(job.state, "complete") self.assertEqual(job.cancelled, False) self.assertEqual(job.errored, False) # Try to cancel, check that it is not modified JobSchedulerClient.cancel_job(job.id) job = freshen(job) self.assertEqual(job.state, "complete") self.assertEqual(job.cancelled, False) self.assertEqual(job.errored, False) self.assertFalse(self.job_scheduler._lock_cache.get_by_job(job))
def test_ost_changes(self): self.fs = self.set_and_assert_state(self.fs, "stopped") ost_new, ost_new_tms = ManagedOst.create_for_volume(self._test_lun( self.host).id, filesystem=self.fs) ObjectCache.add(ManagedTarget, ost_new.managedtarget_ptr) for tm in ost_new_tms: ObjectCache.add(ManagedTargetMount, tm) self.mgt.managedtarget_ptr = self.set_and_assert_state( freshen(self.mgt.managedtarget_ptr), "mounted") self.mdt.managedtarget_ptr = self.set_and_assert_state( freshen(self.mdt.managedtarget_ptr), "mounted") self.ost.managedtarget_ptr = self.set_and_assert_state( freshen(self.ost.managedtarget_ptr), "mounted") ost_new.managedtarget_ptr = self.set_and_assert_state( ost_new.managedtarget_ptr, "mounted") self.assertState(self.fs, "available") ost_new.managedtarget_ptr = self.set_and_assert_state( ost_new.managedtarget_ptr, "unmounted") self.assertState(self.fs, "unavailable") ost_new.managedtarget_ptr = self.set_and_assert_state( ost_new.managedtarget_ptr, "removed") self.assertState(self.fs, "available")
def test_dismissing_alert(self): """Test dismissing alert by fs users is prevented""" alert = self.make_alertstate(HostOfflineAlert, dismissed=False, severity=WARNING, created_at=timezone.now()) self.assertEqual(alert.dismissed, False) data = {"dismissed": 'true'} response = self.api_client.patch("/api/alert/%s/" % alert.pk, data=data) self.assertHttpUnauthorized(response) alert = freshen(alert) self.assertEqual(alert.dismissed, False)
def test_dismissing_alert(self): """Test dismissing alert by fs admins is allowed""" alert = self.make_alertstate(HostOfflineAlert, dismissed=False, severity=WARNING, created_at=timezone.now()) self.assertEqual(alert.dismissed, False) data = {"dismissed": "true"} response = self.api_client.patch("/api/alert/%s/" % alert.pk, data=data) self.assertHttpAccepted(response) alert = freshen(alert) self.assertEqual(alert.dismissed, True)
def setUp(self): super(TestDetectedFSTransitions, self).setUp() self.create_simple_filesystem(self.host, start=False) self.assertEqual( ManagedMgs.objects.get(pk=self.mgt.pk).state, "unformatted") self.assertEqual( ManagedMdt.objects.get(pk=self.mdt.pk).state, "unformatted") self.assertEqual( ManagedOst.objects.get(pk=self.ost.pk).state, "unformatted") self.fs = self.set_and_assert_state(self.fs, "available") for obj in [self.mgt, self.mdt, self.ost, self.fs]: obj = freshen(obj) obj.immutable_state = True obj.save()
def test_host_complete_job(self): """If a DeployHostJob completes in failure, the host should be in state "undeploy" """ job_scheduler = JobScheduler() load_default_profile() host = synthetic_host() host.state = "undeployed" host.save() deploy_host_job = DeployHostJob.objects.create(managed_host=host) deploy_host_job.locks_json = "{}" job_scheduler._complete_job(deploy_host_job, errored=True, cancelled=False) host = freshen(host) self.assertEqual(host.state, "undeployed")
def test_dismissing_alert(self): """Test dismissing alert, not logged in is prevented""" alert = self.make_alertstate(HostOfflineAlert, dismissed=False, severity=WARNING, created_at=timezone.now()) self.assertEqual(alert.dismissed, False) self.api_client.client.logout() # ensure logged off self.assertFalse(self.api_client.client.session) data = {"dismissed": "true"} response = self.api_client.patch("/api/alert/%s/" % alert.pk, data=data) self.assertHttpUnauthorized(response) alert = freshen(alert) self.assertEqual(alert.dismissed, False)
def test_dismissing_alert(self): """Send a API PATCH to update Alert.dismissed to True""" alert = self.make_alertstate(HostOfflineAlert, dismissed=False, severity=WARNING, created_at=timezone.now()) self.assertEqual(alert.dismissed, False) path = '/api/alert/%s/' % alert.pk # reject if severity isn't descriptive string as per the api response = self.api_client.patch(path, data={ 'dismissed': True, 'severity': 10 }) self.assertHttpBadRequest(response) response = self.api_client.patch(path, data={'dismissed': True}) self.assertHttpAccepted(response) alert = freshen(alert) self.assertEqual(alert.dismissed, True)
def test_reformat_idempotency(self): """ Test that if a volume format passes its initial check for existing filesystems, then it will format successfully even if the initial format operation is stopped and restarted. To do that it has to pass reformat=True the second time """ path = self.mgt.managedtargetmount_set.get().volume_node.path try: MockAgentRpc.fail_commands = [ ( "format_target", { "device": path, "target_types": "mgs", "backfstype": "ldiskfs", "device_type": "linux", "target_name": "MGS", }, ) ] command = self.set_and_assert_state(self.mgt.managedtarget_ptr, "formatted", check=False) self.assertEqual(freshen(command).complete, True) self.assertEqual(freshen(command).errored, True) finally: MockAgentRpc.fail_commands = [] # Check that the initial format did not pass the reformat flag self.assertEqual( MockAgentRpc.skip_calls(["device_plugin"]), ( "format_target", { "device": path, "target_types": "mgs", "backfstype": "ldiskfs", "device_type": "linux", "target_name": "MGS", }, ), ) # This one should succeed self.set_and_assert_state(self.mgt.managedtarget_ptr, "formatted", check=True) # Check that it passed the reformat flag self.assertEqual( MockAgentRpc.skip_calls(["device_plugin", "export_target"]), ( "format_target", { "device": path, "target_types": "mgs", "backfstype": "ldiskfs", "device_type": "linux", "target_name": "MGS", "reformat": True, }, ), )
def create_targets(*args, **kwargs): targets, command = context.old_create_targets(*args, **kwargs) context.test_case.drain_progress() return [freshen(t) for t in targets], freshen(command)
def assertState(self, obj, state): obj = freshen(obj) self.assertEqual(obj.state, state) return obj