def setUp(self):
super(TestOrderedTargets, self).setUp()
# If the test that just ran imported storage_plugin_manager, it will
# have instantiated its singleton, and created some DB records.
# Django TestCase rolls back the database, so make sure that we
# also roll back (reset) this singleton.
import chroma_core.lib.storage_plugin.manager
chroma_core.lib.storage_plugin.manager.storage_plugin_manager = (
chroma_core.lib.storage_plugin.manager.StoragePluginManager())
load_default_profile()
self.job_scheduler = JobScheduler()
self.no_of_nodes = 10
self.nodes = []
for node in range(0, self.no_of_nodes):
self.nodes.append(synthetic_host("node%s" % node))
for node in self.nodes:
synthetic_volume_full(
node, secondary_hosts=list(set(self.nodes) - set([node])))
self.volume_ids = [volume.id for volume in Volume.objects.all()]
def setUp(self):
super(TestAvailableJobs, self).setUp()
from chroma_core.services.job_scheduler.job_scheduler import JobScheduler
from tests.unit.chroma_core.helpers import load_default_profile
load_default_profile()
self.JobScheduler = JobScheduler
self.js = JobScheduler()
volume = synthetic_volume(with_storage=False)
# Create object before ObjectCache init, so they are in the cache.
self.host = synthetic_host()
self.mgs = ManagedMgs.objects.create(volume=volume)
self.fs = ManagedFilesystem.objects.create(name="mgsfs", mgs=self.mgs)
self.mdt = ManagedMdt.objects.create(volume=volume,
filesystem=self.fs,
index=1)
self.ost = ManagedOst.objects.create(volume=volume,
filesystem=self.fs,
index=1)
# If you create object after init of this case, they will not be in it.
ObjectCache.getInstance()
def setUp(self):
super(TestAvailableJobs, self).setUp()
from chroma_core.services.job_scheduler.job_scheduler import JobScheduler
from tests.unit.chroma_core.helpers import load_default_profile
load_default_profile()
self.JobScheduler = JobScheduler
self.js = JobScheduler()
# Create object before ObjectCache init, so they are in the cache.
self.host = synthetic_host()
(mgt, fs, mdt, ost) = create_simple_fs()
self.mgs = mgt
self.fs = fs
self.mdt = mdt
self.ost = ost
# If you create object after init of this case, they will not be in it.
ObjectCache.getInstance()
def setUp(self):
super(ChromaApiTestCase, self).setUp()
from chroma_api.authentication import CsrfAuthentication
self.old_is_authenticated = CsrfAuthentication.is_authenticated
CsrfAuthentication.is_authenticated = mock.Mock(return_value=True)
self.api_client.client.login(username=self.username,
password=self.password)
from tests.unit.chroma_core.helpers import load_default_profile
load_default_profile()
# If the test that just ran imported storage_plugin_manager, it will
# have instantiated its singleton, and created some DB records.
# Django TestCase rolls back the database, so make sure that we
# also roll back (reset) this singleton.
import chroma_core.lib.storage_plugin.manager
chroma_core.lib.storage_plugin.manager.storage_plugin_manager = chroma_core.lib.storage_plugin.manager.StoragePluginManager(
)
# Fetching available transitions from the JobSchedule would require
# Avoiding moving this to integration scope by mocking the rpc
# fetched method.
@classmethod
def fake_available_transitions(cls, object_list):
transitions = defaultdict(list)
for obj_ct, obj_id in object_list:
obj = JobScheduler._retrieve_stateful_object(obj_ct, obj_id)
# fake receiving ['fake_trans', 'make_fake_trans'] from JS
# and converting to this form with verbs
transitions[obj.id] = [{
'state': 'fake_trans',
'verb': 'make_fake_trans'
}, {
'state': 'fake_other',
'verb': 'make_fake_other'
}]
return transitions
from chroma_core.services.job_scheduler import job_scheduler_client
self.old_available_transitions = job_scheduler_client.JobSchedulerClient.available_transitions
job_scheduler_client.JobSchedulerClient.available_transitions = fake_available_transitions
@classmethod
def fake_available_jobs(cls, object_list):
jobs = defaultdict(list)
for obj_ct, obj_id in object_list:
obj = JobScheduler._retrieve_stateful_object(obj_ct, obj_id)
jobs[obj.id] = []
return jobs
self.old_available_jobs = job_scheduler_client.JobSchedulerClient.available_jobs
job_scheduler_client.JobSchedulerClient.available_jobs = fake_available_jobs
@classmethod
def fake_get_locks(cls, obj_key, obj_id):
return {'read': [1, 2], 'write': [3, 4]}
self.old_get_locks = job_scheduler_client.JobSchedulerClient.get_locks
job_scheduler_client.JobSchedulerClient.get_locks = fake_get_locks
def setUp(self):
super(TestAdvertisedHostJobs, self).setUp()
load_default_profile()
self.host = synthetic_host()
self.set_managed(True)
self.host.state = self.normal_host_state
def setUp(self):
super(TestRebalancePassthrough, self).setUp()
load_default_profile()
# Initialise storage plugin stuff for the benefit of synthetic_volume_full
import chroma_core.lib.storage_plugin.manager
chroma_core.lib.storage_plugin.manager.storage_plugin_manager = chroma_core.lib.storage_plugin.manager.StoragePluginManager(
)
def setUp(self):
super(TestAvailableTransitions, self).setUp()
self.js = JobScheduler()
load_default_profile()
self.host = synthetic_host()
self.assertEqual(self.host.state, "managed")
def setUp(self):
super(TestAdvertisedTargetJobs, self).setUp()
load_default_profile()
self.target = mock.Mock()
self.target.immutable_state = False
self.target.failover_hosts = [synthetic_host()]
self.target.primary_host = synthetic_host()
self.target.active_host = self.target.primary_host
def setUp(self):
super(TestAvailableTransitions, self).setUp()
self.js = JobScheduler()
self.volume = synthetic_volume(with_storage=False)
load_default_profile()
self.host = synthetic_host()
self.assertEqual(self.host.state, 'managed')
def setUp(self):
super(TestAudit, self).setUp()
load_default_profile()
mock.patch(
'chroma_core.services.job_scheduler.job_scheduler_notify.notify'
).start()
self.addCleanup(mock.patch.stopall)
def setUp(self):
super(TestNidStrings, self).setUp()
# If the test that just ran imported storage_plugin_manager, it will
# have instantiated its singleton, and created some DB records.
# Django TestCase rolls back the database, so make sure that we
# also roll back (reset) this singleton.
import chroma_core.lib.storage_plugin.manager
chroma_core.lib.storage_plugin.manager.storage_plugin_manager = chroma_core.lib.storage_plugin.manager.StoragePluginManager()
load_default_profile()
def setUp(self):
super(TestRegistration, self).setUp()
load_default_profile()
self.old_create_host = JobSchedulerClient.create_host
JobSchedulerClient.create_host = mock.Mock(
side_effect=lambda *args, **kwargs:
(synthetic_host('mynewhost', **self.mock_servers['mynewhost']),
mock.Mock(id='bar')))
ValidatedClientView.valid_certs = {}
def setUp(self):
super(TestFetchJobs, self).setUp()
# Don't need to load all of JobScheduler's dependencies.
from chroma_core.services.job_scheduler import job_scheduler
self.old__init__ = job_scheduler.JobScheduler.__init__
del job_scheduler.JobScheduler.__init__
self.job_scheduler = job_scheduler.JobScheduler()
from tests.unit.chroma_core.helpers import load_default_profile
load_default_profile()
def setUp(self):
super(TestJobs, self).setUp()
helpers.load_default_profile()
for host_id in range(0, 10):
host = helpers.synthetic_host(address='myserver%s' % host_id)
self.hosts.append(host)
self.reset_invoke_capture()
assert 'fake' not in str(Step.invoke_agent)
mock.patch("chroma_core.lib.job.Step.invoke_agent", self._fake_invoke_agent).start()
self.addCleanup(mock.patch.stopall)
def setUp(self):
super(TestClientManagementJobs, self).setUp()
load_default_profile()
worker_profile = self.load_worker_profile()
self.worker = synthetic_host(server_profile=worker_profile.name)
self.worker.immutable_state = False
self.worker.state = self.normal_host_state
self.server = synthetic_host()
self.server.immutable_state = False
self.server.state = self.normal_host_state
# If the test that just ran imported storage_plugin_manager, it will
# have instantiated its singleton, and created some DB records.
# Django TestCase rolls back the database, so make sure that we
# also roll back (reset) this singleton.
import chroma_core.lib.storage_plugin.manager
chroma_core.lib.storage_plugin.manager.storage_plugin_manager = chroma_core.lib.storage_plugin.manager.StoragePluginManager()
def test_host_complete_job(self):
"""If a DeployHostJob completes in failure, the host should be in state "undeploy" """
job_scheduler = JobScheduler()
load_default_profile()
host = synthetic_host()
host.state = "undeployed"
host.save()
deploy_host_job = DeployHostJob.objects.create(managed_host=host)
deploy_host_job.locks_json = "{}"
job_scheduler._complete_job(deploy_host_job,
errored=True,
cancelled=False)
host = freshen(host)
self.assertEqual(host.state, "undeployed")
def test_host_available_states_undeployed(self):
"""Test that host is set to 'deploy-failed' when Job.on_error is called."""
load_default_profile()
host = synthetic_host()
host.state = "undeployed"
host.save()
self.assertEquals(host.state, "undeployed")
# Check the available states changes depending on how installed.
for install_method, states in {
ManagedHost.INSTALL_MANUAL: [],
ManagedHost.INSTALL_SSHSKY: ["managed"],
ManagedHost.INSTALL_SSHPKY: ["managed"],
ManagedHost.INSTALL_SSHSKY: ["managed"],
}.items():
host.install_method = install_method
self.assertEquals(host.get_available_states("undeployed"), states)
def setUp(self):
super(TestNidStrings, self).setUp()
# If the test that just ran imported storage_plugin_manager, it will
# have instantiated its singleton, and created some DB records.
# Django TestCase rolls back the database, so make sure that we
# also roll back (reset) this singleton.
import chroma_core.lib.storage_plugin.manager
chroma_core.lib.storage_plugin.manager.storage_plugin_manager = (
chroma_core.lib.storage_plugin.manager.StoragePluginManager())
load_default_profile()
def get_targets_fn():
ids = [x.id for x in ManagedHost.objects.all()]
host_id = ids[0]
return [
{
"name": "MGS",
"active_host_id": host_id,
"host_ids": [ids[0]]
},
{
"name": "MGS_ha",
"active_host_id": host_id,
"host_ids": [ids[0], ids[1]]
},
]
self.get_targets_mock = mock.MagicMock(side_effect=get_targets_fn)
mock.patch("chroma_core.lib.graphql.get_targets",
new=self.get_targets_mock).start()
(mgt, fs, mdt, ost) = create_simple_fs()
self.mgt = mgt
self.fs = fs
def setUp(self):
super(TestHandlers, self).setUp()
load_default_profile()
self.host = synthetic_host("myaddress")
def before_feature(context, feature):
context.runner.setup_test_environment()
context.old_db_config = context.runner.setup_databases()
from tests.unit.services.job_scheduler.job_test_case import JobTestCase
class BehaveFeatureTest(JobTestCase):
mock_servers = {}
def runTest(self):
pass
context.test_case = BehaveFeatureTest()
context.test_case._pre_setup()
context.test_case.setUp()
patch_test_host_contact_task(context)
from tests.unit.chroma_core.helpers import synthetic_host
from tests.unit.chroma_core.helpers import freshen
from chroma_core.lib.cache import ObjectCache
from chroma_core.models import ManagedHost, Command
from chroma_core.services.job_scheduler.agent_rpc import AgentRpc
def create_host_ssh(address, server_profile, root_pw, pkey, pkey_pw):
host_data = AgentRpc.mock_servers[address]
host = synthetic_host(address, nids=host_data["nids"], fqdn=host_data["fqdn"], nodename=host_data["nodename"])
ObjectCache.add(ManagedHost, host)
command = Command.objects.create(complete=True, message="Mock create_host_ssh")
return host, command
from chroma_core.services.job_scheduler.job_scheduler_client import JobSchedulerClient
context.old_create_host_ssh = JobSchedulerClient.create_host_ssh
JobSchedulerClient.create_host_ssh = mock.Mock(side_effect=create_host_ssh)
context.old_create_targets = JobSchedulerClient.create_targets
def create_targets(*args, **kwargs):
targets, command = context.old_create_targets(*args, **kwargs)
context.test_case.drain_progress()
return [freshen(t) for t in targets], freshen(command)
JobSchedulerClient.create_targets = mock.Mock(side_effect=create_targets)
context.old_create_filesystem = JobSchedulerClient.create_filesystem
def create_filesystem(*args, **kwargs):
filesystem_id, command_id = context.old_create_filesystem(*args, **kwargs)
context.test_case.drain_progress()
return filesystem_id, command_id
JobSchedulerClient.create_filesystem = mock.Mock(side_effect=create_filesystem)
context.old_set_state = Command.set_state
def set_state(objects, message=None, **kwargs):
command = context.old_set_state(objects, message=message, **kwargs)
context.test_case.drain_progress()
if command:
return freshen(command)
else:
return command
Command.set_state = mock.Mock(side_effect=set_state)
context.old_run_jobs = JobSchedulerClient.command_run_jobs
def command_run_jobs(jobs, message):
command_id = context.old_run_jobs(jobs, message)
context.test_case.drain_progress()
return command_id
JobSchedulerClient.command_run_jobs = mock.Mock(side_effect=command_run_jobs)
from chroma_cli.api import ApiHandle
context.old_api_client = ApiHandle.ApiClient
from tests.unit.chroma_api.tastypie_test import TestApiClient
ApiHandle.ApiClient = TestApiClient
from chroma_api.authentication import CsrfAuthentication
context.old_is_authenticated = CsrfAuthentication.is_authenticated
CsrfAuthentication.is_authenticated = mock.Mock(return_value=True)
from tests.unit.chroma_core.helpers import load_default_profile
from chroma_core.models import ServerProfile
if not ServerProfile.objects.all().exists():
load_default_profile()
def setUpTestData(cls):
load_default_profile()
def setUp(self):
super(ChromaApiTestCase, self).setUp()
from chroma_api.authentication import CsrfAuthentication
self.old_is_authenticated = CsrfAuthentication.is_authenticated
CsrfAuthentication.is_authenticated = mock.Mock(return_value=True)
self.assertTrue(
self.api_client.client.login(username=self.username,
password=self.password))
from tests.unit.chroma_core.helpers import load_default_profile
load_default_profile()
# If the test that just ran imported storage_plugin_manager, it will
# have instantiated its singleton, and created some DB records.
# Django TestCase rolls back the database, so make sure that we
# also roll back (reset) this singleton.
import chroma_core.lib.storage_plugin.manager
chroma_core.lib.storage_plugin.manager.storage_plugin_manager = (
chroma_core.lib.storage_plugin.manager.StoragePluginManager())
# Fetching available transitions from the JobSchedule would require
# Avoiding moving this to integration scope by mocking the rpc
# fetched method.
@classmethod
def fake_available_transitions(cls, object_list):
transitions = defaultdict(list)
for obj_ct, obj_id in object_list:
obj = JobScheduler._retrieve_stateful_object(obj_ct, obj_id)
# fake receiving ['fake_trans', 'make_fake_trans'] from JS
# and converting to this form with verbs
transitions[obj.id] = [
{
"state": "fake_trans",
"verb": "make_fake_trans"
},
{
"state": "fake_other",
"verb": "make_fake_other"
},
]
return transitions
from chroma_core.services.job_scheduler import job_scheduler_client
self.old_available_transitions = job_scheduler_client.JobSchedulerClient.available_transitions
job_scheduler_client.JobSchedulerClient.available_transitions = fake_available_transitions
@classmethod
def fake_available_jobs(cls, object_list):
jobs = defaultdict(list)
for obj_ct, obj_id in object_list:
obj = JobScheduler._retrieve_stateful_object(obj_ct, obj_id)
jobs[obj.id] = []
return jobs
self.old_available_jobs = job_scheduler_client.JobSchedulerClient.available_jobs
job_scheduler_client.JobSchedulerClient.available_jobs = fake_available_jobs
@classmethod
def fake_get_locks(cls, obj_key, obj_id):
return {"read": [1, 2], "write": [3, 4]}
self.old_get_locks = job_scheduler_client.JobSchedulerClient.get_locks
job_scheduler_client.JobSchedulerClient.get_locks = fake_get_locks
def get_targets_fn():
from chroma_core.models import ManagedHost
ids = [x.id for x in ManagedHost.objects.all()]
host_id = ids[0]
return [
{
"name": "MGS",
"active_host_id": host_id,
"host_ids": ids,
"uuid": "uuid_mgt"
},
{
"name": "testfs-MDT0000",
"active_host_id": host_id,
"host_ids": ids,
"uuid": "uuid_mdt"
},
{
"name": "testfs-OST0000",
"active_host_id": host_id,
"host_ids": ids,
"uuid": "uuid_ost0"
},
{
"name": "testfs-OST0001",
"active_host_id": host_id,
"host_ids": ids,
"uuid": "uuid_ost1"
},
]
self.get_targets_mock = mock.MagicMock(side_effect=get_targets_fn)
mock.patch("chroma_core.lib.graphql.get_targets",
new=self.get_targets_mock).start()
self.addCleanup(mock.patch.stopall)
def setUp(self):
super(TestDetection, self).setUp()
load_default_profile()
def setUp(self):
super(JobTestCase, self).setUp()
from chroma_core.services.http_agent import HttpAgentRpc
from chroma_core.services.http_agent import Service as HttpAgentService
# FIXME: have to do self before every test because otherwise
# one test will get all the setup of StoragePluginClass records,
# the in-memory instance of storage_plugin_manager will expect
# them to still be there but they'll have been cleaned
# out of the database. Setting up self stuff should be done
# as part of the initial DB setup before any test is started
# so that it's part of the baseline that's rolled back to
# after each test.
import chroma_core.lib.storage_plugin.manager
chroma_core.lib.storage_plugin.manager.storage_plugin_manager = (
chroma_core.lib.storage_plugin.manager.StoragePluginManager())
# Intercept attempts to call out to lustre servers
import chroma_core.services.job_scheduler.agent_rpc
self.old_agent_rpc = chroma_core.services.job_scheduler.agent_rpc.AgentRpc
self.old_agent_ssh = chroma_core.services.job_scheduler.agent_rpc.AgentSsh
MockAgentRpc.mock_servers = self.mock_servers
MockAgentSsh.mock_servers = self.mock_servers
chroma_core.services.job_scheduler.agent_rpc.AgentRpc = MockAgentRpc
chroma_core.services.job_scheduler.agent_rpc.AgentSsh = MockAgentSsh
# Any RPCs that are going to get called need explicitly overriding to
# turn into local calls -- self is a catch-all to prevent any RPC classes
# from trying to do network comms during unit tests
ServiceRpcInterface._call = mock.Mock(side_effect=NotImplementedError)
ServiceQueue.put = mock.Mock(side_effect=NotImplementedError)
ServiceQueue.purge = mock.Mock()
# Create an instance for the purposes of the test
from chroma_core.services.plugin_runner.resource_manager import ResourceManager
resource_manager = ResourceManager()
from chroma_core.services.plugin_runner import AgentPluginHandlerCollection
def patch_daemon_rpc(rpc_class, test_daemon):
# Patch AgentDaemonRpc to call our instance instead of trying to do an RPC
def rpc_local(fn_name, *args, **kwargs):
# Run the response through a serialize/deserialize cycle to
# give it that special RPC flavor.
retval = json.loads(
json.dumps(getattr(test_daemon, fn_name)(*args, **kwargs)))
log.info("patch_daemon_rpc: %s(%s %s) -> %s" %
(fn_name, args, kwargs, retval))
return retval
rpc_class._call = mock.Mock(side_effect=rpc_local)
aphc = AgentPluginHandlerCollection(resource_manager)
patch_daemon_rpc(AgentDaemonRpcInterface, aphc)
aphc.update_host_resources = mock.Mock(
side_effect=parse_synthentic_device_info)
patch_daemon_rpc(HttpAgentRpc, HttpAgentService())
from chroma_core.services.job_scheduler.dep_cache import DepCache
from chroma_core.services.job_scheduler.job_scheduler import JobScheduler, RunJobThread
from chroma_core.services.job_scheduler.job_scheduler_client import JobSchedulerRpc
from chroma_core.services.job_scheduler.job_scheduler_notify import NotificationQueue
ObjectCache.clear()
self.job_scheduler = JobScheduler()
patch_daemon_rpc(JobSchedulerRpc, self.job_scheduler)
# self.job_scheduler.progress.put = mock.Mock(side_effect = lambda msg: self.job_scheduler.progress._handle(msg))
# self.job_scheduler.progress.advance = mock.Mock(side_effect = lambda msg: self.job_scheduler.progress._handle(msg))
from chroma_core.services.job_scheduler import QueueHandler
job_scheduler_queue_handler = QueueHandler(self.job_scheduler)
def job_scheduler_queue_immediate(body):
log.info("job_scheduler_queue_immediate: %s" % body)
job_scheduler_queue_handler.on_message(body)
NotificationQueue.put = mock.Mock(
side_effect=job_scheduler_queue_immediate)
import chroma_core.services.job_scheduler.job_scheduler
chroma_core.services.job_scheduler.job_scheduler._disable_database = mock.Mock(
)
def _spawn_job(job):
log.debug("functional spawn job")
thread = RunJobThread(self.job_scheduler.progress,
self.job_scheduler._db_quota, job,
job.get_steps())
self.job_scheduler._run_threads[job.id] = thread
thread._run()
self.job_scheduler._spawn_job = mock.Mock(side_effect=_spawn_job)
def run_next():
while True:
runnable_jobs = self.job_scheduler._job_collection.ready_jobs
log.info(
"run_next: %d runnable jobs of (%d pending, %d tasked)" % (
len(runnable_jobs),
len(self.job_scheduler._job_collection.pending_jobs),
len(self.job_scheduler._job_collection.tasked_jobs),
))
if not runnable_jobs:
break
dep_cache = DepCache()
ok_jobs, cancel_jobs = self.job_scheduler._check_jobs(
runnable_jobs, dep_cache)
self.job_scheduler._job_collection.update_many(
ok_jobs, "tasked")
for job in cancel_jobs:
self.job_scheduler._complete_job(job, False, True)
for job in ok_jobs:
self.job_scheduler._spawn_job(job)
self.drain_progress(skip_advance=True)
JobScheduler._run_next = mock.Mock(side_effect=run_next)
#
# def complete_job(job, errored = False, cancelled = False):
# ObjectCache.clear()
# self.job_scheduler._complete_job(job, errored, cancelled)
# JobScheduler.complete_job = mock.Mock(side_effect=complete_job)
# Patch host removal because we use a _test_lun function that generates Volumes
# with no corresponding StorageResourceRecords, so the real implementation wouldn't
# remove them
def fake_remove_host_resources(host_id):
from chroma_core.models.host import Volume, VolumeNode
for vn in VolumeNode.objects.filter(host__id=host_id):
vn.mark_deleted()
for volume in Volume.objects.all():
if volume.volumenode_set.count() == 0:
volume.mark_deleted()
AgentDaemonRpcInterface.remove_host_resources = mock.Mock(
side_effect=fake_remove_host_resources)
load_default_profile()