def tearDown(self):
super(TestAvailableJobs, self).tearDown()
ObjectCache.clear()
connection.use_debug_cursor = False
def spider_api(self):
ObjectCache.clear()
from chroma_api.urls import api
xs = filter(lambda x: x[0] != "action", api._registry.items())
xs = filter(lambda x: "get" in x[1]._meta.list_allowed_methods, xs)
for _, resource in xs:
list_uri = resource.get_resource_uri()
response = self.api_client.get(list_uri, data={"limit": 0})
self.assertEqual(
response.status_code, 200, "%s: %s %s" %
(list_uri, response.status_code, self.deserialize(response)))
if "get" in resource._meta.detail_allowed_methods:
objects = self.deserialize(response)["objects"]
for object in objects:
# Deal with the newer bulk data format, if it is in that format.
if ("resource_uri" not in object) and (
"traceback" in object) and ("error" in object):
del object["traceback"]
del object["error"]
self.assertEqual(len(object), 1)
object = object.values()[0]
response = self.api_client.get(object["resource_uri"])
self.assertEqual(
response.status_code,
200,
"resource_url: %s, %s %s %s" %
(object["resource_uri"], response.status_code,
self.deserialize(response), object),
)
def spider_api(self):
ObjectCache.clear()
from chroma_api.urls import api
for name, resource in api._registry.items():
if 'get' in resource._meta.list_allowed_methods:
list_uri = resource.get_resource_uri()
response = self.api_client.get(list_uri, data={'limit': 0})
self.assertEqual(
response.status_code, 200,
"%s: %s %s" % (list_uri, response.status_code,
self.deserialize(response)))
if 'get' in resource._meta.detail_allowed_methods:
objects = self.deserialize(response)['objects']
for object in objects:
# Deal with the newer bulk data format, if it is in that format.
if ('resource_uri' not in object) and (
'traceback' in object) and ('error' in object):
del object['traceback']
del object['error']
self.assertEqual(len(object), 1)
object = object.values()[0]
response = self.api_client.get(object['resource_uri'])
self.assertEqual(
response.status_code, 200,
"resource_url: %s, %s %s %s" %
(object['resource_uri'], response.status_code,
self.deserialize(response), object))
def tearDown(self):
from chroma_api.authentication import CsrfAuthentication
CsrfAuthentication.is_authenticated = self.old_is_authenticated
# Restore
from chroma_core.services.job_scheduler import job_scheduler_client
job_scheduler_client.JobSchedulerClient.available_transitions = self.old_available_transitions
job_scheduler_client.JobSchedulerClient.available_jobs = self.old_available_jobs
ObjectCache.clear()
def _measure_scaling(self,
create_n,
measured_resource,
scaled_resource=None):
"""
:param create_n: Function to create N of scaled_resource
:param measured_resource: The resource we will measure the query load for
:param scaled_resource: The object which is actually being scaled with N
:return: Instance of Order1, OrderN, OrderBad
"""
if scaled_resource is None:
scaled_resource = measured_resource
query_counts = {}
samples = [5, 6, 7, 8]
for n in samples:
ObjectCache.clear()
create_n(n)
# Queries get reset at the start of a request
self.assertEqual(scaled_resource._meta.queryset.count(), n)
with CaptureQueriesContext(connection) as queries:
response = self.api_client.get(
"/api/%s/" % measured_resource._meta.resource_name,
data={"limit": 0})
self.assertEqual(
response.status_code, 200, "%s:%s" %
(response.content, measured_resource._meta.resource_name))
query_count = len(queries)
self.assertEqual(len(self.deserialize(response)["objects"]),
measured_resource._meta.queryset.count())
query_counts[n] = query_count
# Ignore samples[0], it was just to clear out any setup overhead from first call to API
# gradient between samples[1] and samples[2]
grad1 = (query_counts[samples[2]] -
query_counts[samples[1]]) / (samples[2] - samples[1])
# gradient between samples[2] and samples[3]
grad2 = (query_counts[samples[3]] -
query_counts[samples[2]]) / (samples[3] - samples[2])
if grad1 == 0 and grad2 == 0:
# Hoorah, O(1)
return Order1(query_counts[samples[3]])
elif grad1 > 0 and grad1 == grad2:
# O(N)
return OrderN(grad1)
else:
# Worse than O(N)
return OrderBad()
def tearDown(self):
super(TestAvailableTransitions, self).tearDown()
ObjectCache.clear()
def _passthrough_create_filesystem(target_data):
ObjectCache.clear()
return JobScheduler().create_filesystem(target_data)
def tearDown(self):
super(TestAvailableJobs, self).tearDown()
ObjectCache.clear()
def setUp(self):
super(JobTestCase, self).setUp()
from chroma_core.services.http_agent import HttpAgentRpc
from chroma_core.services.http_agent import Service as HttpAgentService
# FIXME: have to do self before every test because otherwise
# one test will get all the setup of StoragePluginClass records,
# the in-memory instance of storage_plugin_manager will expect
# them to still be there but they'll have been cleaned
# out of the database. Setting up self stuff should be done
# as part of the initial DB setup before any test is started
# so that it's part of the baseline that's rolled back to
# after each test.
import chroma_core.lib.storage_plugin.manager
chroma_core.lib.storage_plugin.manager.storage_plugin_manager = (
chroma_core.lib.storage_plugin.manager.StoragePluginManager())
# Intercept attempts to call out to lustre servers
import chroma_core.services.job_scheduler.agent_rpc
self.old_agent_rpc = chroma_core.services.job_scheduler.agent_rpc.AgentRpc
self.old_agent_ssh = chroma_core.services.job_scheduler.agent_rpc.AgentSsh
MockAgentRpc.mock_servers = self.mock_servers
MockAgentSsh.mock_servers = self.mock_servers
chroma_core.services.job_scheduler.agent_rpc.AgentRpc = MockAgentRpc
chroma_core.services.job_scheduler.agent_rpc.AgentSsh = MockAgentSsh
# Any RPCs that are going to get called need explicitly overriding to
# turn into local calls -- self is a catch-all to prevent any RPC classes
# from trying to do network comms during unit tests
ServiceRpcInterface._call = mock.Mock(side_effect=NotImplementedError)
ServiceQueue.put = mock.Mock()
ServiceQueue.purge = mock.Mock()
# Create an instance for the purposes of the test
from chroma_core.services.plugin_runner.resource_manager import ResourceManager
resource_manager = ResourceManager()
from chroma_core.services.plugin_runner import AgentPluginHandlerCollection
def patch_daemon_rpc(rpc_class, test_daemon):
# Patch AgentDaemonRpc to call our instance instead of trying to do an RPC
def rpc_local(fn_name, *args, **kwargs):
# Run the response through a serialize/deserialize cycle to
# give it that special RPC flavor.
retval = json.loads(
json.dumps(getattr(test_daemon, fn_name)(*args, **kwargs)))
log.info("patch_daemon_rpc: %s(%s %s) -> %s" %
(fn_name, args, kwargs, retval))
return retval
rpc_class._call = mock.Mock(side_effect=rpc_local)
aphc = AgentPluginHandlerCollection(resource_manager)
patch_daemon_rpc(AgentDaemonRpcInterface, aphc)
aphc.update_host_resources = mock.Mock(
side_effect=parse_synthentic_device_info)
patch_daemon_rpc(HttpAgentRpc, HttpAgentService())
from chroma_core.services.job_scheduler.dep_cache import DepCache
from chroma_core.services.job_scheduler.job_scheduler import JobScheduler, RunJobThread
from chroma_core.services.job_scheduler.job_scheduler_client import JobSchedulerRpc
from chroma_core.services.job_scheduler.job_scheduler_notify import NotificationQueue
ObjectCache.clear()
self.job_scheduler = JobScheduler()
patch_daemon_rpc(JobSchedulerRpc, self.job_scheduler)
# self.job_scheduler.progress.put = mock.Mock(side_effect = lambda msg: self.job_scheduler.progress._handle(msg))
# self.job_scheduler.progress.advance = mock.Mock(side_effect = lambda msg: self.job_scheduler.progress._handle(msg))
from chroma_core.services.job_scheduler import QueueHandler
job_scheduler_queue_handler = QueueHandler(self.job_scheduler)
def job_scheduler_queue_immediate(body):
log.info("job_scheduler_queue_immediate: %s" % body)
job_scheduler_queue_handler.on_message(body)
NotificationQueue.put = mock.Mock(
side_effect=job_scheduler_queue_immediate)
import chroma_core.services.job_scheduler.job_scheduler
chroma_core.services.job_scheduler.job_scheduler._disable_database = mock.Mock(
)
def _spawn_job(job):
log.debug("functional spawn job")
thread = RunJobThread(self.job_scheduler.progress,
self.job_scheduler._db_quota, job,
job.get_steps())
self.job_scheduler._run_threads[job.id] = thread
thread._run()
self.job_scheduler._spawn_job = mock.Mock(side_effect=_spawn_job)
def run_next():
while True:
runnable_jobs = self.job_scheduler._job_collection.ready_jobs
log.info(
"run_next: %d runnable jobs of (%d pending, %d tasked)" % (
len(runnable_jobs),
len(self.job_scheduler._job_collection.pending_jobs),
len(self.job_scheduler._job_collection.tasked_jobs),
))
if not runnable_jobs:
break
dep_cache = DepCache()
ok_jobs, cancel_jobs = self.job_scheduler._check_jobs(
runnable_jobs, dep_cache)
self.job_scheduler._job_collection.update_many(
ok_jobs, "tasked")
for job in cancel_jobs:
self.job_scheduler._complete_job(job, False, True)
for job in ok_jobs:
self.job_scheduler._spawn_job(job)
self.drain_progress(skip_advance=True)
JobScheduler._run_next = mock.Mock(side_effect=run_next)
#
# def complete_job(job, errored = False, cancelled = False):
# ObjectCache.clear()
# self.job_scheduler._complete_job(job, errored, cancelled)
# JobScheduler.complete_job = mock.Mock(side_effect=complete_job)
# Patch host removal because we use a _test_lun function that generates Volumes
# with no corresponding StorageResourceRecords, so the real implementation wouldn't
# remove them
def fake_remove_host_resources(host_id):
from chroma_core.models.host import Volume, VolumeNode
for vn in VolumeNode.objects.filter(host__id=host_id):
vn.mark_deleted()
for volume in Volume.objects.all():
if volume.volumenode_set.count() == 0:
volume.mark_deleted()
AgentDaemonRpcInterface.remove_host_resources = mock.Mock(
side_effect=fake_remove_host_resources)
def tearDown(self):
super(TestClientManagementJobs, self).tearDown()
# clear out the singleton to avoid polluting other tests
ObjectCache.clear()
def tearDown(self):
JobSchedulerClient.command_run_jobs = self.old_command_run_jobs
ObjectCache.clear()