def test_allow_partial(self):
def absent_uuids(response):
return [part for part in response.json()['error'].split() if util.is_valid_uuid(part)]
job_uuid_1, resp = util.submit_job(self.cook_url)
self.assertEqual(201, resp.status_code)
job_uuid_2, resp = util.submit_job(self.cook_url)
self.assertEqual(201, resp.status_code)
# Only valid job uuids
resp = util.query_jobs(self.cook_url, job=[job_uuid_1, job_uuid_2])
self.assertEqual(200, resp.status_code)
# Mixed valid, invalid job uuids
bogus_uuid = str(uuid.uuid4())
resp = util.query_jobs(self.cook_url, job=[job_uuid_1, job_uuid_2, bogus_uuid])
self.assertEqual(404, resp.status_code)
self.assertEqual([bogus_uuid], absent_uuids(resp))
resp = util.query_jobs(self.cook_url, job=[job_uuid_1, job_uuid_2, bogus_uuid], partial='false')
self.assertEqual(404, resp.status_code, resp.json())
self.assertEqual([bogus_uuid], absent_uuids(resp))
# Partial results with mixed valid, invalid job uuids
resp = util.query_jobs(self.cook_url, job=[job_uuid_1, job_uuid_2, bogus_uuid], partial='true')
self.assertEqual(200, resp.status_code, resp.json())
self.assertEqual(2, len(resp.json()))
self.assertEqual([job_uuid_1, job_uuid_2].sort(), [job['uuid'] for job in resp.json()].sort())
# Only valid instance uuids
job = util.wait_for_job(self.cook_url, job_uuid_1, 'completed')
instance_uuid_1 = job['instances'][0]['task_id']
job = util.wait_for_job(self.cook_url, job_uuid_2, 'completed')
instance_uuid_2 = job['instances'][0]['task_id']
resp = util.query_jobs(self.cook_url, instance=[instance_uuid_1, instance_uuid_2])
self.assertEqual(200, resp.status_code)
# Mixed valid, invalid instance uuids
resp = util.query_jobs(self.cook_url, instance=[instance_uuid_1, instance_uuid_2, bogus_uuid])
self.assertEqual(404, resp.status_code)
self.assertEqual([bogus_uuid], absent_uuids(resp))
resp = util.query_jobs(self.cook_url, instance=[instance_uuid_1, instance_uuid_2, bogus_uuid], partial='false')
self.assertEqual(404, resp.status_code)
self.assertEqual([bogus_uuid], absent_uuids(resp))
# Partial results with mixed valid, invalid instance uuids
resp = util.query_jobs(self.cook_url, instance=[instance_uuid_1, instance_uuid_2, bogus_uuid], partial='true')
self.assertEqual(200, resp.status_code)
self.assertEqual(2, len(resp.json()))
self.assertEqual([job_uuid_1, job_uuid_2].sort(), [job['uuid'] for job in resp.json()].sort())
def is_rate_limit_triggered(_):
jobs1 = util.query_jobs(self.cook_url,
True,
uuid=job_uuids).json()
waiting_jobs = [
j for j in jobs1 if j['status'] == 'waiting'
]
running_jobs = [
j for j in jobs1 if j['status'] == 'running'
]
# We submitted just under two buckets. We should only see a bucket + some extra running. No more.
return len(running_jobs) >= bucket_size and len(
waiting_jobs) > 0
def is_rate_limit_triggered(_):
jobs1 = util.query_jobs(self.cook_url,
True,
uuid=job_uuids).json()
running_jobs = [
j for j in jobs1 if j['status'] == 'running'
]
waiting_jobs = [
j for j in jobs1 if j['status'] == 'waiting'
]
self.logger.debug(
f'There are {len(waiting_jobs)} waiting jobs')
return len(waiting_jobs) > 0 and len(
running_jobs) >= bucket_size
def test_federated_query(self):
# Submit to cluster #1
job_uuid_1, resp = util.submit_job(self.cook_url_1)
self.assertEqual(resp.status_code, 201)
# Submit to cluster #2
job_uuid_2, resp = util.submit_job(self.cook_url_2)
self.assertEqual(resp.status_code, 201)
# Ask for both jobs from cluster #1, expect to get the first
resp = util.query_jobs(self.cook_url_1,
job=[job_uuid_1, job_uuid_2],
partial='true')
self.assertEqual(200, resp.status_code, resp.json())
self.assertEqual(1, len(resp.json()))
self.assertEqual([job_uuid_1], [job['uuid'] for job in resp.json()])
# Ask for both jobs from cluster #2, expect to get the second
resp = util.query_jobs(self.cook_url_2,
job=[job_uuid_1, job_uuid_2],
partial='true')
self.assertEqual(200, resp.status_code, resp.json())
self.assertEqual(1, len(resp.json()))
self.assertEqual([job_uuid_2], [job['uuid'] for job in resp.json()])
def get_job(self, job_uuid):
"""Loads a job by UUID using GET /rawscheduler"""
return util.query_jobs(self.cook_url, job=[job_uuid]).json()[0]
def test_rate_limit_launching_jobs(self):
settings = util.settings(self.cook_url)
if settings['rate-limit']['job-launch'] is None:
pytest.skip(
"Can't test job launch rate limit without launch rate limit set."
)
# Allow an environmental variable override.
name = os.getenv('COOK_LAUNCH_RATE_LIMIT_NAME')
if name is not None:
user = self.user_factory.user_class(name)
else:
user = self.user_factory.new_user()
if not settings['rate-limit']['job-launch']['enforce?']:
pytest.skip("Enforcing must be on for test to run")
bucket_size = settings['rate-limit']['job-launch']['bucket-size']
token_rate = settings['rate-limit']['job-launch'][
'tokens-replenished-per-minute']
# In some environments, e.g., minimesos, we can only launch so many concurrent jobs.
if token_rate < 5 or token_rate > 20:
pytest.skip(
"Job launch rate limit test is only validated to reliably work correctly with certain token rates."
)
if bucket_size < 10 or bucket_size > 20:
pytest.skip(
"Job launch rate limit test is only validated to reliably work correctly with certain token bucket sizes."
)
with user:
job_uuids = []
try:
jobspec = {"command": "sleep 240", 'cpus': 0.03, 'mem': 32}
self.logger.info(
f'Submitting initial batch of {bucket_size-1} jobs')
initial_uuids, initial_response = util.submit_jobs(
self.cook_url, jobspec, bucket_size - 1)
job_uuids.extend(initial_uuids)
self.assertEqual(201,
initial_response.status_code,
msg=initial_response.content)
def submit_jobs():
self.logger.info(
f'Submitting subsequent batch of {bucket_size-1} jobs')
subsequent_uuids, subsequent_response = util.submit_jobs(
self.cook_url, jobspec, bucket_size - 1)
job_uuids.extend(subsequent_uuids)
self.assertEqual(201,
subsequent_response.status_code,
msg=subsequent_response.content)
def is_rate_limit_triggered(_):
jobs1 = util.query_jobs(self.cook_url,
True,
uuid=job_uuids).json()
waiting_jobs = [
j for j in jobs1 if j['status'] == 'waiting'
]
running_jobs = [
j for j in jobs1 if j['status'] == 'running'
]
self.logger.debug(
f'There are {len(waiting_jobs)} waiting jobs')
# We submitted just under two buckets. We should only see a bucket + some extra running. No more.
return len(running_jobs) >= bucket_size and len(
running_jobs) < (bucket_size + token_rate /
2) and len(waiting_jobs) > 0
util.wait_until(submit_jobs, is_rate_limit_triggered)
jobs2 = util.query_jobs(self.cook_url, True,
uuid=job_uuids).json()
running_jobs = [j for j in jobs2 if j['status'] == 'running']
self.assertEqual(len(running_jobs), bucket_size)
finally:
util.kill_jobs(self.cook_url, job_uuids)
