def test_retries(self):
# Default retries = 2
cp, uuids = cli.submit('-- ls -al', 'localhost:99999', '--verbose')
stderr = cli.decode(cp.stderr)
self.assertEqual(1, cp.returncode, stderr)
self.assertIn('Starting new HTTP connection (1)', stderr)
self.assertIn('Starting new HTTP connection (2)', stderr)
self.assertIn('Starting new HTTP connection (3)', stderr)
self.assertNotIn('Starting new HTTP connection (4)', stderr)
# Set retries = 0
config = {'http': {'retries': 0}}
with cli.temp_config_file(config) as path:
cp, uuids = cli.submit('-- ls -al', 'localhost:99999',
'--verbose --config %s' % path)
stderr = cli.decode(cp.stderr)
self.assertEqual(1, cp.returncode, stderr)
self.assertIn('Starting new HTTP connection (1)', stderr)
self.assertNotIn('Starting new HTTP connection (2)', stderr)
# Set retries = 4
config = {'http': {'retries': 4}}
with cli.temp_config_file(config) as path:
cp, uuids = cli.submit('-- ls -al', 'localhost:99999',
'--verbose --config %s' % path)
stderr = cli.decode(cp.stderr)
self.assertEqual(1, cp.returncode, stderr)
self.assertIn('Starting new HTTP connection (1)', stderr)
self.assertIn('Starting new HTTP connection (2)', stderr)
self.assertIn('Starting new HTTP connection (3)', stderr)
self.assertIn('Starting new HTTP connection (4)', stderr)
self.assertIn('Starting new HTTP connection (5)', stderr)
self.assertNotIn('Starting new HTTP connection (6)', stderr)
def test_entity_ref_support(self):
# Submit to cluster #1
cp, uuids = cli.submit('job1', self.cook_url_1)
self.assertEqual(0, cp.returncode, cp.stderr)
uuid = uuids[0]
# Submit to cluster #2 with the same uuid
cp, uuids = cli.submit('job2',
self.cook_url_2,
submit_flags=f'--uuid {uuid}')
self.assertEqual(0, cp.returncode, cp.stderr)
config = self.__two_cluster_config()
with cli.temp_config_file(config) as path:
flags = f'--config {path}'
# Query for both jobs with uuid
cp, jobs = cli.show_jobs([uuid], flags=flags)
self.assertEqual(0, cp.returncode, cp.stderr)
self.assertEqual(2, len(jobs), jobs)
self.assertEqual(uuid, jobs[0]['uuid'])
self.assertEqual(uuid, jobs[1]['uuid'])
# Query cook1 with entity ref
cp, jobs = cli.show_jobs([f'{self.cook_url_1}/jobs/{uuid}'],
flags=flags)
self.assertEqual(0, cp.returncode, cp.stderr)
self.assertEqual(1, len(jobs), jobs)
self.assertEqual(uuid, jobs[0]['uuid'])
self.assertEqual('job1', jobs[0]['command'])
# Query cook2 with entity ref
cp, jobs = cli.show_jobs([f'{self.cook_url_2}/jobs/{uuid}'],
flags=flags)
self.assertEqual(0, cp.returncode, cp.stderr)
self.assertEqual(1, len(jobs), jobs)
self.assertEqual(uuid, jobs[0]['uuid'])
self.assertEqual('job2', jobs[0]['command'])
def test_no_matching_data_error_shows_only_cluster_of_interest(self):
name = uuid.uuid4()
config = {
'clusters': [{
'name': 'FOO',
'url': f'{self.cook_url_1}'
}, {
'name': 'BAR',
'url': f'{self.cook_url_2}'
}]
}
with cli.temp_config_file(config) as path:
flags = f'--config {path}'
cp, uuids = cli.submit('ls',
flags=flags,
submit_flags=f'--name {name}')
self.assertEqual(0, cp.returncode, cp.stderr)
user = util.get_user(self.cook_url_1, uuids[0])
jobs_flags = f'--user {user} --name {name} --all'
cp, jobs = cli.jobs_json(self.cook_url_1, jobs_flags)
self.assertEqual(0, cp.returncode, cp.stderr)
self.assertEqual(1, len(jobs))
cs = f'{cli.command()} {flags}'
netloc_1 = urlparse(self.cook_url_1).netloc
netloc_2 = urlparse(self.cook_url_2).netloc
command = f'{cs} jobs {jobs_flags} -1 | sed "s/{netloc_1}/{netloc_2}/" | {cs} show'
self.logger.info(command)
cp = subprocess.run(command, shell=True, stdout=subprocess.PIPE)
self.assertEqual(1, cp.returncode, cp.stderr)
self.assertIn(
'No matching data found in BAR.\nDo you need to add another cluster',
cli.stdout(cp))
def test_federated_query(self):
# Submit to cluster #1
cp, uuids = cli.submit('ls', self.cook_url_1)
self.assertEqual(0, cp.returncode, cp.stderr)
uuid_1 = uuids[0]
# Submit to cluster #2
cp, uuids = cli.submit('ls', self.cook_url_2)
self.assertEqual(0, cp.returncode, cp.stderr)
uuid_2 = uuids[0]
# Single query for both jobs, federated across clusters
config = {'clusters': [{'name': 'cook1', 'url': self.cook_url_1},
{'name': 'cook2', 'url': self.cook_url_2}]}
with cli.temp_config_file(config) as path:
cp = cli.wait([uuid_1, uuid_2], flags='--config %s' % path)
self.assertEqual(0, cp.returncode, cp.stderr)
cp, jobs = cli.show_json([uuid_1, uuid_2], flags='--config %s' % path)
uuids = [job['uuid'] for job in jobs]
self.assertEqual(0, cp.returncode, cp.stderr)
self.assertEqual(2, len(jobs), jobs)
self.assertIn(str(uuid_1), uuids)
self.assertIn(str(uuid_2), uuids)
self.assertEqual('completed', jobs[0]['status'])
self.assertEqual('completed', jobs[1]['status'])
def test_config_defaults_are_respected(self):
# Submit job with defaults in config file
config = {
'defaults': {
'submit': {
'mem': 256,
'cpus': 2,
'priority': 16,
'max-retries': 2,
'max-runtime': 300
}
}
}
with cli.temp_config_file(config) as path:
cp, uuids = cli.submit('ls', self.cook_url, '--config %s' % path)
self.assertEqual(0, cp.returncode, cp.stderr)
# Assert that the job was submitted with the defaults from the file
cp, jobs = cli.show_json(uuids, self.cook_url)
self.assertEqual(0, cp.returncode, cp.stderr)
self.assertEqual(1, len(jobs))
job = jobs[0]
defaults = config['defaults']['submit']
self.assertEqual(defaults['mem'], job['mem'])
self.assertEqual(defaults['cpus'], job['cpus'])
self.assertEqual(defaults['priority'], job['priority'])
self.assertEqual(defaults['max-retries'], job['max_retries'])
self.assertEqual(defaults['max-runtime'], job['max_runtime'])
def test_no_cluster(self):
config = {'clusters': []}
with cli.temp_config_file(config) as path:
flags = '--config %s' % path
cp, uuids = cli.submit('ls', flags=flags)
self.assertEqual(1, cp.returncode, cp.stderr)
self.assertIn('must specify at least one cluster',
cli.decode(cp.stderr))
def test_ssh(self):
# Submit to cluster #2
cp, uuids = cli.submit('ls', self.cook_url_2)
self.assertEqual(0, cp.returncode, cp.stderr)
instance = util.wait_for_instance(self.cook_url_2, uuids[0])
# Run ssh for the submitted job, with both clusters configured
config = self.__two_cluster_config()
with cli.temp_config_file(config) as path:
hostname = instance['hostname']
env = os.environ
env['CS_SSH'] = 'echo'
cp = cli.ssh(uuids[0], env=env, flags=f'--config {path}')
stdout = cli.stdout(cp)
self.assertEqual(0, cp.returncode, cli.output(cp))
self.assertIn(f'Attempting ssh for job instance {instance["task_id"]}', stdout)
self.assertIn('Executing ssh', stdout)
self.assertIn(hostname, stdout)
self.assertIn(f'-t {hostname} cd', stdout)
self.assertIn('; bash', stdout)
def test_federated_show(self):
# Submit to cluster #1
cp, uuids = cli.submit('ls', self.cook_url_1)
self.assertEqual(0, cp.returncode, cp.stderr)
uuid_1 = uuids[0]
# Submit to cluster #2
cp, uuids = cli.submit('ls', self.cook_url_2)
self.assertEqual(0, cp.returncode, cp.stderr)
uuid_2 = uuids[0]
# Single query for both jobs, federated across clusters
config = self.__two_cluster_config()
with cli.temp_config_file(config) as path:
cp, jobs = cli.show_jobs([uuid_1, uuid_2], flags='--config %s' % path)
uuids = [job['uuid'] for job in jobs]
self.assertEqual(0, cp.returncode, cp.stderr)
self.assertEqual(2, len(jobs), jobs)
self.assertIn(str(uuid_1), uuids)
self.assertIn(str(uuid_2), uuids)
def test_specifying_cluster_name_explicitly(self):
config = {
'clusters': [{
'name': 'foo',
'url': self.cook_url
}],
'defaults': {
'submit': {
'mem': 256,
'cpus': 2,
'max-retries': 2
}
}
}
with cli.temp_config_file(config) as path:
flags = '--config %s --cluster foo' % path
cp, uuids = cli.submit('ls', flags=flags)
self.assertEqual(0, cp.returncode, cp.stderr)
cp, jobs = cli.show_json(uuids, flags=flags)
self.assertEqual(0, cp.returncode, cp.stderr)
cp = cli.wait(uuids, flags=flags)
self.assertEqual(0, cp.returncode, cp.stderr)
def test_multicluster_usage(self):
pools_1, _ = util.all_pools(self.cook_url_1)
pools_2, _ = util.all_pools(self.cook_url_2)
pools = pools_1 + pools_2
user = '******'
half_of_the_pools = [pool['name'] for pool in pools[:len(pools) // 2]]
expected_pools_1 = set(half_of_the_pools) & set(
[pool['name'] for pool in pools_1])
expected_pools_2 = set(half_of_the_pools) & set(
[pool['name'] for pool in pools_2])
config = self.__two_cluster_config()
with cli.temp_config_file(config) as path:
# filter half
cp, usage = cli.usage(user,
None,
' '.join(f'--pool {pool}'
for pool in half_of_the_pools),
flags='--config %s' % path)
self.assertEqual(0, cp.returncode, cp.stderr)
if expected_pools_1:
self.assertEqual(
set(usage['clusters']['cook1']['pools'].keys()),
expected_pools_1)
if expected_pools_2:
self.assertEqual(
set(usage['clusters']['cook2']['pools'].keys()),
expected_pools_2)
self.assertEqual('', cli.decode(cp.stderr))
# filter half with one bad pool
cp, usage = cli.usage(user,
None,
'-p zzzz ' +
' '.join(f'--pool {pool}'
for pool in half_of_the_pools),
flags='--config %s' % path)
self.assertEqual(0, cp.returncode, cp.stderr)
if expected_pools_1:
self.assertEqual(
set(usage['clusters']['cook1']['pools'].keys()),
expected_pools_1)
if expected_pools_2:
self.assertEqual(
set(usage['clusters']['cook2']['pools'].keys()),
expected_pools_2)
self.assertIn("zzzz is not a valid pool in ",
cli.decode(cp.stderr))
# filter half with two bad pools
cp, usage = cli.usage(user,
None,
'-p zzzz -p zzzzx ' +
' '.join(f'--pool {pool}'
for pool in half_of_the_pools),
flags='--config %s' % path)
self.assertEqual(0, cp.returncode, cp.stderr)
if expected_pools_1:
self.assertEqual(
set(usage['clusters']['cook1']['pools'].keys()),
expected_pools_1)
if expected_pools_2:
self.assertEqual(
set(usage['clusters']['cook2']['pools'].keys()),
expected_pools_2)
self.assertIn(" are not valid pools in ", cli.decode(cp.stderr))
