def test_slurmenv_api_resources():
if not in_slurm_allocation() or get_num_slurm_nodes() < 2:
pytest.skip('test not run in slurm allocation or allocation is smaller than 2 nodes')
resources, allocation = get_slurm_resources_binded()
set_pythonpath_to_qcg_module()
tmpdir = str(tempfile.mkdtemp(dir=SHARED_PATH))
try:
m = LocalManager(['--log', 'debug', '--wd', tmpdir, '--report-format', 'json'], {'wdir': str(tmpdir)})
api_res = m.resources()
assert all(('total_nodes' in api_res, 'total_cores' in api_res))
assert all((api_res['total_nodes'] == resources.total_nodes, api_res['total_cores'] == resources.total_cores))
aux_dir = find_single_aux_dir(str(tmpdir))
assert all((exists(join(tmpdir, '.qcgpjm-client', 'api.log')),
exists(join(aux_dir, 'service.log'))))
finally:
if m:
m.finish()
# stopManager is using 'terminate' method on service process, which is not a best option when using
# pytest and gathering code coverage
# m.stopManager()
m.cleanup()
rmtree(tmpdir)
def test_local_manager_resources(tmpdir):
cores = 4
# switch on debugging (by default in api.log file)
m = LocalManager(['--wd', str(tmpdir), '--nodes',
str(cores)], {'wdir': str(tmpdir)})
res = m.resources()
assert all(('total_nodes' in res, 'total_cores'
in res, res['total_nodes'] == 1, res['total_cores'] == cores))
m.finish()
# m.stopManager()
m.cleanup()
def test_local_manager_submit_simple(tmpdir):
cores = 4
# switch on debugging (by default in api.log file)
m = LocalManager(['--wd', str(tmpdir), '--nodes',
str(cores)], {'wdir': str(tmpdir)})
try:
res = m.resources()
assert all(
('total_nodes' in res, 'total_cores'
in res, res['total_nodes'] == 1, res['total_cores'] == cores))
ids = m.submit(Jobs().add(name='host',
exec='/bin/hostname',
args=['--fqdn'],
stdout='host.stdout').add(
name='date',
exec='/bin/date',
stdout='date.stdout',
numCores={'exact': 2}))
assert len(m.list()) == 2
m.wait4(ids)
jinfos = m.info(ids)
assert all(
('jobs' in jinfos, len(jinfos['jobs'].keys()) == 2, 'host'
in jinfos['jobs'], 'date' in jinfos['jobs'],
jinfos['jobs']['host'].get('data', {}).get('status',
'') == 'SUCCEED',
jinfos['jobs']['date'].get('data', {}).get('status',
'') == 'SUCCEED'))
aux_dir = find_single_aux_dir(str(tmpdir))
assert all(
(exists(tmpdir.join('.qcgpjm-client', 'api.log')),
exists(join(aux_dir,
'service.log')), exists(tmpdir.join('host.stdout')),
exists(tmpdir.join('date.stdout'))))
finally:
m.finish()
# m.stopManager()
m.cleanup()
def test_local_manager_resources_nodes(tmpdir):
nodes = 2
cores_per_node = 3
res_desc = ','.join([str(cores_per_node) for i in range(nodes)])
# switch on debugging (by default in api.log file)
m = LocalManager(['--wd', str(tmpdir), '--nodes', res_desc],
{'wdir': str(tmpdir)})
res = m.resources()
assert all(
('total_nodes' in res, 'total_cores' in res, res['total_nodes'] == 2,
res['total_cores'] == cores_per_node * nodes))
m.finish()
# m.stopManager()
m.cleanup()
else:
PYTHON_CMD = "python3"
"""
######################################################################
config PilotJob
######################################################################
"""
from qcg.pilotjob.api.manager import Manager
from qcg.pilotjob.api.manager import LocalManager
from qcg.pilotjob.api.job import Jobs
# m = LocalManager(cfg={'log_level': 'DEBUG'}, server_args=['--log', 'debug'])
m = LocalManager()
# get available resources
print("\n\navailable resources:\n%s\n" % str(m.resources()))
# submit jobs and save their names in 'ids' list
jobs = Jobs()
print("Start Adding jobs . . .\n\n")
INSTANCE_INDEX = 0
for i in range(NUM_INSTANCES):
for SUBMODEL in ['macro', 'micro']:
cmd = '%s run_mscale.py --submodel %s --data_dir=%s --instance_index %d --coupling_type %s --num_instances %d --weather_coupling %s' % (
PYTHON_CMD, SUBMODEL, DATA_DIR, INSTANCE_INDEX, COUPLING_TYPE,
NUM_INSTANCES, WEATHER_COUPLING)
print("\tAdd job with cmd = %s" % (cmd))