def main():
# Input job id and njs_service URL
if len(sys.argv) == 3:
job_id = sys.argv[1]
njs_url = sys.argv[2]
else:
print("Incorrect usage")
sys.exit(1)
config = {}
config['workdir'] = os.environ.get("JOB_DIR", '/tmp/')
if not os.path.exists(config['workdir']):
os.makedirs(config['workdir'])
config['catalog-service-url'] = njs_url.replace('njs_wrapper', 'catalog')
auth_ext = 'auth/api/legacy/KBase/Sessions/Login'
config['auth-service-url'] = njs_url.replace('njs_wrapper', auth_ext)
if 'USE_SHIFTER' in os.environ:
config['runtime'] = 'shifter'
if 'JR_MAX_TASKS' in os.environ:
config['max_tasks'] = int(os.environ['JR_MAX_TASKS'])
token = _get_token()
at = _get_admin_token()
if not os.path.exists(config['workdir']):
os.makedirs(config['workdir'])
try:
jr = JobRunner(config, njs_url, job_id, token, at)
jr.run()
except Exception as e:
print("An unhandled error was encountered")
print(e)
sys.exit(2)
def test_run_volume(self, mock_njs, mock_auth):
self._cleanup(self.jobid)
params = deepcopy(NJS_JOB_PARAMS)
params[0]['method'] = 'mock_app.voltest'
params[0]['params'] = {'param1': 'value1'}
jr = JobRunner(self.config, self.njs_url, self.jobid, self.token,
self.admin_token)
rv = deepcopy(CATALOG_GET_MODULE_VERSION)
rv['docker_img_name'] = 'mock_app:latest'
vols = deepcopy(CATALOG_LIST_VOLUME_MOUNTS)
jr.cc.catalog.get_module_version = MagicMock(return_value=rv)
jr.cc.catalog.list_volume_mounts = MagicMock(return_value=vols)
jr.cc.catalog.get_secure_config_params = MagicMock(return_value=None)
jr.logger.njs.add_job_logs = MagicMock(return_value=rv)
jr.njs.get_job_params.return_value = params
jr.njs.check_job_canceled.return_value = {'finished': False}
jr.auth.get_user.return_value = "bogus"
jr._get_token_lifetime = MagicMock(return_value=self.future)
if not os.path.exists('/tmp/bogus'):
os.mkdir('/tmp/bogus')
with open('/tmp/bogus/input.fa', 'w') as f:
f.write('>contig-50_0 length_64486 read_count_327041\n')
f.write('GTCGTGCTGCTGCCGATCGACCGCGCCTATGCGATGTTGCCGGACGGCATCC\n')
out = jr.run()
self.assertIn('result', out)
self.assertNotIn('error', out)
def test_canceled_job(self, mock_njs, mock_auth):
self._cleanup(self.jobid)
mlog = MockLogger()
os.environ['KB_AUTH_TOKEN'] = 'bogus'
jr = JobRunner(self.config, self.njs_url, self.jobid, self.token,
self.admin_token)
jr.logger = mlog
jr.njs.check_job_canceled.return_value = {'finished': True}
with self.assertRaises(OSError):
jr.run()
self.assertEquals(mlog.errors[0], 'Job already run or canceled')
def test_token(self, mock_njs, mock_auth):
self._cleanup(self.jobid)
params = deepcopy(NJS_JOB_PARAMS)
os.environ['KB_AUTH_TOKEN'] = 'bogus'
jr = JobRunner(self.config, self.njs_url, self.jobid, self.token,
self.admin_token)
jr.njs.check_job_canceled.return_value = {'finished': False}
jr.njs.get_job_params.return_value = params
jr.auth.get_user.side_effect = OSError()
with self.assertRaises(Exception):
jr.run()
def test_special(self, mock_njs, mock_auth):
jr = JobRunner(self.config, self.njs_url, self.jobid, self.token,
self.admin_token)
params = deepcopy(NJS_JOB_PARAMS)
submitscript = os.path.join(self.workdir, 'workdir/tmp', 'submit.sl')
with open(submitscript, 'w') as f:
f.write('#!/bin/sh')
f.write("echo hello")
params[0]['method'] = 'special.slurm'
params[0]['params'] = [{'submit_script': 'submit.sl'}]
jr._submit_special(self.config, '1234', params[0])
def test_run_online(self, mock_njs):
self._cleanup(self.jobid)
params = deepcopy(NJS_JOB_PARAMS)
jr = JobRunner(self.config, self.njs_url, self.jobid, self.token,
self.admin_token)
rv = deepcopy(CATALOG_GET_MODULE_VERSION)
rv['docker_img_name'] = 'mock_app:latest'
jr.logger.njs.add_job_logs = MagicMock(return_value=rv)
jr.njs.check_job_canceled.return_value = {'finished': False}
jr.njs.get_job_params.return_value = params
out = jr.run()
self.assertIn('result', out)
self.assertNotIn('error', out)
def test_error_update(self, mock_njs, mock_auth):
self._cleanup(self.jobid)
mlog = MockLogger()
os.environ['KB_AUTH_TOKEN'] = 'bogus'
jr = JobRunner(self.config, self.njs_url, self.jobid, self.token,
self.admin_token)
jr.logger = mlog
jr.njs.check_job_canceled.return_value = {'finished': False}
jr.njs.update_job.side_effect = ConnectionError()
jr.njs.get_job_params.side_effect = ConnectionError()
with self.assertRaises(ConnectionError):
jr.run()
emsg = 'Failed to get job parameters. Exiting.'
self.assertEquals(mlog.errors[0], emsg)
def test_run(self, mock_njs, mock_auth):
self._cleanup(self.jobid)
params = deepcopy(NJS_JOB_PARAMS)
params[0]['method'] = 'mock_app.bogus'
params[0]['params'] = {'param1': 'value1'}
jr = JobRunner(self.config, self.njs_url, self.jobid, self.token,
self.admin_token)
rv = deepcopy(CATALOG_GET_MODULE_VERSION)
rv['docker_img_name'] = 'mock_app:latest'
jr.cc.catalog.get_module_version = MagicMock(return_value=rv)
jr.cc.catalog.list_volume_mounts = MagicMock(return_value=[])
jr.cc.catalog.get_secure_config_params = MagicMock(return_value=None)
jr.logger.njs.add_job_logs = MagicMock(return_value=rv)
jr.njs.get_job_params.return_value = params
jr.njs.check_job_canceled.return_value = {'finished': False}
jr.auth.get_user.return_value = "bogus"
jr._get_token_lifetime = MagicMock(return_value=self.future)
out = jr.run()
self.assertIn('result', out)
self.assertNotIn('error', out)
def test_max_jobs(self, mock_njs, mock_auth):
self._cleanup(self.jobid)
params = deepcopy(NJS_JOB_PARAMS)
params[0]['method'] = 'RunTester.run_RunTester'
params[0]['params'] = [{'depth': 2, 'size': 1000, 'parallel': 5}]
config = deepcopy(self.config)
config['max_tasks'] = 2
jr = JobRunner(config, self.njs_url, self.jobid, self.token,
self.admin_token)
rv = deepcopy(CATALOG_GET_MODULE_VERSION)
rv['docker_img_name'] = 'test/runtester:latest'
jr.cc.catalog.get_module_version = MagicMock(return_value=rv)
jr.cc.catalog.list_volume_mounts = MagicMock(return_value=[])
jr.cc.catalog.get_secure_config_params = MagicMock(return_value=None)
jr.logger.njs.add_job_logs = MagicMock(return_value=rv)
jr.njs.get_job_params.return_value = params
jr.njs.check_job_canceled.return_value = {'finished': False}
jr.auth.get_user.return_value = "bogus"
out = jr.run()
self.assertIn('error', out)
def test_run_sub(self, mock_njs, mock_auth):
self._cleanup(self.jobid)
params = deepcopy(NJS_JOB_PARAMS)
params[0]['method'] = 'RunTester.run_RunTester'
params[0]['params'] = [{'depth': 3, 'size': 1000, 'parallel': 5}]
params[1]['auth-service-url'] = self.config['auth-service-url']
params[1]['auth.service.url.v2'] = self.config['auth2-url']
jr = JobRunner(self.config, self.njs_url, self.jobid, self.token,
self.admin_token)
rv = deepcopy(CATALOG_GET_MODULE_VERSION)
rv['docker_img_name'] = 'test/runtester:latest'
jr.cc.catalog.get_module_version = MagicMock(return_value=rv)
jr.cc.catadmin.list_volume_mounts = MagicMock(return_value=[])
jr.logger.njs.add_job_logs = MagicMock(return_value=rv)
jr.njs.get_job_params.return_value = params
jr.njs.check_job_canceled.return_value = {'finished': False}
jr.auth.get_user.return_value = "bogus"
out = jr.run()
self.assertIn('result', out)
self.assertNotIn('error', out)
def test_cancel(self, mock_njs, mock_auth):
self._cleanup(self.jobid)
params = deepcopy(NJS_JOB_PARAMS)
params[0]['method'] = 'RunTester.run_RunTester'
params[0]['params'] = [{'depth': 3, 'size': 1000, 'parallel': 4}]
jr = JobRunner(self.config, self.njs_url, self.jobid, self.token,
self.admin_token)
rv = deepcopy(CATALOG_GET_MODULE_VERSION)
rv['docker_img_name'] = 'test/runtester:latest'
jr.cc.catalog.get_module_version = MagicMock(return_value=rv)
jr.cc.catalog.list_volume_mounts = MagicMock(return_value=[])
jr.cc.catalog.get_secure_config_params = MagicMock(return_value=None)
jr.logger.njs.add_job_logs = MagicMock(return_value=rv)
jr._get_token_lifetime = MagicMock(return_value=self.future)
jr.njs.get_job_params.return_value = params
nf = {'finished': False}
jr.njs.check_job_canceled.side_effect = [nf, nf, nf, nf, nf,
{'finished': True}]
jr.auth.get_user.return_value = "bogus"
out = jr.run()
self.assertIsNotNone(out)
def test_token_lifetime(self, mock_req, mock_auth, mock_njs):
# Test get token lifetime
config = NJS_JOB_PARAMS[1]
resp = AUTH_V2_TOKEN
mock_req.get.return_value = MockAuth(resp)
jr = JobRunner(self.config, self.njs_url, self.jobid, self.token,
self.admin_token)
mlog = MockLogger()
jr.logger = mlog
exp = jr._get_token_lifetime(config)
self.assertGreater(exp, 0)
mock_req.get.side_effect = OSError("bad request")
with self.assertRaises(OSError):
jr._get_token_lifetime(config)
