def test_update_job_status(self): """ Tests that a job can update its status """ job_type = 'SIMILARITY' params = { 'search_type': 'SIMILARITY', 'structure': '[H]C1(CCCN1C(=N)N)CC1=NC(=NO1)C1C=CC(=CC=1)NC1=NC(=CS1)C1C=CC(Br)=CC=1', 'threshold': '70' } docker_image_url = 'some url' with self.flask_app.app_context(): job_must_be = delayed_job_models.get_or_create( job_type, params, docker_image_url) job_id = job_must_be.id new_data = { 'progress': 50, 'status_log': 'Loading database', 'status_description': '{"msg":"Smiles file loaded"}' } token = token_generator.generate_job_token(job_id) headers = {'X-Job-Key': token} client = self.client response = client.patch(f'/status/{job_id}', data=new_data, headers=headers) self.assertEqual(response.status_code, 200, msg='The request should have not failed') job_got = delayed_job_models.get_job_by_id(job_id) # be sure to have a fresh version of the object DB.session.rollback() DB.session.expire(job_got) DB.session.refresh(job_got) progress_got = job_got.progress self.assertEqual(progress_got, new_data['progress'], msg=f'The progress was not updated correctly!') status_log_got = job_got.status_log self.assertIsNotNone(status_log_got, msg=f'The status log was not set correctly!') self.assertNotEqual( status_log_got, new_data['status_log'], msg=f'It seems that the status log was not saved' f'correctly. It should be accumulative') status_description_got = job_got.status_description self.assertEqual( status_description_got, new_data['status_description'], msg=f'The status description was not updated correctly!')
def test_job_token_is_generated(self): """ Test that the token for a job is generated """ with self.flask_app.app_context(): job_type = 'SIMILARITY' params = { 'search_type': 'SIMILARITY', 'structure': '[H]C1(CCCN1C(=N)N)CC1=NC(=NO1)C1C=CC(=CC=1)NC1=NC(=CS1)C1C=CC(Br)=CC=1', 'threshold': '70' } docker_image_url = 'some_url' job_must_be = delayed_job_models.get_or_create( job_type, params, docker_image_url) token_got = token_generator.generate_job_token(job_must_be.id) key = RUN_CONFIG.get('server_secret_key') data_got = jwt.decode(token_got, key, algorithms=['HS256']) self.assertEqual(data_got.get('job_id'), job_must_be.id, msg='The token was not generated correctly!')
def create_params_file(job, input_files_desc): """ Creates the parameters file for the job :param job: job oject for which the parmeters file will be created """ job_token = token_generator.generate_job_token(job.id) run_params = { 'job_id': job.id, 'job_token': job_token, 'inputs': prepare_job_inputs(job, input_files_desc), 'output_dir': get_job_output_dir_path(job), 'custom_job_config': get_custom_job_config_repo_params(job), 'status_update_endpoint': { 'url': f'http://{RUN_CONFIG.get("status_update_host")}' f'{RUN_CONFIG.get("base_path", "")}/status/{job.id}', 'method': 'PATCH' }, 'custom_statistics_endpoint': { 'url': f'http://{RUN_CONFIG.get("status_update_host")}' f'{RUN_CONFIG.get("base_path", "")}' f'/custom_statistics/submit_statistics/{job.type.lower()}_job/{job.id}', 'method': 'POST' }, 'job_params': json.loads(job.raw_params), } run_params_path = get_job_run_params_file_path(job) # delete file if existed before, just in case if os.path.exists(run_params_path): os.remove(run_params_path) with open(run_params_path, 'w') as out_file: out_file.write(yaml.dump(run_params))
def test_a_job_cannot_update_another_jobs_statistics(self): """ Tests that a job cannot update the statistics of another job """ job_type = 'TEST' params = { 'instruction': 'RUN_NORMALLY', 'seconds': 1, 'api_url': 'https://www.ebi.ac.uk/chembl/api/data/similarity/CN1C(=O)C=C(c2cccc(Cl)c2)c3cc(ccc13)C@@(c4ccc(Cl)cc4)c5cncn5C/80.json' } docker_image_url = 'some url' with self.flask_app.app_context(): job_must_be = delayed_job_models.get_or_create( job_type, params, docker_image_url) job_id = job_must_be.id statistics = { 'duration': 1, } token = token_generator.generate_job_token('another_id') headers = {'X-JOB-KEY': token} client = self.client response = client.post( f'/custom_statistics/submit_statistics/test_job/{job_id}', data=statistics, headers=headers) self.assertEqual( response.status_code, 401, msg= 'I should not be authorised to upload statistics of another job' )
def test_a_job_cannot_update_another_job_progress(self): """ Tests that a job can not use its token to update another job's status """ job_type = 'SIMILARITY' params = { 'search_type': 'SIMILARITY', 'structure': '[H]C1(CCCN1C(=N)N)CC1=NC(=NO1)C1C=CC(=CC=1)NC1=NC(=CS1)C1C=CC(Br)=CC=1', 'threshold': '70' } docker_image_url = 'some url' with self.flask_app.app_context(): job_must_be = delayed_job_models.get_or_create( job_type, params, docker_image_url) job_id = job_must_be.id new_data = { 'progress': 50, 'status_log': 'Loading database', } token = token_generator.generate_job_token('another_id') headers = {'X-JOB-KEY': token} client = self.client response = client.patch(f'/status/{job_id}', data=new_data, headers=headers) self.assertEqual( response.status_code, 401, msg= 'I should not be authorised to modify the status of another job' )
def test_job_can_be_submitted(self): """ Test that a job can be submitted """ with self.flask_app.app_context(): job_type = 'TEST' docker_image_url = 'some_url' input_files_desc, input_files_hashes, params = self.prepare_mock_job_args( ) print('TEST INPUT ') print(input_files_desc) submission_result = job_submission_service.submit_job( job_type, input_files_desc, input_files_hashes, docker_image_url, params) job_id = submission_result.get('job_id') job_data = delayed_job_models.get_job_by_id(job_id).public_dict() # ----------------------------------------------- # Test Run Dir # ----------------------------------------------- job_run_dir_must_be = os.path.join( job_submission_service.JOBS_RUN_DIR, job_id) self.assertTrue( os.path.isdir(job_run_dir_must_be), msg= f'The run dir for the job ({job_run_dir_must_be}) has not been created!' ) input_files_dir_must_be = os.path.join( job_run_dir_must_be, job_submission_service.INPUT_FILES_DIR_NAME) self.assertTrue( os.path.isdir(input_files_dir_must_be), msg= f'The input files dir for the job ({input_files_dir_must_be}) has not been created!' ) # ----------------------------------------------- # Test Run Params # ----------------------------------------------- params_file_must_be = os.path.join( job_run_dir_must_be, job_submission_service.RUN_PARAMS_FILENAME) self.assertTrue( os.path.isfile(params_file_must_be), msg= f'The run params file for the job ({params_file_must_be}) has not been created!' ) params_file = open(params_file_must_be, 'r') params_got = yaml.load(params_file, Loader=yaml.FullLoader) params_file.close() token_must_be = token_generator.generate_job_token(job_id) token_got = params_got.get('job_token') self.assertEqual(token_must_be, token_got, msg='The token was not generated correctly') job_id_must_be = job_id job_id_got = params_got.get('job_id') self.assertEqual(job_id_must_be, job_id_got, msg='The job id was not generated correctly') status_update_url_must_be = f'http://0.0.0.0:5000/status/{job_id}' status_update_url_got = params_got.get( 'status_update_endpoint').get('url') self.assertEqual( status_update_url_must_be, status_update_url_got, msg='The status update url was not set correctly!') status_update_method_must_be = 'PATCH' status_update_method_got = params_got.get( 'status_update_endpoint').get('method') self.assertEqual( status_update_method_must_be, status_update_method_got, msg='The status update method was not set correctly!') job_params_got = params_got.get('job_params') raw_job_params_must_be = job_data.get('raw_params') self.assertEqual(json.dumps(job_params_got, sort_keys=True), raw_job_params_must_be, msg='The job params were not set correctly') custom_statistics_url_must_be = f'http://0.0.0.0:5000' \ f'/custom_statistics/submit_statistics/{job_type.lower()}_job/{job_id}' custom_statistics_url_got = params_got.get( 'custom_statistics_endpoint', {}).get('url') self.assertEqual( custom_statistics_url_must_be, custom_statistics_url_got, msg='The status update method was not set correctly!') custom_statistics_method_must_be = 'POST' custom_statistics_method_got = params_got.get( 'custom_statistics_endpoint', {}).get('method') self.assertEqual( custom_statistics_method_must_be, custom_statistics_method_got, msg='The custom statistics method was not set correctly!') # ----------------------------------------------- # Test Input Files # ----------------------------------------------- job_input_files_desc_got = params_got.get('inputs') for key, tmp_path in input_files_desc.items(): run_path_must_be = job_input_files_desc_got[key] self.assertTrue( os.path.isfile(run_path_must_be), msg= f'The input file for the job ({run_path_must_be}) has not been created!' ) job_got = delayed_job_models.get_job_by_id(job_id) input_files_got = job_got.input_files num_inputs_files_must_be = len(os.listdir(input_files_dir_must_be)) self.assertEquals( num_inputs_files_must_be, len(input_files_got), msg='The input files were not registered correctly!') for input_file in input_files_got: internal_path_got = input_file.internal_path self.assertTrue( os.path.isfile(internal_path_got), msg= f'The internal path of an input file {internal_path_got} ' f'seems that does not exist!') # ----------------------------------------------- # Test Output Directory # ----------------------------------------------- output_dir_must_be = Path( job_submission_service.JOBS_OUTPUT_DIR).joinpath(job_id) output_dir_got = params_got.get('output_dir') self.assertEqual(str(output_dir_got), str(output_dir_must_be), msg='The job output dir was not set correctly') self.assertTrue( os.path.isdir(output_dir_must_be), msg= f'The output dir for the job ({output_dir_must_be}) has not been created!' ) # ----------------------------------------------- # Submission script file # ----------------------------------------------- submission_script_file_must_be = \ os.path.join(job_run_dir_must_be, job_submission_service.SUBMISSION_FILE_NAME) self.assertTrue( os.path.isfile(submission_script_file_must_be), msg= f'The script file for submitting the job ({submission_script_file_must_be}) ' f'has not been created!') self.assertTrue( os.access(submission_script_file_must_be, os.X_OK), msg= f'The script file for the job ({submission_script_file_must_be}) is not executable!' )