def test_successful_with_full_recipe(self): """Tests calling QueueManager.handle_job_completion() successfully with all jobs in a recipe.""" # Queue the recipe recipe_id = Queue.objects.queue_new_recipe(self.recipe_type, self.data, self.event) # Fake out completing Job 1 job_1 = RecipeJob.objects.select_related('job').get( recipe_id=recipe_id, job_name='Job 1').job job_exe_1 = JobExecution.objects.get(job_id=job_1.id) output_file_1 = product_test_utils.create_product( job_exe=job_exe_1, workspace=self.workspace) output_file_2 = product_test_utils.create_product( job_exe=job_exe_1, workspace=self.workspace) results = JobResults() results.add_file_list_parameter('Test Output 1', [output_file_1.id, output_file_2.id]) JobExecution.objects.post_steps_results(job_exe_1.id, results, ResultsManifest()) Job.objects.filter(pk=job_1.id).update(status='RUNNING') JobExecution.objects.filter(pk=job_exe_1.id).update(status='RUNNING') Queue.objects.handle_job_completion(job_exe_1.id, now()) # Fake out completing Job 2 job_2 = RecipeJob.objects.select_related('job').get( recipe_id=recipe_id, job_name='Job 2').job job_exe_2 = JobExecution.objects.get(job_id=job_2.id) output_file_1 = product_test_utils.create_product( job_exe=job_exe_2, workspace=self.workspace) output_file_2 = product_test_utils.create_product( job_exe=job_exe_2, workspace=self.workspace) results = JobResults() results.add_file_list_parameter('Test Output 2', [output_file_1.id, output_file_2.id]) JobExecution.objects.post_steps_results(job_exe_2.id, results, ResultsManifest()) Job.objects.filter(pk=job_2.id).update(status='RUNNING') JobExecution.objects.filter(pk=job_exe_2.id).update(status='RUNNING') # Call method to test Queue.objects.handle_job_completion(job_exe_2.id, now()) # Make sure processor was called self.assertEqual(self.mock_processor.process_completed.call_count, 2) # Make sure final recipe attributes are updated recipe = Recipe.objects.get(pk=recipe_id) self.assertIsNotNone(recipe.completed)
def test_simple_validation_1_1(self): json_manifest = { "version": "1.1", "output_data": [{ "name": "output_file", "file": { "path": "/tmp/job_exe_231/outputs/output.csv", "geo_metadata": { "data_started": "2015-05-15T10:34:12Z", "data_ended": "2015-05-15T10:36:12Z", "geo_json": { "type": "Polygon", "coordinates": [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0], [1.0, 20.0], [1.0, 10.0]]] } } } }] } input_files = {"input_file": False} output_files = {"output_file": (False, True)} manifest = ResultsManifest(json_manifest) manifest.validate(output_files)
def test_convert_1_0_to_1_1(self): json_manifest = { "version": "1.0", "files": [{ "name": "output_file", "path": "/tmp/job_exe_231/outputs/output.csv" }, { "name": "output_files", "paths": [ "/tmp/job_exe_231/outputs/output.csv", "/tmp/job_exe_231/outputs/output2.csv" ] }], "parse_results": [{ "filename": "myfile.h5", "data_started": "2015-05-15T10:34:12Z", "data_ended": "2015-05-15T10:36:12Z", "data_types": ["H5", "VEG"] }], "errors": [] } new_format = { "version": "1.1", "output_data": [{ "name": "output_file", "file": { "path": "/tmp/job_exe_231/outputs/output.csv" } }, { "name": "output_files", "files": [{ "path": "/tmp/job_exe_231/outputs/output.csv" }, { "path": "/tmp/job_exe_231/outputs/output2.csv" }] }], "parse_results": [{ "filename": "myfile.h5", "data_started": "2015-05-15T10:34:12Z", "data_ended": "2015-05-15T10:36:12Z", "data_types": ["H5", "VEG"] }], "errors": [] } manifest = ResultsManifest(json_manifest) converted = manifest._convert_schema(json_manifest) self.assertEqual(converted, new_format)
def test_simple_validation(self): json_manifest = { "version": "1.0", "files": [{ "name": "foo", "path": "nfs:server//myfile.txt" }] } input_files = {"input_file": False} output_files = {"foo": (False, True)} manifest = ResultsManifest(json_manifest) manifest.validate(output_files)
def test_manifest_supports_file_with_paths(self): json_manifest = { "version": "1.0", "files": [ {"name":"foo", "paths":["nfs:server//myfile.txt"]} ] } try: #This should not throw an exception since it is valid ResultsManifest(json_manifest) except InvalidResultsManifest: self.fail(u'This simple json_manifest is valid')
def test_invalid_results_manifest(self): json_manifest = { "version": "1.0", "files": [ {"name":"foo", "path":"nfs:server//myfile.txt", "paths": ["nfs:server//why_do_i_have_path_and_paths"]} ] } try: ResultsManifest(json_manifest) self.fail(u'files in a results manifest should not have both path and paths') except InvalidResultsManifest: #This should throw an exception since it is invalid pass
def test_manifest_version_1_1(self): json_manifest = { "version": "1.1", "output_data": [{ "name": "output_file", "file": { "path": "/tmp/job_exe_231/outputs/output.csv", "geo_metadata": { "data_started": "2015-05-15T10:34:12Z", "data_ended": "2015-05-15T10:36:12Z", "geo_json": { "type": "Polygon", "coordinates": [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0], [1.0, 20.0], [1.0, 10.0]]] } } } }, { "name": "output_files", "files": [{ "path": "/tmp/job_exe_231/outputs/output.csv", "geo_metadata": { "data_started": "2015-05-15T10:34:12Z", "data_ended": "2015-05-15T10:36:12Z", "geo_json": { "type": "Polygon", "coordinates": [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0], [1.0, 20.0], [1.0, 10.0]]] } } }, { "path": "/tmp/job_exe_231/outputs/output2.csv" }] }], "parse_results": [{ "filename": "myfile.h5", "data_started": "2015-05-15T10:34:12Z", "data_ended": "2015-05-15T10:36:12Z", "data_types": ["H5", "VEG"] }] } manifest = ResultsManifest(json_manifest)
def test_missing_optional_is_ok(self): json_manifest = { "version": "1.0", "files": [{ "name": "foo", "path": "nfs:server//myfile.txt" }] } input_files = {"input_file": False} output_files = { "foo": (False, True), "bar": (False, False) #This is an optional file } manifest = ResultsManifest(json_manifest) try: manifest.validate(output_files) except ResultsManifestAndInterfaceDontMatch: self.fail(u'The missing an optional file')
def test_output_does_not_match(self): json_manifest = { "version": "1.0", "files": [{ "name": "foo", "path": "nfs:server//myfile.txt" }] } input_files = {"input_file": False} output_files = {"bar": (False, True)} manifest = ResultsManifest(json_manifest) try: manifest.validate(output_files) self.fail( u'The outputs do not match the manifest, there should be a failure' ) except ResultsManifestAndInterfaceDontMatch: pass
def test_missing_required_is_bad(self): json_manifest = { "version": "1.0", "files": [ {"name":"foo", "path":"nfs:server//myfile.txt"} ] } input_files = { "input_file": False } output_files = { "foo": (False, True), "bar": (False, True) #This is a missing required file } manifest = ResultsManifest(json_manifest) try: manifest.validate(output_files) self.fail(u'There is a missing required file. Validation should have failed') except MissingRequiredOutput: pass
from django.db.utils import DatabaseError, OperationalError from django.utils.timezone import now from django.test import TransactionTestCase from mock import patch from error.exceptions import ScaleDatabaseError, ScaleIOError, ScaleOperationalError from job.configuration.results.exceptions import InvalidResultsManifest, MissingRequiredOutput from job.configuration.results.job_results import JobResults from job.configuration.results.results_manifest.results_manifest import ResultsManifest from job.management.commands.scale_post_steps import Command as PostCommand from job.models import JobExecutionOutput from job.test import utils as job_utils from trigger.models import TriggerEvent JOB_RESULTS = JobResults() RESULTS_MANIFEST = ResultsManifest() RESULTS = (JOB_RESULTS, RESULTS_MANIFEST) class TestPostJobSteps(TransactionTestCase): def setUp(self): django.setup() cmd = 'command' cmd_args = 'args' interface = { 'version': '1.0', 'command': cmd, 'command_arguments':
def perform_post_steps(self, job_exe, job_data, stdoutAndStderr): """Stores the files and deletes any working directories :param job_exe: The job execution model with related job and job_type fields :type job_exe: :class:`job.models.JobExecution` :param job_data: The job data :type job_data: :class:`job.configuration.data.job_data.JobData` :param stdoutAndStderr: the standard out from the job execution :type stdoutAndStderr: str :return: A tuple of the job results and the results manifest generated by the job execution :rtype: (:class:`job.configuration.results.job_results.JobResults`, :class:`job.configuration.results.results_manifest.results_manifest.ResultsManifest`) """ manifest_data = {} path_to_manifest_file = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH, 'results_manifest.json') if os.path.exists(path_to_manifest_file): logger.info('Opening results manifest...') with open(path_to_manifest_file, 'r') as manifest_file: manifest_data = json.loads(manifest_file.read()) logger.info('Results manifest:') logger.info(manifest_data) else: logger.info('No results manifest found') results_manifest = ResultsManifest(manifest_data) stdout_files = self._get_artifacts_from_stdout(stdoutAndStderr) results_manifest.add_files(stdout_files) results_manifest.validate(self._output_file_manifest_dict) files_to_store = {} for manifest_file_entry in results_manifest.get_files(): param_name = manifest_file_entry['name'] media_type = None output_data_item = self._get_output_data_item_by_name(param_name) if output_data_item: media_type = output_data_item.get('media_type') msg = 'Output %s has invalid/missing file path "%s"' if 'file' in manifest_file_entry: file_entry = manifest_file_entry['file'] if not os.path.isfile(file_entry['path']): raise InvalidResultsManifest( msg % (param_name, file_entry['path'])) if 'geo_metadata' in file_entry: files_to_store[param_name] = (file_entry['path'], media_type, file_entry['geo_metadata']) else: files_to_store[param_name] = (file_entry['path'], media_type) elif 'files' in manifest_file_entry: file_tuples = [] for file_entry in manifest_file_entry['files']: if not os.path.isfile(file_entry['path']): raise InvalidResultsManifest( msg % (param_name, file_entry['path'])) if 'geo_metadata' in file_entry: file_tuples.append((file_entry['path'], media_type, file_entry['geo_metadata'])) else: file_tuples.append((file_entry['path'], media_type)) files_to_store[param_name] = file_tuples job_data_parse_results = {} # parse results formatted for job_data for parse_result in results_manifest.get_parse_results(): filename = parse_result['filename'] assert filename not in job_data_parse_results geo_metadata = parse_result.get('geo_metadata', {}) geo_json = geo_metadata.get('geo_json', None) data_started = geo_metadata.get('data_started', None) data_ended = geo_metadata.get('data_ended', None) data_types = parse_result.get('data_types', []) new_workspace_path = parse_result.get('new_workspace_path', None) if new_workspace_path: new_workspace_path = os.path.join(new_workspace_path, filename) job_data_parse_results[filename] = (geo_json, data_started, data_ended, data_types, new_workspace_path) job_data.save_parse_results(job_data_parse_results) return (job_data.store_output_data_files(files_to_store, job_exe), results_manifest)
def test_empty_results_manifest(self): json_manifest = {} #This should not throw an exception since it is valid ResultsManifest(json_manifest)