def test_simple_validation_1_1(self): json_manifest = { "version": "1.1", "output_data": [{ "name": "output_file", "file": { "path": "/tmp/job_exe_231/outputs/output.csv", "geo_metadata": { "data_started": "2015-05-15T10:34:12Z", "data_ended": "2015-05-15T10:36:12Z", "geo_json": { "type": "Polygon", "coordinates": [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0], [1.0, 20.0], [1.0, 10.0]]] } } } }] } input_files = {"input_file": False} output_files = {"output_file": (False, True)} manifest = ResultsManifest(json_manifest) manifest.validate(output_files)
def test_simple_validation_1_1(self): json_manifest = { "version": "1.1", "output_data": [ { "name" : "output_file", "file": { "path" : "/tmp/job_exe_231/outputs/output.csv", "geo_metadata": { "data_started": "2015-05-15T10:34:12Z", "data_ended" : "2015-05-15T10:36:12Z", "geo_json": { "type": "Polygon", "coordinates": [ [ [ 1.0, 10.0 ], [ 2.0, 10.0 ], [ 2.0, 20.0 ],[ 1.0, 20.0 ], [ 1.0, 10.0 ] ] ] } } } } ] } input_files = { "input_file": False } output_files = { "output_file": (False, True) } manifest = ResultsManifest(json_manifest) manifest.validate(output_files)
def test_successful_with_full_recipe(self): """Tests calling QueueManager.handle_job_completion() successfully with all jobs in a recipe.""" # Queue the recipe recipe_id = Queue.objects.queue_new_recipe(self.recipe_type, self.data, self.event) # Fake out completing Job 1 job_1 = RecipeJob.objects.select_related('job').get( recipe_id=recipe_id, job_name='Job 1').job job_exe_1 = JobExecution.objects.get(job_id=job_1.id) output_file_1 = product_test_utils.create_product( job_exe=job_exe_1, workspace=self.workspace) output_file_2 = product_test_utils.create_product( job_exe=job_exe_1, workspace=self.workspace) results = JobResults() results.add_file_list_parameter('Test Output 1', [output_file_1.id, output_file_2.id]) JobExecution.objects.post_steps_results(job_exe_1.id, results, ResultsManifest()) Job.objects.filter(pk=job_1.id).update(status='RUNNING') JobExecution.objects.filter(pk=job_exe_1.id).update(status='RUNNING') Queue.objects.handle_job_completion(job_exe_1.id, now()) # Fake out completing Job 2 job_2 = RecipeJob.objects.select_related('job').get( recipe_id=recipe_id, job_name='Job 2').job job_exe_2 = JobExecution.objects.get(job_id=job_2.id) output_file_1 = product_test_utils.create_product( job_exe=job_exe_2, workspace=self.workspace) output_file_2 = product_test_utils.create_product( job_exe=job_exe_2, workspace=self.workspace) results = JobResults() results.add_file_list_parameter('Test Output 2', [output_file_1.id, output_file_2.id]) JobExecution.objects.post_steps_results(job_exe_2.id, results, ResultsManifest()) Job.objects.filter(pk=job_2.id).update(status='RUNNING') JobExecution.objects.filter(pk=job_exe_2.id).update(status='RUNNING') # Call method to test Queue.objects.handle_job_completion(job_exe_2.id, now()) # Make sure processor was called self.assertEqual(self.mock_processor.process_completed.call_count, 2) # Make sure final recipe attributes are updated recipe = Recipe.objects.get(pk=recipe_id) self.assertIsNotNone(recipe.completed)
def test_convert_1_0_to_1_1(self): json_manifest = { "version": "1.0", "files": [ {"name" : "output_file", "path" : "/tmp/job_exe_231/outputs/output.csv"}, {"name": "output_files", "paths": ["/tmp/job_exe_231/outputs/output.csv", "/tmp/job_exe_231/outputs/output2.csv"]} ], "parse_results": [ { "filename" : "myfile.h5", "data_started" : "2015-05-15T10:34:12Z", "data_ended" : "2015-05-15T10:36:12Z", "data_types" : ["H5", "VEG"] } ], "errors": [] } new_format = { "version": "1.1", "output_data": [ { "name" : "output_file", "file": { "path" : "/tmp/job_exe_231/outputs/output.csv" } }, { "name" : "output_files", "files": [ { "path" : "/tmp/job_exe_231/outputs/output.csv" }, { "path" : "/tmp/job_exe_231/outputs/output2.csv" } ] } ], "parse_results": [ { "filename" : "myfile.h5", "data_started" : "2015-05-15T10:34:12Z", "data_ended" : "2015-05-15T10:36:12Z", "data_types" : ["H5", "VEG"] } ], "errors": [] } manifest = ResultsManifest(json_manifest) converted = manifest._convert_schema(json_manifest) self.assertEqual(converted, new_format)
def test_convert_1_0_to_1_1(self): json_manifest = { "version": "1.0", "files": [{ "name": "output_file", "path": "/tmp/job_exe_231/outputs/output.csv" }, { "name": "output_files", "paths": [ "/tmp/job_exe_231/outputs/output.csv", "/tmp/job_exe_231/outputs/output2.csv" ] }], "parse_results": [{ "filename": "myfile.h5", "data_started": "2015-05-15T10:34:12Z", "data_ended": "2015-05-15T10:36:12Z", "data_types": ["H5", "VEG"] }], "errors": [] } new_format = { "version": "1.1", "output_data": [{ "name": "output_file", "file": { "path": "/tmp/job_exe_231/outputs/output.csv" } }, { "name": "output_files", "files": [{ "path": "/tmp/job_exe_231/outputs/output.csv" }, { "path": "/tmp/job_exe_231/outputs/output2.csv" }] }], "parse_results": [{ "filename": "myfile.h5", "data_started": "2015-05-15T10:34:12Z", "data_ended": "2015-05-15T10:36:12Z", "data_types": ["H5", "VEG"] }], "errors": [] } manifest = ResultsManifest(json_manifest) converted = manifest._convert_schema(json_manifest) self.assertEqual(converted, new_format)
def test_simple_validation(self): json_manifest = { "version": "1.0", "files": [{ "name": "foo", "path": "nfs:server//myfile.txt" }] } input_files = {"input_file": False} output_files = {"foo": (False, True)} manifest = ResultsManifest(json_manifest) manifest.validate(output_files)
def test_simple_validation(self): json_manifest = { "version": "1.0", "files": [ {"name":"foo", "path":"nfs:server//myfile.txt"} ] } input_files = { "input_file": False } output_files = { "foo": (False, True) } manifest = ResultsManifest(json_manifest) manifest.validate(output_files)
def test_missing_optional_is_ok(self): json_manifest = { "version": "1.0", "files": [{ "name": "foo", "path": "nfs:server//myfile.txt" }] } input_files = {"input_file": False} output_files = { "foo": (False, True), "bar": (False, False) #This is an optional file } manifest = ResultsManifest(json_manifest) try: manifest.validate(output_files) except ResultsManifestAndInterfaceDontMatch: self.fail(u'The missing an optional file')
def test_output_does_not_match(self): json_manifest = { "version": "1.0", "files": [{ "name": "foo", "path": "nfs:server//myfile.txt" }] } input_files = {"input_file": False} output_files = {"bar": (False, True)} manifest = ResultsManifest(json_manifest) try: manifest.validate(output_files) self.fail( u'The outputs do not match the manifest, there should be a failure' ) except ResultsManifestAndInterfaceDontMatch: pass
def test_output_does_not_match(self): json_manifest = { "version": "1.0", "files": [ {"name":"foo", "path":"nfs:server//myfile.txt"} ] } input_files = { "input_file": False } output_files = { "bar": (False, True) } manifest = ResultsManifest(json_manifest) try: manifest.validate(output_files) self.fail(u'The outputs do not match the manifest, there should be a failure') except ResultsManifestAndInterfaceDontMatch: pass
def test_missing_optional_is_ok(self): json_manifest = { "version": "1.0", "files": [ {"name":"foo", "path":"nfs:server//myfile.txt"} ] } input_files = { "input_file": False } output_files = { "foo": (False, True), "bar": (False, False) #This is an optional file } manifest = ResultsManifest(json_manifest) try: manifest.validate(output_files) except ResultsManifestAndInterfaceDontMatch: self.fail(u'The missing an optional file')
def test_missing_required_is_bad(self): json_manifest = { "version": "1.0", "files": [ {"name":"foo", "path":"nfs:server//myfile.txt"} ] } input_files = { "input_file": False } output_files = { "foo": (False, True), "bar": (False, True) #This is a missing required file } manifest = ResultsManifest(json_manifest) try: manifest.validate(output_files) self.fail(u'There is a missing required file. Validation should have failed') except ResultsManifestAndInterfaceDontMatch: pass
def test_missing_required_is_bad(self): json_manifest = { "version": "1.0", "files": [ {"name":"foo", "path":"nfs:server//myfile.txt"} ] } input_files = { "input_file": False } output_files = { "foo": (False, True), "bar": (False, True) #This is a missing required file } manifest = ResultsManifest(json_manifest) try: manifest.validate(output_files) self.fail(u'There is a missing required file. Validation should have failed') except MissingRequiredOutput: pass
def test_manifest_supports_file_with_paths(self): json_manifest = { "version": "1.0", "files": [ {"name":"foo", "paths":["nfs:server//myfile.txt"]} ] } try: #This should not throw an exception since it is valid ResultsManifest(json_manifest) except InvalidResultsManifest: self.fail(u'This simple json_manifest is valid')
def test_invalid_results_manifest(self): json_manifest = { "version": "1.0", "files": [ {"name":"foo", "path":"nfs:server//myfile.txt", "paths": ["nfs:server//why_do_i_have_path_and_paths"]} ] } try: ResultsManifest(json_manifest) self.fail(u'files in a results manifest should not have both path and paths') except InvalidResultsManifest: #This should throw an exception since it is invalid pass
def test_manifest_version_1_1(self): json_manifest = { "version": "1.1", "output_data": [{ "name": "output_file", "file": { "path": "/tmp/job_exe_231/outputs/output.csv", "geo_metadata": { "data_started": "2015-05-15T10:34:12Z", "data_ended": "2015-05-15T10:36:12Z", "geo_json": { "type": "Polygon", "coordinates": [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0], [1.0, 20.0], [1.0, 10.0]]] } } } }, { "name": "output_files", "files": [{ "path": "/tmp/job_exe_231/outputs/output.csv", "geo_metadata": { "data_started": "2015-05-15T10:34:12Z", "data_ended": "2015-05-15T10:36:12Z", "geo_json": { "type": "Polygon", "coordinates": [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0], [1.0, 20.0], [1.0, 10.0]]] } } }, { "path": "/tmp/job_exe_231/outputs/output2.csv" }] }], "parse_results": [{ "filename": "myfile.h5", "data_started": "2015-05-15T10:34:12Z", "data_ended": "2015-05-15T10:36:12Z", "data_types": ["H5", "VEG"] }] } manifest = ResultsManifest(json_manifest)
def test_empty_results_manifest(self): json_manifest = {} #This should not throw an exception since it is valid ResultsManifest(json_manifest)
def perform_post_steps(self, job_exe, job_data, stdoutAndStderr): """Stores the files and deletes any working directories :param job_exe: The job execution model with related job and job_type fields :type job_exe: :class:`job.models.JobExecution` :param job_data: The job data :type job_data: :class:`job.configuration.data.job_data.JobData` :param stdoutAndStderr: the standard out from the job execution :type stdoutAndStderr: str :return: A tuple of the job results and the results manifest generated by the job execution :rtype: (:class:`job.configuration.results.job_results.JobResults`, :class:`job.configuration.results.results_manifest.results_manifest.ResultsManifest`) """ manifest_data = {} path_to_manifest_file = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH, 'results_manifest.json') if os.path.exists(path_to_manifest_file): logger.info('Opening results manifest...') with open(path_to_manifest_file, 'r') as manifest_file: manifest_data = json.loads(manifest_file.read()) logger.info('Results manifest:') logger.info(manifest_data) else: logger.info('No results manifest found') results_manifest = ResultsManifest(manifest_data) stdout_files = self._get_artifacts_from_stdout(stdoutAndStderr) results_manifest.add_files(stdout_files) results_manifest.validate(self._output_file_manifest_dict) files_to_store = {} for manifest_file_entry in results_manifest.get_files(): param_name = manifest_file_entry['name'] media_type = None output_data_item = self._get_output_data_item_by_name(param_name) if output_data_item: media_type = output_data_item.get('media_type') msg = 'Output %s has invalid/missing file path "%s"' if 'file' in manifest_file_entry: file_entry = manifest_file_entry['file'] if not os.path.isfile(file_entry['path']): raise InvalidResultsManifest( msg % (param_name, file_entry['path'])) if 'geo_metadata' in file_entry: files_to_store[param_name] = (file_entry['path'], media_type, file_entry['geo_metadata']) else: files_to_store[param_name] = (file_entry['path'], media_type) elif 'files' in manifest_file_entry: file_tuples = [] for file_entry in manifest_file_entry['files']: if not os.path.isfile(file_entry['path']): raise InvalidResultsManifest( msg % (param_name, file_entry['path'])) if 'geo_metadata' in file_entry: file_tuples.append((file_entry['path'], media_type, file_entry['geo_metadata'])) else: file_tuples.append((file_entry['path'], media_type)) files_to_store[param_name] = file_tuples job_data_parse_results = {} # parse results formatted for job_data for parse_result in results_manifest.get_parse_results(): filename = parse_result['filename'] assert filename not in job_data_parse_results geo_metadata = parse_result.get('geo_metadata', {}) geo_json = geo_metadata.get('geo_json', None) data_started = geo_metadata.get('data_started', None) data_ended = geo_metadata.get('data_ended', None) data_types = parse_result.get('data_types', []) new_workspace_path = parse_result.get('new_workspace_path', None) if new_workspace_path: new_workspace_path = os.path.join(new_workspace_path, filename) job_data_parse_results[filename] = (geo_json, data_started, data_ended, data_types, new_workspace_path) job_data.save_parse_results(job_data_parse_results) return (job_data.store_output_data_files(files_to_store, job_exe), results_manifest)
from django.db.utils import DatabaseError, OperationalError from django.utils.timezone import now from django.test import TransactionTestCase from mock import patch from error.exceptions import ScaleDatabaseError, ScaleIOError, ScaleOperationalError from job.configuration.results.exceptions import InvalidResultsManifest, MissingRequiredOutput from job.configuration.results.job_results import JobResults from job.configuration.results.results_manifest.results_manifest import ResultsManifest from job.management.commands.scale_post_steps import Command as PostCommand from job.models import JobExecutionOutput from job.test import utils as job_utils from trigger.models import TriggerEvent JOB_RESULTS = JobResults() RESULTS_MANIFEST = ResultsManifest() RESULTS = (JOB_RESULTS, RESULTS_MANIFEST) class TestPostJobSteps(TransactionTestCase): def setUp(self): django.setup() cmd = 'command' cmd_args = 'args' interface = { 'version': '1.0', 'command': cmd, 'command_arguments':
def perform_post_steps(self, job_exe, job_data, stdoutAndStderr): """Stores the files and deletes any working directories :param job_exe: The job execution model with related job and job_type fields :type job_exe: :class:`job.models.JobExecution` :param job_data: The job data :type job_data: :class:`job.configuration.data.job_data.JobData` :param stdoutAndStderr: the standard out from the job execution :type stdoutAndStderr: str :return: A tuple of the job results and the results manifest generated by the job execution :rtype: (:class:`job.configuration.results.job_results.JobResults`, :class:`job.configuration.results.results_manifest.results_manifest.ResultsManifest`) """ manifest_data = {} path_to_manifest_file = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH, 'results_manifest.json') if os.path.exists(path_to_manifest_file): logger.info('Opening results manifest...') with open(path_to_manifest_file, 'r') as manifest_file: manifest_data = json.loads(manifest_file.read()) logger.info('Results manifest:') logger.info(manifest_data) else: logger.info('No results manifest found') results_manifest = ResultsManifest(manifest_data) stdout_files = self._get_artifacts_from_stdout(stdoutAndStderr) results_manifest.add_files(stdout_files) results_manifest.validate(self._output_file_manifest_dict) files_to_store = {} for manifest_file_entry in results_manifest.get_files(): param_name = manifest_file_entry['name'] media_type = None output_data_item = self._get_output_data_item_by_name(param_name) if output_data_item: media_type = output_data_item.get('media_type') if 'file' in manifest_file_entry: file_entry = manifest_file_entry['file'] if 'geo_metadata' in file_entry: files_to_store[param_name] = (file_entry['path'], media_type, file_entry['geo_metadata']) else: files_to_store[param_name] = (file_entry['path'], media_type) elif 'files' in manifest_file_entry: file_tuples = [] for file_entry in manifest_file_entry['files']: if 'geo_metadata' in file_entry: file_tuples.append((file_entry['path'], media_type, file_entry['geo_metadata'])) else: file_tuples.append((file_entry['path'], media_type)) files_to_store[param_name] = file_tuples job_data_parse_results = {} # parse results formatted for job_data for parse_result in results_manifest.get_parse_results(): filename = parse_result['filename'] assert filename not in job_data_parse_results geo_metadata = parse_result.get('geo_metadata', {}) geo_json = geo_metadata.get('geo_json', None) data_started = geo_metadata.get('data_started', None) data_ended = geo_metadata.get('data_ended', None) data_types = parse_result.get('data_types', []) new_workspace_path = parse_result.get('new_workspace_path', None) if new_workspace_path: new_workspace_path = os.path.join(new_workspace_path, filename) job_data_parse_results[filename] = (geo_json, data_started, data_ended, data_types, new_workspace_path) job_data.save_parse_results(job_data_parse_results) return (job_data.store_output_data_files(files_to_store, job_exe), results_manifest)