예제 #1
0
    def test_simple_validation_1_1(self):
        json_manifest = {
            "version":
            "1.1",
            "output_data": [{
                "name": "output_file",
                "file": {
                    "path": "/tmp/job_exe_231/outputs/output.csv",
                    "geo_metadata": {
                        "data_started": "2015-05-15T10:34:12Z",
                        "data_ended": "2015-05-15T10:36:12Z",
                        "geo_json": {
                            "type":
                            "Polygon",
                            "coordinates": [[[1.0, 10.0], [2.0, 10.0],
                                             [2.0, 20.0], [1.0, 20.0],
                                             [1.0, 10.0]]]
                        }
                    }
                }
            }]
        }

        input_files = {"input_file": False}
        output_files = {"output_file": (False, True)}
        manifest = ResultsManifest(json_manifest)
        manifest.validate(output_files)
예제 #2
0
    def test_simple_validation_1_1(self):
        json_manifest = {
            "version": "1.1",
            "output_data": [
                {
                    "name" : "output_file",
                    "file": {
                        "path" : "/tmp/job_exe_231/outputs/output.csv",
                        "geo_metadata": {
                            "data_started": "2015-05-15T10:34:12Z",
                            "data_ended" : "2015-05-15T10:36:12Z",
                            "geo_json": {
                                "type": "Polygon",
                                "coordinates": [ [ [ 1.0, 10.0 ], [ 2.0, 10.0 ], [ 2.0, 20.0 ],[ 1.0, 20.0 ], [ 1.0, 10.0 ] ] ]
                            }
                        }
                    }
                }
            ]
        }

        input_files = {
            "input_file": False
        }
        output_files = {
            "output_file": (False, True)
        }
        manifest = ResultsManifest(json_manifest)
        manifest.validate(output_files)
예제 #3
0
 def test_simple_validation(self):
     json_manifest = {
         "version": "1.0",
         "files": [{
             "name": "foo",
             "path": "nfs:server//myfile.txt"
         }]
     }
     input_files = {"input_file": False}
     output_files = {"foo": (False, True)}
     manifest = ResultsManifest(json_manifest)
     manifest.validate(output_files)
예제 #4
0
 def test_simple_validation(self):
     json_manifest = {
         "version": "1.0",
         "files": [
             {"name":"foo", "path":"nfs:server//myfile.txt"}
         ]
     }
     input_files = {
         "input_file": False
     }
     output_files = {
         "foo": (False, True)
     }
     manifest = ResultsManifest(json_manifest)
     manifest.validate(output_files)
예제 #5
0
 def test_missing_optional_is_ok(self):
     json_manifest = {
         "version": "1.0",
         "files": [{
             "name": "foo",
             "path": "nfs:server//myfile.txt"
         }]
     }
     input_files = {"input_file": False}
     output_files = {
         "foo": (False, True),
         "bar": (False, False)  #This is an optional file
     }
     manifest = ResultsManifest(json_manifest)
     try:
         manifest.validate(output_files)
     except ResultsManifestAndInterfaceDontMatch:
         self.fail(u'The missing an optional file')
예제 #6
0
 def test_output_does_not_match(self):
     json_manifest = {
         "version": "1.0",
         "files": [{
             "name": "foo",
             "path": "nfs:server//myfile.txt"
         }]
     }
     input_files = {"input_file": False}
     output_files = {"bar": (False, True)}
     manifest = ResultsManifest(json_manifest)
     try:
         manifest.validate(output_files)
         self.fail(
             u'The outputs do not match the manifest, there should be a failure'
         )
     except ResultsManifestAndInterfaceDontMatch:
         pass
예제 #7
0
 def test_missing_optional_is_ok(self):
     json_manifest = {
         "version": "1.0",
         "files": [
             {"name":"foo", "path":"nfs:server//myfile.txt"}
         ]
     }
     input_files = {
         "input_file": False
     }
     output_files = {
         "foo": (False, True),
         "bar": (False, False)  #This is an optional file
     }
     manifest = ResultsManifest(json_manifest)
     try:
         manifest.validate(output_files)
     except ResultsManifestAndInterfaceDontMatch:
         self.fail(u'The missing an optional file')
예제 #8
0
 def test_output_does_not_match(self):
     json_manifest = {
         "version": "1.0",
         "files": [
             {"name":"foo", "path":"nfs:server//myfile.txt"}
         ]
     }
     input_files = {
         "input_file": False
     }
     output_files = {
         "bar": (False, True)
     }
     manifest = ResultsManifest(json_manifest)
     try:
         manifest.validate(output_files)
         self.fail(u'The outputs do not match the manifest, there should be a failure')
     except ResultsManifestAndInterfaceDontMatch:
         pass
예제 #9
0
 def test_missing_required_is_bad(self):
     json_manifest = {
         "version": "1.0",
         "files": [
             {"name":"foo", "path":"nfs:server//myfile.txt"}
         ]
     }
     input_files = {
         "input_file": False
     }
     output_files = {
         "foo": (False, True),
         "bar": (False, True)  #This is a missing required file
     }
     manifest = ResultsManifest(json_manifest)
     try:
         manifest.validate(output_files)
         self.fail(u'There is a missing required file.  Validation should have failed')
     except ResultsManifestAndInterfaceDontMatch:
         pass
예제 #10
0
 def test_missing_required_is_bad(self):
     json_manifest = {
         "version": "1.0",
         "files": [
             {"name":"foo", "path":"nfs:server//myfile.txt"}
         ]
     }
     input_files = {
         "input_file": False
     }
     output_files = {
         "foo": (False, True),
         "bar": (False, True)  #This is a missing required file
     }
     manifest = ResultsManifest(json_manifest)
     try:
         manifest.validate(output_files)
         self.fail(u'There is a missing required file.  Validation should have failed')
     except MissingRequiredOutput:
         pass
예제 #11
0
    def perform_post_steps(self, job_exe, job_data, stdoutAndStderr):
        """Stores the files and deletes any working directories

        :param job_exe: The job execution model with related job and job_type fields
        :type job_exe: :class:`job.models.JobExecution`
        :param job_data: The job data
        :type job_data: :class:`job.configuration.data.job_data.JobData`
        :param stdoutAndStderr: the standard out from the job execution
        :type stdoutAndStderr: str
        :return: A tuple of the job results and the results manifest generated by the job execution
        :rtype: (:class:`job.configuration.results.job_results.JobResults`,
            :class:`job.configuration.results.results_manifest.results_manifest.ResultsManifest`)
        """

        manifest_data = {}
        path_to_manifest_file = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH,
                                             'results_manifest.json')
        if os.path.exists(path_to_manifest_file):
            logger.info('Opening results manifest...')
            with open(path_to_manifest_file, 'r') as manifest_file:
                manifest_data = json.loads(manifest_file.read())
                logger.info('Results manifest:')
                logger.info(manifest_data)
        else:
            logger.info('No results manifest found')

        results_manifest = ResultsManifest(manifest_data)
        stdout_files = self._get_artifacts_from_stdout(stdoutAndStderr)
        results_manifest.add_files(stdout_files)

        results_manifest.validate(self._output_file_manifest_dict)

        files_to_store = {}
        for manifest_file_entry in results_manifest.get_files():
            param_name = manifest_file_entry['name']

            media_type = None
            output_data_item = self._get_output_data_item_by_name(param_name)
            if output_data_item:
                media_type = output_data_item.get('media_type')

            msg = 'Output %s has invalid/missing file path "%s"'
            if 'file' in manifest_file_entry:
                file_entry = manifest_file_entry['file']
                if not os.path.isfile(file_entry['path']):
                    raise InvalidResultsManifest(
                        msg % (param_name, file_entry['path']))
                if 'geo_metadata' in file_entry:
                    files_to_store[param_name] = (file_entry['path'],
                                                  media_type,
                                                  file_entry['geo_metadata'])
                else:
                    files_to_store[param_name] = (file_entry['path'],
                                                  media_type)
            elif 'files' in manifest_file_entry:
                file_tuples = []
                for file_entry in manifest_file_entry['files']:
                    if not os.path.isfile(file_entry['path']):
                        raise InvalidResultsManifest(
                            msg % (param_name, file_entry['path']))
                    if 'geo_metadata' in file_entry:
                        file_tuples.append((file_entry['path'], media_type,
                                            file_entry['geo_metadata']))
                    else:
                        file_tuples.append((file_entry['path'], media_type))
                files_to_store[param_name] = file_tuples

        job_data_parse_results = {}  # parse results formatted for job_data
        for parse_result in results_manifest.get_parse_results():
            filename = parse_result['filename']
            assert filename not in job_data_parse_results
            geo_metadata = parse_result.get('geo_metadata', {})
            geo_json = geo_metadata.get('geo_json', None)
            data_started = geo_metadata.get('data_started', None)
            data_ended = geo_metadata.get('data_ended', None)
            data_types = parse_result.get('data_types', [])
            new_workspace_path = parse_result.get('new_workspace_path', None)
            if new_workspace_path:
                new_workspace_path = os.path.join(new_workspace_path, filename)
            job_data_parse_results[filename] = (geo_json, data_started,
                                                data_ended, data_types,
                                                new_workspace_path)

        job_data.save_parse_results(job_data_parse_results)
        return (job_data.store_output_data_files(files_to_store,
                                                 job_exe), results_manifest)
예제 #12
0
    def perform_post_steps(self, job_exe, job_data, stdoutAndStderr):
        """Stores the files and deletes any working directories

        :param job_exe: The job execution model with related job and job_type fields
        :type job_exe: :class:`job.models.JobExecution`
        :param job_data: The job data
        :type job_data: :class:`job.configuration.data.job_data.JobData`
        :param stdoutAndStderr: the standard out from the job execution
        :type stdoutAndStderr: str
        :return: A tuple of the job results and the results manifest generated by the job execution
        :rtype: (:class:`job.configuration.results.job_results.JobResults`,
            :class:`job.configuration.results.results_manifest.results_manifest.ResultsManifest`)
        """

        manifest_data = {}
        path_to_manifest_file = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH, 'results_manifest.json')
        if os.path.exists(path_to_manifest_file):
            logger.info('Opening results manifest...')
            with open(path_to_manifest_file, 'r') as manifest_file:
                manifest_data = json.loads(manifest_file.read())
                logger.info('Results manifest:')
                logger.info(manifest_data)
        else:
            logger.info('No results manifest found')

        results_manifest = ResultsManifest(manifest_data)
        stdout_files = self._get_artifacts_from_stdout(stdoutAndStderr)
        results_manifest.add_files(stdout_files)

        results_manifest.validate(self._output_file_manifest_dict)

        files_to_store = {}
        for manifest_file_entry in results_manifest.get_files():
            param_name = manifest_file_entry['name']

            media_type = None
            output_data_item = self._get_output_data_item_by_name(param_name)
            if output_data_item:
                media_type = output_data_item.get('media_type')

            if 'file' in manifest_file_entry:
                file_entry = manifest_file_entry['file']
                if 'geo_metadata' in file_entry:
                    files_to_store[param_name] = (file_entry['path'], media_type, file_entry['geo_metadata'])
                else:
                    files_to_store[param_name] = (file_entry['path'], media_type)
            elif 'files' in manifest_file_entry:
                file_tuples = []
                for file_entry in manifest_file_entry['files']:
                    if 'geo_metadata' in file_entry:
                        file_tuples.append((file_entry['path'], media_type, file_entry['geo_metadata']))
                    else:
                        file_tuples.append((file_entry['path'], media_type))
                files_to_store[param_name] = file_tuples

        job_data_parse_results = {}  # parse results formatted for job_data
        for parse_result in results_manifest.get_parse_results():
            filename = parse_result['filename']
            assert filename not in job_data_parse_results
            geo_metadata = parse_result.get('geo_metadata', {})
            geo_json = geo_metadata.get('geo_json', None)
            data_started = geo_metadata.get('data_started', None)
            data_ended = geo_metadata.get('data_ended', None)
            data_types = parse_result.get('data_types', [])
            new_workspace_path = parse_result.get('new_workspace_path', None)
            if new_workspace_path:
                new_workspace_path = os.path.join(new_workspace_path, filename)
            job_data_parse_results[filename] = (geo_json, data_started, data_ended, data_types, new_workspace_path)

        job_data.save_parse_results(job_data_parse_results)
        return (job_data.store_output_data_files(files_to_store, job_exe), results_manifest)