def perform_post_steps(self, job_exe, job_data, stdoutAndStderr):
"""Stores the files and deletes any working directories
:param job_exe: The job execution model with related job and job_type fields
:type job_exe: :class:`job.models.JobExecution`
:param job_data: The job data
:type job_data: :class:`job.configuration.data.job_data.JobData`
:param stdoutAndStderr: the standard out from the job execution
:type stdoutAndStderr: str
:return: A tuple of the job results and the results manifest generated by the job execution
:rtype: (:class:`job.configuration.results.job_results.JobResults`,
:class:`job.configuration.results.results_manifest.results_manifest.ResultsManifest`)
"""
manifest_data = {}
path_to_manifest_file = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH,
'results_manifest.json')
if os.path.exists(path_to_manifest_file):
logger.info('Opening results manifest...')
with open(path_to_manifest_file, 'r') as manifest_file:
manifest_data = json.loads(manifest_file.read())
logger.info('Results manifest:')
logger.info(manifest_data)
else:
logger.info('No results manifest found')
results_manifest = ResultsManifest(manifest_data)
stdout_files = self._get_artifacts_from_stdout(stdoutAndStderr)
results_manifest.add_files(stdout_files)
results_manifest.validate(self._output_file_manifest_dict)
files_to_store = {}
for manifest_file_entry in results_manifest.get_files():
param_name = manifest_file_entry['name']
media_type = None
output_data_item = self._get_output_data_item_by_name(param_name)
if output_data_item:
media_type = output_data_item.get('media_type')
msg = 'Output %s has invalid/missing file path "%s"'
if 'file' in manifest_file_entry:
file_entry = manifest_file_entry['file']
if not os.path.isfile(file_entry['path']):
raise InvalidResultsManifest(
msg % (param_name, file_entry['path']))
if 'geo_metadata' in file_entry:
files_to_store[param_name] = (file_entry['path'],
media_type,
file_entry['geo_metadata'])
else:
files_to_store[param_name] = (file_entry['path'],
media_type)
elif 'files' in manifest_file_entry:
file_tuples = []
for file_entry in manifest_file_entry['files']:
if not os.path.isfile(file_entry['path']):
raise InvalidResultsManifest(
msg % (param_name, file_entry['path']))
if 'geo_metadata' in file_entry:
file_tuples.append((file_entry['path'], media_type,
file_entry['geo_metadata']))
else:
file_tuples.append((file_entry['path'], media_type))
files_to_store[param_name] = file_tuples
job_data_parse_results = {} # parse results formatted for job_data
for parse_result in results_manifest.get_parse_results():
filename = parse_result['filename']
assert filename not in job_data_parse_results
geo_metadata = parse_result.get('geo_metadata', {})
geo_json = geo_metadata.get('geo_json', None)
data_started = geo_metadata.get('data_started', None)
data_ended = geo_metadata.get('data_ended', None)
data_types = parse_result.get('data_types', [])
new_workspace_path = parse_result.get('new_workspace_path', None)
if new_workspace_path:
new_workspace_path = os.path.join(new_workspace_path, filename)
job_data_parse_results[filename] = (geo_json, data_started,
data_ended, data_types,
new_workspace_path)
job_data.save_parse_results(job_data_parse_results)
return (job_data.store_output_data_files(files_to_store,
job_exe), results_manifest)
def perform_post_steps(self, job_exe, job_data, stdoutAndStderr):
"""Stores the files and deletes any working directories
:param job_exe: The job execution model with related job and job_type fields
:type job_exe: :class:`job.models.JobExecution`
:param job_data: The job data
:type job_data: :class:`job.configuration.data.job_data.JobData`
:param stdoutAndStderr: the standard out from the job execution
:type stdoutAndStderr: str
:return: A tuple of the job results and the results manifest generated by the job execution
:rtype: (:class:`job.configuration.results.job_results.JobResults`,
:class:`job.configuration.results.results_manifest.results_manifest.ResultsManifest`)
"""
manifest_data = {}
path_to_manifest_file = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH, 'results_manifest.json')
if os.path.exists(path_to_manifest_file):
logger.info('Opening results manifest...')
with open(path_to_manifest_file, 'r') as manifest_file:
manifest_data = json.loads(manifest_file.read())
logger.info('Results manifest:')
logger.info(manifest_data)
else:
logger.info('No results manifest found')
results_manifest = ResultsManifest(manifest_data)
stdout_files = self._get_artifacts_from_stdout(stdoutAndStderr)
results_manifest.add_files(stdout_files)
results_manifest.validate(self._output_file_manifest_dict)
files_to_store = {}
for manifest_file_entry in results_manifest.get_files():
param_name = manifest_file_entry['name']
media_type = None
output_data_item = self._get_output_data_item_by_name(param_name)
if output_data_item:
media_type = output_data_item.get('media_type')
if 'file' in manifest_file_entry:
file_entry = manifest_file_entry['file']
if 'geo_metadata' in file_entry:
files_to_store[param_name] = (file_entry['path'], media_type, file_entry['geo_metadata'])
else:
files_to_store[param_name] = (file_entry['path'], media_type)
elif 'files' in manifest_file_entry:
file_tuples = []
for file_entry in manifest_file_entry['files']:
if 'geo_metadata' in file_entry:
file_tuples.append((file_entry['path'], media_type, file_entry['geo_metadata']))
else:
file_tuples.append((file_entry['path'], media_type))
files_to_store[param_name] = file_tuples
job_data_parse_results = {} # parse results formatted for job_data
for parse_result in results_manifest.get_parse_results():
filename = parse_result['filename']
assert filename not in job_data_parse_results
geo_metadata = parse_result.get('geo_metadata', {})
geo_json = geo_metadata.get('geo_json', None)
data_started = geo_metadata.get('data_started', None)
data_ended = geo_metadata.get('data_ended', None)
data_types = parse_result.get('data_types', [])
new_workspace_path = parse_result.get('new_workspace_path', None)
if new_workspace_path:
new_workspace_path = os.path.join(new_workspace_path, filename)
job_data_parse_results[filename] = (geo_json, data_started, data_ended, data_types, new_workspace_path)
job_data.save_parse_results(job_data_parse_results)
return (job_data.store_output_data_files(files_to_store, job_exe), results_manifest)
