def validate(self, output_file_definitions):
"""Validates the results manifest against given output file definitions. This does not validate that the
parse_data matches the job data inputs.
:param output_file_definitions: A dictionary with each output param name mapped to a tuple with
(is_multiple (bool), required(bool))
:type output_file_definitions: dict of tuples
"""
self._trim(output_file_definitions)
file_entry_map = {}
for manifest_file_entry in self._json_manifest['output_data']:
entry_name = manifest_file_entry['name']
if 'files' in manifest_file_entry and not manifest_file_entry['files']:
# skip empty lists
continue
file_entry_map[entry_name] = manifest_file_entry
for file_name, (is_multiple, is_required) in output_file_definitions.items():
if file_name not in file_entry_map:
if is_required:
raise MissingRequiredOutput(file_name)
else:
continue
manifest_file_entry = file_entry_map[file_name]
if is_multiple and 'files' not in manifest_file_entry:
msg = 'The output parameter %s must have a files object in the results manifest' % file_name
raise InvalidResultsManifest(msg)
if not is_multiple and 'file' not in manifest_file_entry:
msg = 'The output parameter %s must have a file object in the results manifest' % file_name
raise InvalidResultsManifest(msg)
def validate(self, output_file_definitions):
"""Validates the results manifest against given output file definitions. Throws a
:class `job.configuration.results.exceptions.ResultsManifestAndInterfaceDontMatch`: if the
manifest doesn't match the outputs. This does not validate that the parse_data matches the job
data inputs.
:param output_file_definitions: A dictionary with each output param name mapped to a tuple with
(is_multiple (bool), required(bool))
:type output_file_definitions: dict of tuples
"""
self._trim(output_file_definitions)
file_entry_map = {}
for manifest_file_entry in self._json_manifest['files']:
entry_name = manifest_file_entry['name']
file_entry_map[entry_name] = manifest_file_entry
for file_name, (is_multiple,
is_required) in output_file_definitions.items():
if file_name not in file_entry_map:
if is_required:
raise MissingRequiredOutput(file_name)
else:
continue
manifest_file_entry = file_entry_map[file_name]
if is_multiple and 'paths' not in manifest_file_entry:
msg = 'The output parameter %s must have a paths object in the results manifest' % file_name
raise InvalidResultsManifest(msg)
if not is_multiple and 'path' not in manifest_file_entry:
msg = 'The output parameter %s must have a path object in the results manifest' % file_name
raise InvalidResultsManifest(msg)
def _validate_manifest(self):
'''validates portions of the manifest that cannot be validated with the json_schema'''
file_entries = set()
for manifest_file_entry in self._json_manifest[u'output_data']:
entry_name = manifest_file_entry[u'name']
if entry_name in file_entries:
raise InvalidResultsManifest(u'output names cannot be repeated')
file_entries.add(entry_name)
if u'file' in manifest_file_entry and u'files' in manifest_file_entry:
raise InvalidResultsManifest(u'an output_data entry can only have file or files, not both')
if u'file' not in manifest_file_entry and u'files' not in manifest_file_entry:
raise InvalidResultsManifest(u'an output_data entry must have either file or files')
def __init__(self, json_manifest=None):
'''Creates a result manifest from the json_manifest
:param json_manifest: a dict in the format described by RESULTS_MANIFEST_SCHEMA
:type json_manifest: dict
'''
if json_manifest is None:
json_manifest = {}
if u'version' in json_manifest:
version = json_manifest[u'version']
else:
version = MANIFEST_VERSION
if version != MANIFEST_VERSION:
json_manifest = self._convert_schema(json_manifest)
self._json_manifest = json_manifest
try:
validate(json_manifest, RESULTS_MANIFEST_SCHEMA)
except ValidationError as validation_error:
raise InvalidResultsManifest(validation_error)
self._populate_defaults()
self._validate_manifest()
def parse_geo_json(geo_json):
"""Parses GeoJSON and returns a geometry object and metadata.
:param geo_json: The geo json to parse
:type geo_json: dict
:rtype: GEOSGeometry, dict
:returns: the geometry and metadata
"""
geom = None
geom_json = None
props = None
if geo_json['type'] == 'Feature':
geom_json = geo_json['geometry']
if 'properties' in geo_json:
props = geo_json['properties']
elif geo_json['type'] == 'FeatureCollection':
# Currently handles collections by just grabbing first entry
geom_json = geo_json['features'][0]['geometry']
if 'properties' in geo_json['features'][0]:
props = geo_json['features'][0]['properties']
else:
# The GeoJSON is just a geometry
geom_json = geo_json
# Parse geometry
if geom_json:
try:
geom = geos.GEOSGeometry(json.dumps(geom_json), srid=4326)
except geos.GEOSException as geos_error:
raise InvalidResultsManifest(str(geos_error))
return geom, props
def test_determine_error(self):
"""Tests that a post-task successfully determines the correct error"""
scale_errors = [
ScaleDatabaseError(),
ScaleIOError(),
ScaleOperationalError(),
InvalidResultsManifest(''),
MissingRequiredOutput('')
]
for scale_error in scale_errors:
config = ExecutionConfiguration()
config.create_tasks(['pre'])
config.set_task_ids(self.job_exe.get_cluster_id())
task = PostTask('agent_1', self.job_exe, self.job_exe.job_type,
config)
update = job_test_utils.create_task_status_update(
task.id, task.agent_id, TaskStatusUpdate.RUNNING, now())
task.update(update)
update = job_test_utils.create_task_status_update(
task.id,
task.agent_id,
TaskStatusUpdate.FAILED,
now(),
exit_code=scale_error.exit_code)
error = task.determine_error(update)
self.assertEqual(scale_error.error_name, error.name)
def _validate_manifest(self):
"""validates portions of the manifest that cannot be validated with the json_schema"""
file_entries = set()
for manifest_file_entry in self._json_manifest['output_data']:
entry_name = manifest_file_entry['name']
if entry_name in file_entries:
msg = 'The output parameter %s appears multiple times in the results manifest' % entry_name
raise InvalidResultsManifest(msg)
file_entries.add(entry_name)
if 'file' in manifest_file_entry and 'files' in manifest_file_entry:
msg = 'The output parameter %s cannot have both file and files objects in the results manifest'
raise InvalidResultsManifest(msg % entry_name)
if 'file' not in manifest_file_entry and 'files' not in manifest_file_entry:
msg = 'The output parameter %s must have either a file or files object in the results manifest'
raise InvalidResultsManifest(msg % entry_name)
def ready(self):
"""Registers components related to jobs"""
# Register job errors
from error.exceptions import register_error
from job.configuration.exceptions import MissingMount, MissingSetting
from job.configuration.results.exceptions import InvalidResultsManifest, MissingRequiredOutput
register_error(InvalidResultsManifest(''))
register_error(MissingMount(''))
register_error(MissingRequiredOutput(''))
register_error(MissingSetting(''))
# Register job message types
from job.messages.blocked_jobs import BlockedJobs
from job.messages.failed_jobs import FailedJobs
from job.messages.job_exe_end import CreateJobExecutionEnd
from job.messages.pending_jobs import PendingJobs
from job.messages.running_jobs import RunningJobs
from messaging.messages.factory import add_message_type
add_message_type(BlockedJobs)
add_message_type(FailedJobs)
add_message_type(CreateJobExecutionEnd)
add_message_type(PendingJobs)
add_message_type(RunningJobs)
def _convert_schema(self, json_manifest):
'''Convert the previous manifest schema to the 1.0 manifest schema
:param json_manifest: The old manifest
:type json_manifest: dict
:return: converted manifest
:rtype: dict
'''
# There are no supported manifest versions before 1.0
raise InvalidResultsManifest()
def ready(self):
"""Registers job errors"""
from error.exceptions import register_error
from job.configuration.exceptions import MissingMount, MissingSetting
from job.configuration.results.exceptions import InvalidResultsManifest, MissingRequiredOutput
register_error(InvalidResultsManifest(''))
register_error(MissingMount(''))
register_error(MissingRequiredOutput(''))
register_error(MissingSetting(''))
def _convert_schema(self, json_manifest):
"""Convert the previous manifest schema to the 1.0 manifest schema
:param json_manifest: The old manifest
:type json_manifest: dict
:return: converted manifest
:rtype: dict
"""
# There are no supported manifest versions before 1.0
raise InvalidResultsManifest(
'%s is an invalid results manifest version' %
json_manifest['version'])
def __init__(self, json_manifest):
'''Creates a result manifest from the json_manifest
:param json_manifest: a dict in the format described by RESULTS_MANIFEST_SCHEMA
:type json_manifest: dict
'''
self._json_manifest = json_manifest
try:
validate(json_manifest, RESULTS_MANIFEST_SCHEMA)
except ValidationError as validation_error:
raise InvalidResultsManifest(validation_error)
self._populate_defaults()
self._validate_manifest()
def ready(self):
"""Registers components related to jobs"""
# Register job errors
from error.exceptions import register_error
from job.configuration.results.exceptions import InvalidResultsManifest, MissingRequiredOutput, UnexpectedMultipleOutputs
from job.execution.configuration.exceptions import MissingMount, MissingSetting
register_error(InvalidResultsManifest(''))
register_error(MissingMount(''))
register_error(MissingRequiredOutput(''))
register_error(UnexpectedMultipleOutputs(''))
register_error(MissingSetting(''))
# Register job message types
from job.messages.blocked_jobs import BlockedJobs
from job.messages.cancel_jobs import CancelJobs
from job.messages.cancel_jobs_bulk import CancelJobsBulk
from job.messages.completed_jobs import CompletedJobs
from job.messages.create_jobs import CreateJobs
from job.messages.failed_jobs import FailedJobs
from job.messages.job_exe_end import CreateJobExecutionEnd
from job.messages.pending_jobs import PendingJobs
from job.messages.process_job_input import ProcessJobInput
from job.messages.publish_job import PublishJob
from job.messages.purge_jobs import PurgeJobs
from job.messages.running_jobs import RunningJobs
from job.messages.spawn_delete_files_job import SpawnDeleteFilesJob
from job.messages.uncancel_jobs import UncancelJobs
from job.messages.unpublish_jobs import UnpublishJobs
from messaging.messages.factory import add_message_type
add_message_type(BlockedJobs)
add_message_type(CancelJobs)
add_message_type(CancelJobsBulk)
add_message_type(CompletedJobs)
add_message_type(CreateJobs)
add_message_type(FailedJobs)
add_message_type(CreateJobExecutionEnd)
add_message_type(PendingJobs)
add_message_type(ProcessJobInput)
add_message_type(PublishJob)
add_message_type(PurgeJobs)
add_message_type(RunningJobs)
add_message_type(SpawnDeleteFilesJob)
add_message_type(UncancelJobs)
add_message_type(UnpublishJobs)
def test_scale_post_steps_invalid_manifest_error(self,
mock_job_exe_manager,
mock_sys_exit):
"""Tests executing scale_post_steps when an invalid manifest occurs."""
# Set up mocks
mock_job_exe_manager.get_job_exe_with_job_and_job_type.return_value.get_job_interface.return_value.perform_post_steps.side_effect = InvalidResultsManifest(
'')
# Call method to test
cmd = PostCommand()
cmd.run_from_argv(
['manage.py', 'scale_post_steps', '-i',
str(self.job_exe.id)])
# Check results
mock_sys_exit.assert_called_with(InvalidResultsManifest('').exit_code)
def test_scale_post_steps_invalid_manifest_error(self, mock_env_vars,
mock_job_exe_manager,
mock_sys_exit):
"""Tests executing scale_post_steps when an invalid manifest occurs."""
# Set up mocks
def get_env_vars(name, *args, **kwargs):
return str(self.job.id) if name == 'SCALE_JOB_ID' else str(
self.job_exe.exe_num)
mock_env_vars.side_effect = get_env_vars
mock_job_exe_manager.get_job_exe_with_job_and_job_type.return_value.job_type.get_job_interface.return_value.perform_post_steps.side_effect = InvalidResultsManifest(
'')
# Call method to test
cmd = PostCommand()
cmd.run_from_argv(['manage.py', 'scale_post_steps'])
# Check results
mock_sys_exit.assert_called_with(InvalidResultsManifest('').exit_code)
def perform_post_steps(self, job_exe, job_data, stdoutAndStderr):
"""Stores the files and deletes any working directories
:param job_exe: The job execution model with related job and job_type fields
:type job_exe: :class:`job.models.JobExecution`
:param job_data: The job data
:type job_data: :class:`job.configuration.data.job_data.JobData`
:param stdoutAndStderr: the standard out from the job execution
:type stdoutAndStderr: str
:return: A tuple of the job results and the results manifest generated by the job execution
:rtype: (:class:`job.configuration.results.job_results.JobResults`,
:class:`job.configuration.results.results_manifest.results_manifest.ResultsManifest`)
"""
manifest_data = {}
path_to_manifest_file = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH,
'results_manifest.json')
if os.path.exists(path_to_manifest_file):
logger.info('Opening results manifest...')
with open(path_to_manifest_file, 'r') as manifest_file:
manifest_data = json.loads(manifest_file.read())
logger.info('Results manifest:')
logger.info(manifest_data)
else:
logger.info('No results manifest found')
results_manifest = ResultsManifest(manifest_data)
stdout_files = self._get_artifacts_from_stdout(stdoutAndStderr)
results_manifest.add_files(stdout_files)
results_manifest.validate(self._output_file_manifest_dict)
files_to_store = {}
for manifest_file_entry in results_manifest.get_files():
param_name = manifest_file_entry['name']
media_type = None
output_data_item = self._get_output_data_item_by_name(param_name)
if output_data_item:
media_type = output_data_item.get('media_type')
msg = 'Output %s has invalid/missing file path "%s"'
if 'file' in manifest_file_entry:
file_entry = manifest_file_entry['file']
if not os.path.isfile(file_entry['path']):
raise InvalidResultsManifest(
msg % (param_name, file_entry['path']))
if 'geo_metadata' in file_entry:
files_to_store[param_name] = (file_entry['path'],
media_type,
file_entry['geo_metadata'])
else:
files_to_store[param_name] = (file_entry['path'],
media_type)
elif 'files' in manifest_file_entry:
file_tuples = []
for file_entry in manifest_file_entry['files']:
if not os.path.isfile(file_entry['path']):
raise InvalidResultsManifest(
msg % (param_name, file_entry['path']))
if 'geo_metadata' in file_entry:
file_tuples.append((file_entry['path'], media_type,
file_entry['geo_metadata']))
else:
file_tuples.append((file_entry['path'], media_type))
files_to_store[param_name] = file_tuples
job_data_parse_results = {} # parse results formatted for job_data
for parse_result in results_manifest.get_parse_results():
filename = parse_result['filename']
assert filename not in job_data_parse_results
geo_metadata = parse_result.get('geo_metadata', {})
geo_json = geo_metadata.get('geo_json', None)
data_started = geo_metadata.get('data_started', None)
data_ended = geo_metadata.get('data_ended', None)
data_types = parse_result.get('data_types', [])
new_workspace_path = parse_result.get('new_workspace_path', None)
if new_workspace_path:
new_workspace_path = os.path.join(new_workspace_path, filename)
job_data_parse_results[filename] = (geo_json, data_started,
data_ended, data_types,
new_workspace_path)
job_data.save_parse_results(job_data_parse_results)
return (job_data.store_output_data_files(files_to_store,
job_exe), results_manifest)