def validate_properties(self, property_names):
"""Validates the given property names to ensure they are all populated correctly and exist if they are required.
:param property_names: Dict of property names mapped to a bool indicating if they are required
:type property_names: dict of str -> bool
:returns: A list of warnings discovered during validation.
:rtype: list[:class:`job.configuration.data.job_data.ValidationWarning`]
:raises :class:`job.configuration.data.exceptions.InvalidData`: If there is a configuration problem.
"""
warnings = []
for name in property_names:
if name in self.data_inputs_by_name:
# Have this input, make sure it is a valid property
property_input = self.data_inputs_by_name[name]
if not 'value' in property_input:
msg = 'Invalid job data: Data input %s is a property and must have a "value" field' % name
raise InvalidData(msg)
value = property_input['value']
if not isinstance(value, basestring):
raise InvalidData('Invalid job data: Data input %s must have a string in its "value" field' % name)
else:
# Don't have this input, check if it is required
if property_names[name]:
raise InvalidData('Invalid job data: Data input %s is required and was not provided' % name)
return warnings
def validate_input_json(self, input_json):
"""Validates the given property names to ensure they are all populated correctly and exist if they are required.
:param input_json: List of Seed input json fields
:type input_json: [:class:`job.seed.types.SeedInputJson`]
:returns: A list of warnings discovered during validation.
:rtype: [:class:`job.configuration.data.job_data.ValidationWarning`]
:raises :class:`job.configuration.data.exceptions.InvalidData`: If there is a configuration problem.
"""
warnings = []
for in_js in input_json:
name = in_js.name
if name in self._new_data.values:
if isinstance(self._new_data.values[name], JsonValue):
# Have this input, make sure it is a valid property
property_input = self._new_data.values[name]
value = property_input.value
if not isinstance(value, in_js.python_type):
msg = 'Invalid job data: Data input %s must have a json type %s in its "value" field'
raise InvalidData(msg % (name, in_js.type))
else:
# Don't have this input, check if it is required
if in_js.required:
raise InvalidData(
'Invalid job data: Data input %s is required and was not provided'
% name)
return warnings
def fully_populate_command_argument(self, job_data, job_environment,
job_exe_id):
"""Return a fully populated command arguments string. If pre-steps are necessary
(see are_pre_steps_needed), they should be run before this. populated with information
from the job_data, job_environment, job_input_dir, and job_output_dir.This gets the properties and
input_files from the job_data, the shared_resources from the job_environment, and ${input}
${output_dir} from the work_dir.
Throws a :class:`job.configuration.interface.exceptions.InvalidEnvironment` if the necessary
pre-steps have not been performed
:param job_data: The job data
:type job_data: :class:`job.configuration.data.job_data.JobData`
:param job_environment: The job environment
:type job_environment: dict
:param job_exe_id: The job execution ID
:type job_exe_id: int
"""
# TODO: don't ignore job_envirnoment
command_arguments = self.populate_command_argument_properties(job_data)
param_replacements = {}
for input_data in self.definition['input_data']:
input_name = input_data['name']
input_type = input_data['type']
input_required = input_data['required']
if input_type == 'file':
param_dir = os.path.join(SCALE_JOB_EXE_INPUT_PATH, input_name)
if os.path.isdir(param_dir):
file_path = self._get_one_file_from_directory(param_dir)
param_replacements[input_name] = file_path
elif input_required:
raise InvalidData(
'Unable to create run command. Expected required file in %s'
% param_dir)
else:
param_replacements[input_name] = ''
elif input_type == 'files':
param_dir = os.path.join(SCALE_JOB_EXE_INPUT_PATH, input_name)
if os.path.isdir(param_dir):
param_replacements[input_name] = param_dir
elif input_required:
raise InvalidData(
'Unable to create run command. Expected required files in %s'
% param_dir)
else:
param_replacements[input_name] = ''
param_replacements['job_output_dir'] = SCALE_JOB_EXE_OUTPUT_PATH
command_arguments = self._replace_command_parameters(
command_arguments, param_replacements)
# Remove extra whitespace
command_arguments = ' '.join(command_arguments.split())
return command_arguments
def __init__(self, data=None):
"""Creates a job data object from the given dictionary. The general format is checked for correctness, but the
actual input and output details are not checked for correctness against the job interface. If the data is
invalid, a :class:`job.configuration.data.exceptions.InvalidData` will be thrown.
:param data: The job data
:type data: dict
"""
if not data:
data = {}
self.data_dict = data
self.param_names = set()
self.data_inputs_by_name = {} # string -> dict
self.data_outputs_by_name = {} # string -> dict
if 'version' not in self.data_dict:
self.data_dict['version'] = DEFAULT_VERSION
if not self.data_dict['version'] == '1.0':
raise InvalidData(
'Invalid job data: %s is an unsupported version number' %
self.data_dict['version'])
if 'input_data' not in self.data_dict:
self.data_dict['input_data'] = []
for data_input in self.data_dict['input_data']:
if 'name' not in data_input:
raise InvalidData(
'Invalid job data: Every data input must have a "name" field'
)
name = data_input['name']
if name in self.param_names:
raise InvalidData(
'Invalid job data: %s cannot be defined more than once' %
name)
else:
self.param_names.add(name)
self.data_inputs_by_name[name] = data_input
if 'output_data' not in self.data_dict:
self.data_dict['output_data'] = []
for data_output in self.data_dict['output_data']:
if 'name' not in data_output:
raise InvalidData(
'Invalid job data: Every data output must have a "name" field'
)
name = data_output['name']
if name in self.param_names:
raise InvalidData(
'Invalid job data: %s cannot be defined more than once' %
name)
else:
self.param_names.add(name)
self.data_outputs_by_name[name] = data_output
def validate_output_files(self, files):
'''Validates the given file parameters to make sure they are valid with respect to the job interface.
:param files: List of file parameter names
:type files: list of str
:returns: A list of warnings discovered during validation.
:rtype: list[:class:`job.configuration.data.job_data.ValidationWarning`]
:raises :class:`job.configuration.data.exceptions.InvalidData`: If there is a configuration problem.
'''
warnings = []
workspace_ids = set()
for name in files:
if not name in self.data_outputs_by_name:
raise InvalidData(
u'Invalid job data: Data output %s was not provided' %
name)
file_output = self.data_outputs_by_name[name]
if not u'workspace_id' in file_output:
raise InvalidData(
u'Invalid job data: Data output %s must have a "workspace_id" field'
% name)
workspace_id = file_output[u'workspace_id']
if not isinstance(workspace_id, Integral):
msg = u'Invalid job data: Data output %s must have an integer in its "workspace_id" field' % name
raise InvalidData(msg)
workspace_ids.add(workspace_id)
data_file_store = DATA_FILE_STORE[u'DATA_FILE_STORE']
if not data_file_store:
raise Exception(u'No data file store found')
workspaces = data_file_store.get_workspaces(workspace_ids)
for workspace_id in workspaces:
active = workspaces[workspace_id]
if not active:
raise InvalidData(
u'Invalid job data: Workspace for ID %i is not active' %
workspace_id)
workspace_ids.remove(workspace_id)
# Check if there were any workspace IDs that weren't found
if workspace_ids:
raise InvalidData(
u'Invalid job data: Workspace for ID(s): %s do not exist' %
str(workspace_ids))
return warnings
def _validate_file_ids(self, file_ids, file_desc):
"""Validates the files with the given IDs against the given file description. If invalid, a
:class:`job.configuration.data.exceptions.InvalidData` will be thrown.
:param file_ids: List of file IDs
:type file_ids: [long]
:param file_desc: The description of the required file meta-data for validation
:type file_desc: :class:`job.configuration.interface.scale_file.ScaleFileDescription`
:returns: A list of warnings discovered during validation.
:rtype: [:class:`job.configuration.data.job_data.ValidationWarning`]
:raises :class:`job.configuration.data.exceptions.InvalidData`: If any of the files are missing.
"""
warnings = []
found_ids = set()
for scale_file in ScaleFile.objects.filter(id__in=file_ids):
found_ids.add(scale_file.id)
media_type = scale_file.media_type
if not file_desc.is_media_type_allowed(media_type):
warn = ValidationWarning(
'media_type', 'Invalid media type for file: %i -> %s' %
(scale_file.id, media_type))
warnings.append(warn)
# Check if there were any file IDs that weren't found in the query
for file_id in file_ids:
if file_id not in found_ids:
raise InvalidData(
'Invalid job data: Data file for ID %i does not exist' %
file_id)
return warnings
def validate_input_files(self, files):
"""Validates the given file parameters to make sure they are valid with respect to the job interface.
:param files: List of Seed Input Files
:type files: [:class:`job.seed.types.SeedInputFiles`]
:returns: A list of warnings discovered during validation.
:rtype: [:class:`job.configuration.data.job_data.ValidationWarning`]
:raises :class:`job.configuration.data.exceptions.InvalidData`: If there is a configuration problem.
"""
warnings = []
for input_file in files:
name = input_file.name
required = input_file.required
file_desc = ScaleFileDescription()
if name in self._new_data.values:
if isinstance(self._new_data.values[name], FileValue):
# Have this input, make sure it is valid
file_input = self._new_data.values[name]
file_ids = []
for media_type in input_file.media_types:
file_desc.add_allowed_media_type(media_type)
for file_id in file_input.file_ids:
if not isinstance(file_id, Integral):
msg = (
'Invalid job data: Data input %s must have a list of integers in its "file_ids" '
'field')
raise InvalidData(msg % name)
file_ids.append(long(file_id))
warnings.extend(
self._validate_file_ids(file_ids, file_desc))
else:
# Don't have this input, check if it is required
if required:
raise InvalidData(
'Invalid job data: Data input %s is required and was not provided'
% name)
return warnings
def validate_properties(self, property_names):
"""Validates the given property names to ensure they are all populated correctly and exist if they are required.
:param property_names: Dict of property names mapped to a bool indicating if they are required
:type property_names: {string: bool}
:returns: A list of warnings discovered during validation.
:rtype: [:class:`job.configuration.data.job_data.ValidationWarning`]
:raises :class:`job.configuration.data.exceptions.InvalidData`: If there is a configuration problem.
"""
warnings = []
for name in property_names:
if name in self.data_inputs_by_name:
# Have this input, make sure it is a valid property
property_input = self.data_inputs_by_name[name]
if 'value' not in property_input:
msg = 'Invalid job data: Data input %s is a property and must have a "value" field' % name
raise InvalidData(msg)
value = property_input['value']
if not isinstance(value, basestring):
raise InvalidData(
'Invalid job data: Data input %s must have a string in its "value" field'
% name)
else:
# Don't have this input, check if it is required
if property_names[name]:
raise InvalidData(
'Invalid job data: Data input %s is required and was not provided'
% name)
# Handle extra inputs in the data that are not defined in the interface
for name in list(self.data_inputs_by_name.keys()):
data_input = self.data_inputs_by_name[name]
if 'value' in data_input:
if name not in property_names:
warn = ValidationWarning(
'unknown_input',
'Unknown input %s will be ignored' % name)
warnings.append(warn)
self._delete_input(name)
return warnings
def validate_input_files(self, files):
"""Validates the given file parameters to make sure they are valid with respect to the job interface.
:param files: Dict of file parameter names mapped to a tuple with three items: whether the parameter is required
(True), if the parameter is for multiple files (True), and the description of the expected file meta-data
:type files: dict of str ->
tuple(bool, bool, :class:`job.configuration.interface.scale_file.ScaleFileDescription`)
:returns: A list of warnings discovered during validation.
:rtype: list[:class:`job.configuration.data.job_data.ValidationWarning`]
:raises :class:`job.configuration.data.exceptions.InvalidData`: If there is a configuration problem.
"""
warnings = []
for name in files:
required = files[name][0]
multiple = files[name][1]
file_desc = files[name][2]
if name in self.data_inputs_by_name:
# Have this input, make sure it is valid
file_input = self.data_inputs_by_name[name]
file_ids = []
if multiple:
if not 'file_ids' in file_input:
if 'file_id' in file_input:
file_input['file_ids'] = [file_input['file_id']]
else:
msg = 'Invalid job data: Data input %s is a list of files and must have a "file_ids" or ' \
'"file_id" field'
raise InvalidData(msg % name)
if 'file_id' in file_input:
del file_input['file_id']
value = file_input['file_ids']
if not isinstance(value, list):
msg = 'Invalid job data: Data input %s must have a list of integers in its "file_ids" field'
raise InvalidData(msg % name)
for file_id in value:
if not isinstance(file_id, Integral):
msg = 'Invalid job data: Data input %s must have a list of integers in its "file_ids" ' \
'field'
raise InvalidData(msg % name)
file_ids.append(long(file_id))
else:
if not 'file_id' in file_input:
msg = 'Invalid job data: Data input %s is a file and must have a "file_id" field' % name
raise InvalidData(msg)
if 'file_ids' in file_input:
del file_input['file_ids']
file_id = file_input['file_id']
if not isinstance(file_id, Integral):
msg = 'Invalid job data: Data input %s must have an integer in its "file_id" field' % name
raise InvalidData(msg)
file_ids.append(long(file_id))
warnings.extend(self._validate_file_ids(file_ids, file_desc))
else:
# Don't have this input, check if it is required
if required:
raise InvalidData('Invalid job data: Data input %s is required and was not provided' % name)
return warnings
def test_invalid_args(self, mock_queue):
"""Tests calling the queue status view with invalid job_data for the job."""
mock_queue.side_effect = InvalidData('Invalid args')
json_data = {
'job_type_id': 123,
'job_data': {},
}
url = '/queue/new-job/'
response = self.client.generic('POST', url, json.dumps(json_data), 'application/json')
self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
def _get_one_file_from_directory(dir_path):
"""Checks a directory for one and only one file. If there is not one file, raise a
:exception:`job.configuration.data.exceptions.InvalidData`. If there is one file, this method
returns the full path of that file.
:param dir_path: The directories path
:type dir_path: string
:return: The path to the one file in a given directory
:rtype: str
"""
entries_in_dir = os.listdir(dir_path)
if len(entries_in_dir) != 1:
raise InvalidData('Unable to create run command. Expected one file in %s', dir_path)
return os.path.join(dir_path, entries_in_dir[0])
