def _validate_string_from_list(key, value, validation_rule):
if not isinstance(value, str):
return ValidationResult(False, f'"{key}" input must be a string', None)
if value not in validation_rule.options:
formatted_options = ", ".join([f'"{option}"' for option in validation_rule.options])
return ValidationResult(False, f'"{key}" input must be one of: {formatted_options}', None)
return ValidationResult(True, None, value)
def _validate_byte_quantity(key, value, validation_rule):
wrong_format_result = ValidationResult(
False,
f'"{key}" input must be a string integer value ending in "m" (e.g. "512m" for 512 megabytes)',
None
)
if not isinstance(value, str):
return wrong_format_result
if is_str_empty(value):
return wrong_format_result
if value[-1] != "m":
return wrong_format_result
try:
numerical_value = int(value[:-1])
except ValueError:
return wrong_format_result
except TypeError:
return wrong_format_result
return ValidationResult(True, None, value) \
if validation_rule.minimum <= numerical_value <= validation_rule.maximum \
else ValidationResult(
False,
f'"{key}" input must be in range [{validation_rule.minimum}m, {validation_rule.maximum}m]',
None
)
def _validate_label(value):
validation_result = _validate_non_empty_string("label", value)
if not validation_result.is_valid:
return validation_result
if len(value) > 63:
# Kubernetes spark operator restriction
return ValidationResult(False, '"label" input has a maximum length of 63 characters', None)
match = re.match('^[0-9A-Za-z]([0-9A-Za-z\\-]*)?[0-9A-Za-z]$|^[0-9A-Za-z]$', value)
if match is None:
return ValidationResult(False, f'"label" input must obey naming convention: '
f'see https://github.com/ukaea/piezo/wiki/WebAppUserGuide#submit-a-job', None)
return ValidationResult(True, None, value)
def test_submit_job_returns_ui_url_as_unavailable_if_failure_in_setup(
self):
# Arrange
body = {'name': 'test-spark-job', 'language': 'example-language'}
self.mock_validation_service.validate_request_keys.return_value = ValidationResult(
True, "", None)
self.mock_validation_service.validate_request_values.return_value = ValidationResult(
True, "", body)
self.mock_spark_job_customiser.rename_job.return_value = 'test-spark-job-abcd1234'
self.mock_spark_ui_service.expose_spark_ui.return_value = "Unavailable"
# Act
response = self.test_service.submit_job(body)
# Assert)
assert response['spark_ui'] == 'Unavailable'
def _validate_integer(key, value, validation_rule):
try:
numerical_value = int(value)
except ValueError:
return ValidationResult(False, f'"{key}" input must be an integer', None)
except TypeError:
return ValidationResult(False, f'"{key}" input must be an integer', None)
return ValidationResult(True, None, numerical_value) \
if validation_rule.minimum <= numerical_value <= validation_rule.maximum \
else ValidationResult(
False,
f'"{key}" input must be in range [{validation_rule.minimum}, {validation_rule.maximum}]',
None
)
def test_submit_job_calls_spark_ui_service_correctly_to_expose_ui(self):
# Arrange
body = {'name': 'test-spark-job', 'language': 'example-language'}
self.mock_validation_service.validate_request_keys.return_value = ValidationResult(
True, "", None)
self.mock_validation_service.validate_request_values.return_value = ValidationResult(
True, "", body)
self.mock_spark_job_customiser.rename_job.return_value = 'test-spark-job-abcd1234'
self.mock_spark_ui_service.expose_spark_ui.return_value = "some.url"
# Act
response = self.test_service.submit_job(body)
# Assert
self.mock_spark_ui_service.expose_spark_ui.assert_called_once_with(
'test-spark-job-abcd1234')
assert response['spark_ui'] == 'some.url'
def test_submit_job_returns_invalid_body_values(self):
# Arrange
body = {'name': 'test-spark-job', 'language': 'example-language'}
self.mock_validation_service.validate_request_keys.return_value = ValidationResult(
True, "", None)
self.mock_validation_service.validate_request_values.return_value = ValidationResult(
False, "Msg", None)
# Act
result = self.test_service.submit_job(body)
# Assert
self.mock_kubernetes_adapter.create_namespaced_custom_object.assert_not_called(
)
self.assertDictEqual(result, {
'status': StatusCodes.Bad_request.value,
'message': 'Msg'
})
def validate_request_keys(self, request_body):
is_valid = True
error_msg = "The following errors were found:\n"
# Get keys required/supported
required_keys = self._validation_ruleset.get_keys_of_required_inputs()
if 'language' in request_body:
language = request_body['language']
rule = self._validation_ruleset.get_validation_rule_for_key('language')
validation_result = validate('language', language, rule)
if validation_result.is_valid:
required_keys += self._validation_ruleset.get_keys_for_language(request_body['language'])
else:
is_valid = False
error_msg += f'Unsupported language "{language}" provided\n'
supported_keys = required_keys + self._validation_ruleset.get_keys_of_optional_inputs()
# Find any discrepancies
missing_keys = get_set_difference(required_keys, request_body)
unsupported_keys = get_set_difference(request_body, supported_keys)
# Group the results together
for key in missing_keys:
is_valid = False
error_msg += f'Missing required input "{key}"\n'
for key in unsupported_keys:
is_valid = False
error_msg += f'Unsupported input "{key}" provided\n'
result = ValidationResult(
is_valid,
"All input keys provided are valid\n" if is_valid else error_msg,
None
)
return result
def test_set_output_dir_as_first_argument_prepends_dir_to_other_arguments(
self):
# Arrange
mock_storage_service = mock.create_autospec(IStorageService)
mock_storage_service.protocol_route_to_bucket.return_value = 's3a://test-bucket'
validated_body_values = ValidationResult(True, None, {
'other': 'True',
'another': 12,
'arguments': ['1st', '2nd']
})
# Act
result = self.test_customiser.set_output_dir_as_first_argument(
'example-job', mock_storage_service, validated_body_values)
# Assert
self.mock_logger.debug.assert_called_once_with(
'Setting first argument for job "example-job" to be "s3a://test-bucket/outputs/example-job/"'
)
self.assertDictEqual(
result.validated_value, {
'other':
'True',
'another':
12,
'arguments':
['s3a://test-bucket/outputs/example-job/', '1st', '2nd']
})
def _validate_multiple_of_a_tenth(key, value, validation_rule):
not_a_tenth_result = ValidationResult(False, f'"{key}" input must be a multiple of 0.1', None)
try:
numerical_value = float(value)
except ValueError:
return not_a_tenth_result
except TypeError:
return not_a_tenth_result
multiples_of_a_tenth = 10 * numerical_value
if multiples_of_a_tenth % 1 != 0:
return not_a_tenth_result
return ValidationResult(True, None, numerical_value) \
if validation_rule.minimum <= numerical_value <= validation_rule.maximum \
else ValidationResult(
False,
f'"{key}" input must be in range [{validation_rule.minimum}, {validation_rule.maximum}]',
None
)
def test_submit_job_logs_and_returns_api_exception_reason(self):
# Arrange
body = {'name': 'test-spark-job', 'language': 'example-language'}
self.mock_validation_service.validate_request_keys.return_value = ValidationResult(
True, "", None)
self.mock_validation_service.validate_request_values.return_value = ValidationResult(
True, "", body)
self.mock_spark_job_customiser.rename_job.return_value = 'test-spark-job-abcd1234'
manifest = {
'metadata': {
'namespace': NAMESPACE,
'name': 'test-spark-job-abcd1234',
'language': 'example-language'
}
}
self.mock_manifest_populator.build_manifest.return_value = manifest
self.mock_kubernetes_adapter.create_namespaced_custom_object.side_effect = \
ApiException(reason="Reason", status=999)
# Act
result = self.test_service.submit_job(body)
# Assert
self.mock_kubernetes_adapter.create_namespaced_custom_object.assert_called_once_with(
CRD_GROUP, CRD_VERSION, NAMESPACE, CRD_PLURAL, manifest)
expected_message = 'Kubernetes error when trying to submit job: Reason'
self.mock_logger.error.assert_has_calls([
mock.call(expected_message),
mock.call({
'metadata': {
'namespace': NAMESPACE,
'name': 'test-spark-job-abcd1234',
'language': 'example-language'
}
})
])
self.assertDictEqual(result, {
'status': 999,
'message': expected_message
})
def test_submit_job_sends_expected_arguments(self):
# Arrange
body = {'name': 'test-spark-job', 'language': 'example-language'}
self.mock_validation_service.validate_request_keys.return_value = ValidationResult(
True, "", None)
self.mock_validation_service.validate_request_values.return_value = ValidationResult(
True, "", body)
self.mock_spark_job_customiser.rename_job.return_value = 'test-spark-job-abcd1234'
manifest = {
'metadata': {
'namespace': NAMESPACE,
'name': 'test-spark-job-abcd1234',
'language': 'example-language'
}
}
self.mock_manifest_populator.build_manifest.return_value = manifest
self.mock_kubernetes_adapter.create_namespaced_custom_object.return_value = {
'metadata': {
'namespace': NAMESPACE,
'name': 'test-spark-job-abcd1234',
'language': 'example-language'
}
}
self.mock_spark_ui_service.expose_spark_ui.return_value = 'some_url'
# Act
result = self.test_service.submit_job(body)
# Assert
self.mock_kubernetes_adapter.create_namespaced_custom_object.assert_called_once_with(
CRD_GROUP, CRD_VERSION, NAMESPACE, CRD_PLURAL, manifest)
self.assertDictEqual(
result, {
'status': StatusCodes.Okay.value,
'message': 'Job driver created successfully',
'job_name': 'test-spark-job-abcd1234',
'spark_ui': 'some_url'
})
def validate(key, value, validation_rule):
if key == "name":
return _validate_name(key, value)
if key in ["path_to_main_app_file", "main_class"]:
return _validate_non_empty_string(key, value)
if key in ["label"]:
return _validate_label(value)
if key in ["language", "python_version"]:
return _validate_string_from_list(key, value, validation_rule)
if key in ["executors", "executor_cores"]:
return _validate_integer(key, value, validation_rule)
if key in ["driver_cores", "driver_core_limit"]:
return _validate_multiple_of_a_tenth(key, value, validation_rule)
if key in ["driver_memory", "executor_memory"]:
return _validate_byte_quantity(key, value, validation_rule)
if key in ["arguments"]:
return ValidationResult(True, None, value)
raise ValueError(f"Unexpected argument {key}")
def validate_request_values(self, request_body):
validated_dict = {}
error_msg = "The following errors were found:\n"
is_valid = True
for key, input_value in request_body.items():
rule_for_key = self._validation_ruleset.get_validation_rule_for_key(key)
validation_result = validate(key, input_value, rule_for_key)
if validation_result.is_valid is True:
validated_dict[key] = validation_result.validated_value
else:
error_msg += validation_result.message + '\n'
is_valid = False
result = ValidationResult(
is_valid,
"All inputs provided are valid\n" if is_valid else error_msg,
validated_dict if is_valid else None
)
return result
def _validate_name(key, value):
validation_result = _validate_non_empty_string("name", value)
if not validation_result.is_valid:
return validation_result
is_name_valid = True
if len(value) == 1:
is_name_valid = value in string.ascii_lowercase
elif 1 < len(value) <= 29:
match = re.match("^([a-z])([\\.\\-0-9a-z]*)?([0-9a-z])$", value)
if match is None:
is_name_valid = False
for pattern in ["--", "-.", ".-", ".."]:
if pattern in value:
is_name_valid = False
else:
is_name_valid = False
msg = None if is_name_valid else f'"{key}" input must obey naming convention: ' \
f'see https://github.com/ukaea/piezo/wiki/WebAppUserGuide#submit-a-job'
return ValidationResult(is_name_valid, msg, value)
def _validate_non_empty_string(key, value):
if not isinstance(value, str):
return ValidationResult(False, f'"{key}" input must be a string', None)
if is_str_empty(value):
return ValidationResult(False, f'"{key}" input cannot be empty', None)
return ValidationResult(True, None, value)